From 69db93d77eefe1e2a25ba25d58e15d651fe8a46c Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:47:23 -0400 Subject: [PATCH 001/163] feat(playback): verify qsv/vaapi hardware acceleration with ffmpeg probes Extend the NVENC-style verification gate to QSV and VAAPI so hw_accel=auto resolves to a backend only after listing checks plus a bounded single-frame smoke encode pass on a candidate device. Probes walk each backend's candidate devices in order (NVIDIA nodes excluded from the VAAPI set), honor the configured playback.hw_device, and are cached per ffmpeg identity, backend, and device with the existing singleflight/negative-TTL discipline, bounded by a 30s walk budget covered by the raised capability-endpoint slack. Detection now reports per-backend probe outcomes as detected_backends in the hw-capabilities payload, sharing one walk with resolution so the report and the resolved backend cannot disagree. QSV/VAAPI init-chain construction is consolidated into tonemap.QSVInitDeviceArgs/VAAPIInitDeviceArgs, replacing four separate copies. Phase 1 of the node GPU observability plan. Related issue: #780 Co-Authored-By: Claude Fable 5 --- internal/api/handlers/playback_v3.go | 4 +- internal/api/handlers/system.go | 19 +- internal/api/router.go | 6 +- internal/chapterthumbs/extractor.go | 7 +- internal/chapterthumbs/service.go | 2 +- internal/downloads/artifacts.go | 2 +- internal/jellycompat/handlers_playback.go | 8 +- internal/playback/encoder_warmup.go | 25 +- internal/playback/encoder_warmup_test.go | 14 +- internal/playback/gpudetect.go | 463 ++++++++++++----- internal/playback/gpudetect_test.go | 598 +++++++++++++++++++--- internal/playback/transcode.go | 22 +- internal/proxy/server.go | 6 +- internal/tonemap/preflight.go | 6 +- internal/tonemap/probe.go | 16 +- internal/tonemap/probe_test.go | 12 +- internal/tonemap/tonemap.go | 15 + internal/transcodenode/server.go | 7 +- 18 files changed, 987 insertions(+), 245 deletions(-) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index bdd3c687d..7b120fcde 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -282,11 +282,11 @@ func (h *PlaybackHandler) transformationRegistryV3(ctx context.Context) *playbac func (h *PlaybackHandler) localToneMapCapabilitiesV3(ctx context.Context) (tonemap.Capabilities, error) { cfg := h.playbackConfig() ffmpegPath := playback.ResolveFFmpegPath(cfg.FFmpegPath) - resolved := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.HWAccel, cfg.FFmpegPath) + hwDevice := strings.TrimSpace(cfg.HWDevice) + resolved := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.HWAccel, cfg.FFmpegPath, hwDevice) if err := ctx.Err(); err != nil { return nil, err } - hwDevice := strings.TrimSpace(cfg.HWDevice) probe := tonemap.Probe if h.v3ToneMapProbe != nil { probe = h.v3ToneMapProbe diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index dd89e61ca..7d89e010e 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -20,15 +20,21 @@ type SystemHandler struct { transcodePool *nodepool.TranscodePool jwtSecret string ffmpegPath string + hwAccel string + hwDevice string buildInfo buildinfo.Info } -// NewSystemHandler creates a SystemHandler. -func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string, ffmpegPath string) *SystemHandler { +// NewSystemHandler creates a SystemHandler. hwAccel and hwDevice are the +// configured playback settings, so a local probe verifies the same backend and +// devices this host would transcode on. +func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret, ffmpegPath, hwAccel, hwDevice string) *SystemHandler { return &SystemHandler{ transcodePool: transcodePool, jwtSecret: jwtSecret, ffmpegPath: ffmpegPath, + hwAccel: hwAccel, + hwDevice: hwDevice, buildInfo: buildinfo.Current(), } } @@ -67,7 +73,7 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if len(healthy) == 0 { - writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: playback.DetectHWAccelWithFFmpeg(h.ffmpegPath)}) + writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: h.localHWAccel()}) return } @@ -109,11 +115,16 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if !primaried { - inventory.HWAccelInfo = playback.DetectHWAccelWithFFmpeg(h.ffmpegPath) + inventory.HWAccelInfo = h.localHWAccel() } writeJSON(w, http.StatusOK, inventory) } +// localHWAccel probes this host against its configured playback settings. +func (h *SystemHandler) localHWAccel() playback.HWAccelInfo { + return playback.DetectHWAccelWithFFmpeg(h.hwAccel, h.ffmpegPath, h.hwDevice) +} + // HandleBuildInfo handles GET /admin/system/build. func (h *SystemHandler) HandleBuildInfo(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, h.buildInfo) diff --git a/internal/api/router.go b/internal/api/router.go index 243ef26e5..bd956b20c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3172,11 +3172,15 @@ func NewRouter(deps Dependencies) chi.Router { { sysJWTSecret := "" sysFFmpegPath := "" + sysHWAccel := "" + sysHWDevice := "" if deps.Config != nil { sysJWTSecret = deps.Config.Auth.JWTSecret sysFFmpegPath = deps.Config.Playback.FFmpegPath + sysHWAccel = deps.Config.Playback.HWAccel + sysHWDevice = deps.Config.Playback.HWDevice } - systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath) + systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath, sysHWAccel, sysHWDevice) r.Route("/system", func(r chi.Router) { r.Get("/build", systemHandler.HandleBuildInfo) r.Get("/hw-accel", systemHandler.HandleHWAccel) diff --git a/internal/chapterthumbs/extractor.go b/internal/chapterthumbs/extractor.go index 98de48eb6..bd5886a61 100644 --- a/internal/chapterthumbs/extractor.go +++ b/internal/chapterthumbs/extractor.go @@ -183,7 +183,7 @@ func ExtractFrame(ctx context.Context, opts FrameExtractOptions) ([]byte, string softwareToneMapResolver: softwareToneMapResolver, } - resolvedAccel := playback.ResolveHWAccelWithFFmpeg(opts.HWAccel, ffmpegPath) + resolvedAccel := playback.ResolveHWAccelWithFFmpeg(opts.HWAccel, ffmpegPath, opts.HWDevice) if supportsHardwareFrameExtract(resolvedAccel) { // Resolve a multi-device hw_device list to one concrete GPU for this // extraction; the reservation spans only the hardware attempt below. @@ -386,9 +386,8 @@ func buildFrameExtractArgs(inputPath string, seekSeconds float64, hwAccel string if hwDevice == "" { return nil, fmt.Errorf("qsv requires a render device") } + args = append(args, tonemap.QSVInitDeviceArgs(hwDevice)...) args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=va:%s,driver=iHD,kernel_driver=i915,vendor_id=0x8086", hwDevice), - "-init_hw_device", "qsv=qs@va", "-filter_hw_device", "va", "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", @@ -397,8 +396,8 @@ func buildFrameExtractArgs(inputPath string, seekSeconds float64, hwAccel string if hwDevice == "" { return nil, fmt.Errorf("vaapi requires a render device") } + args = append(args, tonemap.VAAPIInitDeviceArgs("hw", hwDevice)...) args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=hw:%s", hwDevice), "-filter_hw_device", "hw", "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", diff --git a/internal/chapterthumbs/service.go b/internal/chapterthumbs/service.go index e1d7cc12e..7a08c3141 100644 --- a/internal/chapterthumbs/service.go +++ b/internal/chapterthumbs/service.go @@ -665,7 +665,7 @@ func (s *Service) extractFrameLocal( func (s *Service) resolveHWConfig() (string, string) { s.hwResolveOnce.Do(func() { - s.resolvedHWAccel = playback.ResolveHWAccelWithFFmpeg(s.hwAccel, s.ffmpegPath) + s.resolvedHWAccel = playback.ResolveHWAccelWithFFmpeg(s.hwAccel, s.ffmpegPath, s.hwDevice) }) // The configured device value passes through raw: ExtractFrame resolves it // (multi-device balancing, empty-value auto-detection) per extraction. diff --git a/internal/downloads/artifacts.go b/internal/downloads/artifacts.go index 67c040faf..edb76a836 100644 --- a/internal/downloads/artifacts.go +++ b/internal/downloads/artifacts.go @@ -503,7 +503,7 @@ func (m *ArtifactManager) localToneMapCapabilities(ctx context.Context) (tonemap if cfg == nil { return nil, nil } - backend := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.Playback.HWAccel, cfg.Playback.FFmpegPath) + backend := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.Playback.HWAccel, cfg.Playback.FFmpegPath, cfg.Playback.HWDevice) if err := ctx.Err(); err != nil { return nil, err } diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 08cc59c1e..f1ca50b20 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -443,14 +443,14 @@ func resolveCompatToneMapRecipeWithPolicy(file *models.MediaFile, capabilities t // localToneMapCapabilities probes the API host's live FFmpeg backend and device. func (h *PlaybackHandler) localToneMapCapabilities(ctx context.Context) (tonemap.Capabilities, error) { - backend := playback.ResolveHWAccelWithFFmpegContext(ctx, h.HWAccel, h.FFmpegPath) - if err := ctx.Err(); err != nil { - return nil, err - } hwDevice := "" if h.cfg != nil { hwDevice = h.cfg.Playback.HWDevice } + backend := playback.ResolveHWAccelWithFFmpegContext(ctx, h.HWAccel, h.FFmpegPath, hwDevice) + if err := ctx.Err(); err != nil { + return nil, err + } probe := tonemap.Probe if h.compatToneMapProbe != nil { probe = h.compatToneMapProbe diff --git a/internal/playback/encoder_warmup.go b/internal/playback/encoder_warmup.go index 6979f2634..fc3d6a2d2 100644 --- a/internal/playback/encoder_warmup.go +++ b/internal/playback/encoder_warmup.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/Silo-Server/silo-server/internal/tonemap" "golang.org/x/sync/singleflight" ) @@ -52,7 +53,7 @@ func warmHardwareEncoderCached( ctx context.Context, ffmpegPath, configuredHWAccel, configuredHWDevice string, state *hardwareEncoderWarmupState, - resolve func(context.Context, string, string) string, + resolve func(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string, run hardwareEncoderWarmupRunner, ) error { if ctx == nil { @@ -62,12 +63,12 @@ func warmHardwareEncoderCached( return nil } ffmpegPath = ResolveFFmpegPath(ffmpegPath) - backend := resolve(ctx, configuredHWAccel, ffmpegPath) + backend := resolve(ctx, configuredHWAccel, ffmpegPath, configuredHWDevice) if backend == "" || backend == HWAccelNone { return nil } devices := hardwareEncoderWarmupDevices(backend, configuredHWDevice) - cacheKey := nvencProbeCacheKey(ffmpegPath) + "\x00" + backend + "\x00" + strings.Join(devices, ",") + cacheKey := ffmpegIdentityKey(ffmpegPath) + "\x00" + backend + "\x00" + strings.Join(devices, ",") now := time.Now() state.Lock() entry, ok := state.entries[cacheKey] @@ -109,7 +110,7 @@ func warmHardwareEncoderWithRunner(ctx context.Context, timeout time.Duration, f } var result error for _, device := range devices { - args := hardwareEncoderWarmupArgs(backend, device) + args := hardwareSmokeEncodeArgs(backend, device) if len(args) == 0 { continue } @@ -155,23 +156,25 @@ func hardwareEncoderWarmupDevices(backend, configured string) []string { return []string{defaultRenderDevicePath} } -func hardwareEncoderWarmupArgs(backend, device string) []string { +// hardwareSmokeEncodeArgs builds the one-frame synthetic encode shared by +// encoder warmup and the auto-detection capability probes: the same +// initialization chain a real transcode uses, driven by testsrc2 so no media +// file is required. +func hardwareSmokeEncodeArgs(backend, device string) []string { base := []string{ffmpegHideBannerArg, ffmpegLogLevelArg, ffmpegErrorLogLevel} switch backend { case transcodeHWQSV: if strings.TrimSpace(device) == "" { device = defaultRenderDevicePath } - base = append(base, - "-init_hw_device", qsvVAAPIInitDevice(device), - "-init_hw_device", "qsv=qs@va", - "-filter_hw_device", "qs", - ) + base = append(base, tonemap.QSVInitDeviceArgs(device)...) + base = append(base, "-filter_hw_device", "qs") case transcodeHWVAAPI: if strings.TrimSpace(device) == "" { device = defaultRenderDevicePath } - base = append(base, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va") + base = append(base, tonemap.VAAPIInitDeviceArgs(vaapiHWDeviceAlias, device)...) + base = append(base, "-filter_hw_device", vaapiHWDeviceAlias) case transcodeHWNVENC: if strings.TrimSpace(device) != "" { base = append(base, "-init_hw_device", "cuda=cu:"+device, "-filter_hw_device", "cu") diff --git a/internal/playback/encoder_warmup_test.go b/internal/playback/encoder_warmup_test.go index 66913a27e..d5390ff80 100644 --- a/internal/playback/encoder_warmup_test.go +++ b/internal/playback/encoder_warmup_test.go @@ -10,7 +10,7 @@ import ( "time" ) -func TestHardwareEncoderWarmupArgs(t *testing.T) { +func TestHardwareSmokeEncodeArgs(t *testing.T) { tests := []struct { name string backend string @@ -31,7 +31,7 @@ func TestHardwareEncoderWarmupArgs(t *testing.T) { backend: transcodeHWVAAPI, device: "/dev/dri/renderD130", contains: []string{ - "vaapi=va:/dev/dri/renderD130", "format=nv12,hwupload", "h264_vaapi", + "vaapi=hw:/dev/dri/renderD130", "format=nv12,hwupload", "h264_vaapi", }, }, { @@ -43,18 +43,18 @@ func TestHardwareEncoderWarmupArgs(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - args := hardwareEncoderWarmupArgs(test.backend, test.device) + args := hardwareSmokeEncodeArgs(test.backend, test.device) for _, want := range test.contains { if !slices.Contains(args, want) { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) = %v, missing %q", test.backend, test.device, args, want) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) = %v, missing %q", test.backend, test.device, args, want) } } framesIndex := slices.Index(args, "-frames:v") if framesIndex < 0 || framesIndex+1 >= len(args) || args[framesIndex+1] != "1" { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) = %v, want one output frame", test.backend, test.device, args) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) = %v, want one output frame", test.backend, test.device, args) } if got := args[len(args)-3:]; !slices.Equal(got, []string{"-f", "null", "-"}) { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) tail = %v, want null sink", test.backend, test.device, got) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) tail = %v, want null sink", test.backend, test.device, got) } }) } @@ -134,7 +134,7 @@ func TestWarmHardwareEncoderCachedSharesSuccessfulWarmupPastCallerCancellation(t return nil, ctx.Err() } } - resolve := func(context.Context, string, string) string { return transcodeHWNVENC } + resolve := func(context.Context, string, string, string) string { return transcodeHWNVENC } callerCtx, cancel := context.WithCancel(context.Background()) result := make(chan error, 1) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index b00a456c1..5e47dcda2 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -23,26 +23,45 @@ var ( defaultNVIDIADeviceGlob = "/dev/nvidia[0-9]*" sysClassDRMDir = "/sys/class/drm" currentGOOS = runtime.GOOS - nvencProbeCommandTimeout = 3 * time.Second - nvencProbeNegativeTTL = 15 * time.Second + hwProbeCommandTimeout = 3 * time.Second + hwProbeNegativeTTL = 15 * time.Second + // hwProbeNow is the cache clock; tests advance it instead of sleeping. + hwProbeNow = time.Now ) -type nvencProbeResult struct { +// hwProbeResult records whether one backend was verified end to end on this +// host. reason is populated only for a failure and is operator-facing. +type hwProbeResult struct { available bool reason string } -type nvencProbeCacheEntry struct { - result nvencProbeResult +type hwProbeCacheEntry struct { + result hwProbeResult expiresAt time.Time } -var nvencProbeCache = struct { +var hwProbeCache = struct { sync.Mutex - byPath map[string]nvencProbeCacheEntry - group singleflight.Group + entries map[string]hwProbeCacheEntry + group singleflight.Group }{ - byPath: make(map[string]nvencProbeCacheEntry), + entries: make(map[string]hwProbeCacheEntry), +} + +// DetectedBackend reports one hardware backend that has candidate devices on +// this host, together with the outcome of its FFmpeg verification probe. +type DetectedBackend struct { + Backend string `json:"backend"` + // Verified reports whether at least one candidate device passed its probe. + Verified bool `json:"verified"` + // Devices lists every candidate considered for this backend, in probe order. + Devices []string `json:"devices,omitempty"` + // Device is the candidate whose probe passed. NVENC addresses its GPU + // through the CUDA runtime, so it stays empty there even when verified. + Device string `json:"device,omitempty"` + // Reason explains a failure, attributed per device when several were tried. + Reason string `json:"reason,omitempty"` } // HWAccelInfo describes the detected hardware acceleration capability. @@ -51,6 +70,7 @@ type HWAccelInfo struct { RenderDevices []string `json:"render_devices"` RenderDeviceDetails []RenderDeviceInfo `json:"render_device_details"` IntelDetected bool `json:"intel_detected"` + DetectedBackends []DetectedBackend `json:"detected_backends,omitempty"` Source string `json:"source"` NodeURL string `json:"node_url,omitempty"` Transformations []TransformationV3 `json:"transformations,omitempty"` @@ -82,29 +102,35 @@ func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Dur // DetectHWAccel probes this host's GPU hardware and returns structured info. func DetectHWAccel() HWAccelInfo { - return DetectHWAccelWithFFmpeg("") + return DetectHWAccelWithFFmpeg("auto", "", "") } // DetectHWAccelWithFFmpeg probes this host's GPU hardware and configured FFmpeg. -func DetectHWAccelWithFFmpeg(ffmpegPath string) HWAccelInfo { - return DetectHWAccelWithFFmpegContext(context.Background(), ffmpegPath) +func DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { + return DetectHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath, hwDevice) } -// DetectHWAccelWithFFmpegContext probes this host without outliving ctx. -func DetectHWAccelWithFFmpegContext(ctx context.Context, ffmpegPath string) HWAccelInfo { - devices := listRenderDevices(defaultDRIDir) - intel := false - for _, d := range devices { - if isIntelDevice(d) { - intel = true - break - } +// DetectHWAccelWithFFmpegContext probes this host without outliving ctx. Unlike +// resolution it verifies every backend with candidate hardware, so an operator +// sees why a present GPU was not selected. Resolved still honors the +// pass-through contract: an explicitly configured backend wins even when its +// probe failed, and the report carries the failure reason. +func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { + candidates := collectHWCandidates(hwDevice) + resolved := HWAccelNone + var detected []DetectedBackend + if currentGOOS == "linux" { + resolved, detected = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) + } + if configured := strings.TrimSpace(hwAccel); configured != "" && configured != "auto" { + resolved = configured } return HWAccelInfo{ - Resolved: ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpegPath), - RenderDevices: devices, - RenderDeviceDetails: renderDeviceDetails(devices), - IntelDetected: intel, + Resolved: resolved, + RenderDevices: candidates.renderDevices, + RenderDeviceDetails: renderDeviceDetails(candidates.renderDevices), + IntelDetected: candidates.intelPresent, + DetectedBackends: detected, Source: "local", } } @@ -126,114 +152,264 @@ func PickRenderDevice(explicit string) string { return dev } -// ResolveHWAccel resolves "auto" using the default FFmpeg binary. -func ResolveHWAccel(hwAccel string) string { - return ResolveHWAccelWithFFmpeg(hwAccel, "") -} - // ResolveHWAccelWithFFmpeg resolves "auto" into a concrete acceleration method // by probing the system and the configured FFmpeg binary. // Preference order: nvenc > qsv > vaapi > none. // Non-"auto" values are returned unchanged. -func ResolveHWAccelWithFFmpeg(hwAccel string, ffmpegPath string) string { - return ResolveHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath) +// hwDevice is the configured playback.hw_device value; probes run against it so +// verification covers the device a transcode will actually open. +func ResolveHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) string { + return ResolveHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath, hwDevice) } // ResolveHWAccelWithFFmpegContext resolves auto hardware without allowing any // FFmpeg capability probe to outlive ctx. -func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel string, ffmpegPath string) string { +func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string { if hwAccel != "auto" { return hwAccel } if currentGOOS != "linux" { - return "none" + return HWAccelNone } + resolved, _ := walkHWAccelBackends(ctx, ffmpegPath, collectHWCandidates(hwDevice), true) + return resolved +} + +// hwAccelPreferenceOrder is the auto-resolution order; the first backend whose +// probe passes wins. +var hwAccelPreferenceOrder = []string{transcodeHWNVENC, transcodeHWQSV, transcodeHWVAAPI} + +// hwAccelWalkTimeout bounds one full backend walk regardless of how many +// candidate devices a host exposes, so a wedged driver cannot stretch detection +// without limit. tonemap.probeEndpointSlack budgets a capability request for it. +const hwAccelWalkTimeout = 30 * time.Second + +// hwCandidates groups the candidate render devices by the backend each one can +// plausibly drive, before any FFmpeg verification. +type hwCandidates struct { + // renderDevices is this host's full inventory, reported to operators even + // when probes are pinned to a configured subset. + renderDevices []string + nvidia []string + intel []string + vaapi []string + nvidiaPresent bool + // intelPresent describes the inventory rather than the probe set, so a + // pinned non-Intel device does not hide an Intel GPU from operators. + intelPresent bool +} - devices := listRenderDevices(defaultDRIDir) - var intelDevice string - var nvidiaDevice string - var vaapiDevice string - for _, dev := range devices { +// collectHWCandidates enumerates render devices once and classifies them by +// sysfs vendor id. Probes run against the configured playback.hw_device set +// when there is one, because that — not whatever sorts first under /dev/dri — +// is what a transcode opens. NVIDIA hardware also counts when only the control +// device is exposed, which is how NVENC-only containers appear. +func collectHWCandidates(configuredDevice string) hwCandidates { + candidates := hwCandidates{renderDevices: listRenderDevices(defaultDRIDir)} + probeDevices := ParseHWDeviceSet(configuredDevice).List() + if len(probeDevices) == 0 { + probeDevices = candidates.renderDevices + } + for _, device := range probeDevices { switch { - case isNVIDIADevice(dev): - if nvidiaDevice == "" { - nvidiaDevice = dev - } - case isIntelDevice(dev): - if intelDevice == "" { - intelDevice = dev - } + case isNVIDIADevice(device): + // NVIDIA render nodes carry no libva driver, so listing one as a + // VAAPI candidate would only fail a probe another GPU can pass. + candidates.nvidia = append(candidates.nvidia, device) + case isIntelDevice(device): + candidates.intel = append(candidates.intel, device) + candidates.vaapi = append(candidates.vaapi, device) default: - if vaapiDevice == "" { - vaapiDevice = dev - } + candidates.vaapi = append(candidates.vaapi, device) } } + candidates.nvidiaPresent = len(candidates.nvidia) > 0 || hasNVIDIADevice() + candidates.intelPresent = len(candidates.intel) > 0 + for _, device := range candidates.renderDevices { + if candidates.intelPresent { + break + } + candidates.intelPresent = isIntelDevice(device) + } + return candidates +} + +// devicesFor returns the devices a backend may drive. VAAPI is the generic +// fallback, so every non-NVIDIA candidate belongs to it. +func (c hwCandidates) devicesFor(backend string) []string { + switch backend { + case transcodeHWNVENC: + return c.nvidia + case transcodeHWQSV: + return c.intel + case transcodeHWVAAPI: + return c.vaapi + default: + return nil + } +} - if nvidiaDevice != "" || hasNVIDIADevice() { - if ok, reason := ffmpegSupportsNVENCContext(ctx, ffmpegPath); ok { - if nvidiaDevice != "" { - slog.Info("hw_accel=auto: NVIDIA GPU detected, using NVENC", "device", nvidiaDevice) - } else { - slog.Info("hw_accel=auto: NVIDIA device detected, using NVENC") - } - return "nvenc" - } else { - slog.Warn("hw_accel=auto: NVIDIA device detected but FFmpeg NVENC probe failed", - "ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", reason) +// presentFor reports whether a backend has hardware worth probing. +func (c hwCandidates) presentFor(backend string) bool { + if backend == transcodeHWNVENC { + return c.nvidiaPresent + } + return len(c.devicesFor(backend)) > 0 +} + +// probeDevicesFor returns the devices a backend's smoke encode is tried +// against, in order. NVENC addresses its GPU through the CUDA runtime rather +// than a render node, so it probes once with no device path. +func (c hwCandidates) probeDevicesFor(backend string) []string { + if backend == transcodeHWNVENC { + return []string{""} + } + return c.devicesFor(backend) +} + +// walkHWAccelBackends verifies each backend with candidate hardware in +// preference order and reports the first one whose probe passes. Resolution +// stops there; detection continues so every candidate backend is reported. +func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCandidates, stopAtFirstVerified bool) (string, []DetectedBackend) { + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithTimeout(ctx, hwAccelWalkTimeout) + defer cancel() + resolved := "" + var detected []DetectedBackend + for _, backend := range hwAccelPreferenceOrder { + if !candidates.presentFor(backend) { + continue + } + // A caller that has already given up must not start further probes. + if ctx.Err() != nil { + break + } + entry := verifyHWAccelBackend(ctx, backend, ffmpegPath, candidates) + if !entry.Verified { + slog.Warn("hw_accel=auto: candidate hardware failed its FFmpeg probe", + "backend", backend, "devices", entry.Devices, + "ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", entry.Reason) + } else if resolved == "" { + resolved = backend + slog.Info("hw_accel=auto: verified hardware backend", "backend", backend, "device", entry.Device) + } + detected = append(detected, entry) + if resolved != "" && stopAtFirstVerified { + return resolved, detected } } + if resolved == "" { + slog.Info("hw_accel=auto: no verified hardware backend, using software encoding") + return HWAccelNone, detected + } + return resolved, detected +} - if intelDevice != "" { - slog.Info("hw_accel=auto: Intel GPU detected, using QSV", "device", intelDevice) - return "qsv" +// verifyHWAccelBackend probes a backend's candidate devices in order and stops +// at the first one that passes, so a broken GPU sorting ahead of a working one +// does not disable the backend for the whole host. +func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candidates hwCandidates) DetectedBackend { + devices := candidates.probeDevicesFor(backend) + entry := DetectedBackend{Backend: backend, Devices: candidates.devicesFor(backend)} + reasons := make([]string, 0, len(devices)) + for _, device := range devices { + if ctx.Err() != nil { + break + } + available, reason := ffmpegSupportsBackendContext(ctx, backend, ffmpegPath, device) + if available { + entry.Verified = true + entry.Device = device + return entry + } + reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) + } + if len(reasons) == 0 { + reasons = append(reasons, "hardware detection budget exhausted before probing "+backend) } + entry.Reason = strings.Join(reasons, "; ") + return entry +} - if vaapiDevice != "" { - slog.Info("hw_accel=auto: non-Intel GPU detected, using VAAPI", "device", vaapiDevice) - return "vaapi" +// hwProbeFailureReason attributes a failure to its device only when several +// candidates were tried; a single candidate reads better bare. +func hwProbeFailureReason(candidateCount int, device, reason string) string { + if candidateCount < 2 || device == "" { + return reason } + return device + ": " + reason +} - slog.Info("hw_accel=auto: no compatible GPU devices found, using software encoding") - return "none" +// hwBackendProbe verifies one backend against an FFmpeg binary and candidate +// device. commandCount is the number of bounded commands the probe may run and +// budgets the shared deadline. +type hwBackendProbe struct { + commandCount int + run func(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult } -func ffmpegSupportsNVENC(ffmpegPath string) (bool, string) { - return ffmpegSupportsNVENCContext(context.Background(), ffmpegPath) +func hwBackendProbeFor(backend string) (hwBackendProbe, bool) { + switch backend { + case transcodeHWNVENC: + return hwBackendProbe{commandCount: 4, run: probeFFmpegNVENCContext}, true + case transcodeHWQSV: + return hwBackendProbe{commandCount: 3, run: probeFFmpegQSVContext}, true + case transcodeHWVAAPI: + return hwBackendProbe{commandCount: 2, run: probeFFmpegVAAPIContext}, true + default: + return hwBackendProbe{}, false + } } -func ffmpegSupportsNVENCContext(ctx context.Context, ffmpegPath string) (bool, string) { +func ffmpegSupportsBackend(backend, ffmpegPath, device string) (bool, string) { + return ffmpegSupportsBackendContext(context.Background(), backend, ffmpegPath, device) +} + +// ffmpegSupportsBackendContext verifies one backend, coalescing concurrent cold +// probes and reusing a positive result for the process lifetime. A failure is +// retried once its short negative TTL expires, so a driver or binary repaired +// underneath a running server is picked up without a restart. +func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, device string) (bool, string) { if ctx == nil { ctx = context.Background() } + probe, ok := hwBackendProbeFor(backend) + if !ok { + return false, "unsupported hardware backend " + backend + } ffmpegPath = normalizeFFmpegPath(ffmpegPath) - cacheKey := nvencProbeCacheKey(ffmpegPath) - commandTimeout := nvencProbeCommandTimeout - negativeTTL := nvencProbeNegativeTTL - nvencProbeCache.Lock() - if entry, ok := nvencProbeCache.byPath[cacheKey]; ok && nvencProbeCacheEntryCurrent(entry, time.Now()) { - nvencProbeCache.Unlock() + cacheKey := hwProbeCacheKey(ffmpegPath, backend, device) + // The flight below outlives an abandoned caller, so every test-mutable seam + // it touches is snapshotted here rather than dereferenced inside it. + commandTimeout := hwProbeCommandTimeout + negativeTTL := hwProbeNegativeTTL + now := hwProbeNow + hwProbeCache.Lock() + if entry, ok := hwProbeCache.entries[cacheKey]; ok && hwProbeCacheEntryCurrent(entry, now()) { + hwProbeCache.Unlock() return entry.result.available, entry.result.reason } - nvencProbeCache.Unlock() + hwProbeCache.Unlock() - resultCh := nvencProbeCache.group.DoChan(cacheKey, func() (any, error) { - nvencProbeCache.Lock() - cached, ok := nvencProbeCache.byPath[cacheKey] - nvencProbeCache.Unlock() - if ok && nvencProbeCacheEntryCurrent(cached, time.Now()) { + resultCh := hwProbeCache.group.DoChan(cacheKey, func() (any, error) { + hwProbeCache.Lock() + cached, ok := hwProbeCache.entries[cacheKey] + hwProbeCache.Unlock() + if ok && hwProbeCacheEntryCurrent(cached, now()) { return cached.result, nil } - probeCtx, cancel := context.WithTimeout(context.Background(), 4*commandTimeout+time.Second) + probeCtx, cancel := context.WithTimeout(context.Background(), time.Duration(probe.commandCount)*commandTimeout+time.Second) defer cancel() - result := probeFFmpegNVENCContext(probeCtx, ffmpegPath, commandTimeout) - entry := nvencProbeCacheEntry{result: result} + result := probe.run(probeCtx, ffmpegPath, device, commandTimeout) + entry := hwProbeCacheEntry{result: result} if !result.available { - entry.expiresAt = time.Now().Add(negativeTTL) + entry.expiresAt = now().Add(negativeTTL) } - nvencProbeCache.Lock() - nvencProbeCache.byPath[cacheKey] = entry - nvencProbeCache.Unlock() + hwProbeCache.Lock() + hwProbeCache.entries[cacheKey] = entry + hwProbeCache.Unlock() return result, nil }) select { @@ -243,21 +419,27 @@ func ffmpegSupportsNVENCContext(ctx context.Context, ffmpegPath string) (bool, s if shared.Err != nil { return false, shared.Err.Error() } - result, ok := shared.Val.(nvencProbeResult) + result, ok := shared.Val.(hwProbeResult) if !ok { - return false, "invalid shared NVENC probe result" + return false, "invalid shared hardware probe result" } return result.available, result.reason } } -func nvencProbeCacheEntryCurrent(entry nvencProbeCacheEntry, now time.Time) bool { +func hwProbeCacheEntryCurrent(entry hwProbeCacheEntry, now time.Time) bool { return entry.result.available || now.Before(entry.expiresAt) } -// nvencProbeCacheKey invalidates cached capability results when an FFmpeg +// hwProbeCacheKey separates results per backend and per candidate device on top +// of the FFmpeg binary's identity. +func hwProbeCacheKey(ffmpegPath, backend, device string) string { + return strings.Join([]string{ffmpegIdentityKey(ffmpegPath), backend, device}, "\x00") +} + +// ffmpegIdentityKey invalidates cached capability results when an FFmpeg // executable is replaced at the same configured path. -func nvencProbeCacheKey(ffmpegPath string) string { +func ffmpegIdentityKey(ffmpegPath string) string { identityPath := ffmpegPath if !strings.ContainsRune(identityPath, os.PathSeparator) { if resolved, err := exec.LookPath(identityPath); err == nil { @@ -285,44 +467,89 @@ func normalizeFFmpegPath(ffmpegPath string) string { return ffmpegPath } -func probeFFmpegNVENCContext(ctx context.Context, ffmpegPath string, commandTimeout time.Duration) nvencProbeResult { +// probeFFmpegNVENCContext verifies the CUDA decode, scaling, and encode path a +// NVENC transcode depends on. NVENC selects its GPU through CUDA, so the +// candidate device path is not part of the command line. +func probeFFmpegNVENCContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-hwaccels"); err != nil { - return nvencProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} + return hwProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} } else if !ffmpegOutputHasToken(output, "cuda") { - return nvencProbeResult{reason: "cuda hwaccel unavailable"} + return hwProbeResult{reason: "cuda hwaccel unavailable"} } if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { - return nvencProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} } else if !ffmpegOutputHasToken(output, "h264_nvenc") { - return nvencProbeResult{reason: "h264_nvenc encoder unavailable"} + return hwProbeResult{reason: "h264_nvenc encoder unavailable"} } else if !ffmpegOutputHasToken(output, "hevc_nvenc") { - return nvencProbeResult{reason: "hevc_nvenc encoder unavailable"} + return hwProbeResult{reason: "hevc_nvenc encoder unavailable"} } if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-filters"); err != nil { - return nvencProbeResult{reason: "filters probe failed: " + FormatFFmpegProbeFailure(err, output)} + return hwProbeResult{reason: "filters probe failed: " + FormatFFmpegProbeFailure(err, output)} } else if !ffmpegOutputHasToken(output, "scale_cuda") { - return nvencProbeResult{reason: "scale_cuda filter unavailable"} + return hwProbeResult{reason: "scale_cuda filter unavailable"} } else if !ffmpegOutputHasToken(output, "hwupload_cuda") { - return nvencProbeResult{reason: "hwupload_cuda filter unavailable"} + return hwProbeResult{reason: "hwupload_cuda filter unavailable"} } - if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, - "-hide_banner", - "-loglevel", "error", - "-f", "lavfi", - "-i", "testsrc2=size=640x360:rate=1", - "-frames:v", "1", - "-an", - "-c:v", "h264_nvenc", - "-f", "null", - "-", - ); err != nil { - return nvencProbeResult{reason: "h264_nvenc smoke encode failed: " + FormatFFmpegProbeFailure(err, output)} + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWNVENC, device, commandTimeout) +} + +// probeFFmpegQSVContext verifies the VAAPI-derived QSV chain against a +// candidate Intel render device. Either hwaccel listing is enough: the chain +// initializes a VAAPI display and derives the QSV device from it. +func probeFFmpegQSVContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-hwaccels"); err != nil { + return hwProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, transcodeHWQSV) && !ffmpegOutputHasToken(output, transcodeHWVAAPI) { + return hwProbeResult{reason: "qsv and vaapi hwaccels unavailable"} } - return nvencProbeResult{available: true} + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, "h264_qsv") { + return hwProbeResult{reason: "h264_qsv encoder unavailable"} + } else if !ffmpegOutputHasToken(output, "hevc_qsv") { + return hwProbeResult{reason: "hevc_qsv encoder unavailable"} + } + + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWQSV, device, commandTimeout) +} + +// probeFFmpegVAAPIContext verifies the generic VAAPI encode path. VAAPI is the +// last fallback and only needs H.264 encoding to be useful, so the listing gate +// stays narrower than QSV's. +func probeFFmpegVAAPIContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, "h264_vaapi") { + return hwProbeResult{reason: "h264_vaapi encoder unavailable"} + } + + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWVAAPI, device, commandTimeout) +} + +// smokeEncodeResult runs the backend's bounded single-frame encode, which is +// the only step that exercises the driver rather than FFmpeg's build flags. +func smokeEncodeResult(ctx context.Context, ffmpegPath, backend, device string, commandTimeout time.Duration) hwProbeResult { + output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, hardwareSmokeEncodeArgs(backend, device)...) + if err != nil { + return hwProbeResult{reason: hardwareEncoder(backend) + " smoke encode failed: " + FormatFFmpegProbeFailure(err, output)} + } + return hwProbeResult{available: true} +} + +// hardwareEncoder returns the H.264 encoder paired with a backend. +func hardwareEncoder(backend string) string { + switch backend { + case transcodeHWQSV: + return "h264_qsv" + case transcodeHWVAAPI: + return "h264_vaapi" + default: + return "h264_nvenc" + } } func runFFmpegProbe(ctx context.Context, timeout time.Duration, ffmpegPath string, args ...string) ([]byte, error) { diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index e8ac38c20..24ab460d2 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -2,9 +2,11 @@ package playback import ( "context" + "encoding/json" "fmt" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -18,14 +20,25 @@ type hwAccelTestEnv struct { } type fakeFFmpegProbe struct { - cuda bool - h264NVENC bool - hevcNVENC bool - scaleCUDA bool - uploadCUDA bool - smokeOK bool - hang bool - delay time.Duration + cuda bool + qsvHWAccel bool + vaapiHWAccel bool + h264NVENC bool + hevcNVENC bool + h264QSV bool + hevcQSV bool + h264VAAPI bool + scaleCUDA bool + uploadCUDA bool + smokeOK bool + // smokeFailures names encoders whose smoke encode fails even when smokeOK + // is set, modeling a listed encoder with no working driver behind it. + smokeFailures []string + // smokeDeviceFailures names render devices (by basename) whose smoke encode + // fails, modeling one broken GPU on a host that has another working one. + smokeDeviceFailures []string + hang bool + delay time.Duration } type fakeFFmpegBinary struct { @@ -37,9 +50,9 @@ func TestResolveHWAccelWithFFmpegAutoPrefersNVENCOverIntel(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x8086") env.addRenderDevice(t, "renderD129", "0x10de") - ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) } } @@ -48,9 +61,9 @@ func TestResolveHWAccelWithFFmpegFallsBackToIntelWhenNVENCProbeFails(t *testing. env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x8086") env.addRenderDevice(t, "renderD129", "0x10de") - ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "qsv" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "qsv" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv", got) } } @@ -59,19 +72,132 @@ func TestResolveHWAccelWithFFmpegFallsBackToVAAPIWhenNVENCProbeFails(t *testing. env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") env.addRenderDevice(t, "renderD129", "0x1002") - ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "vaapi" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi", got) } } +func TestResolveHWAccelWithFFmpegFallsBackToVAAPIWhenQSVListingFails(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + probe := successfulVAAPIProbe() + probe.qsvHWAccel = true + probe.h264QSV = true + // hevc_qsv is missing, so the QSV listing gate rejects an Intel GPU that + // VAAPI can still drive. + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi", got) + } +} + +func TestResolveHWAccelWithFFmpegTriesEveryCandidateDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + env.addRenderDevice(t, "renderD129", "0x1002") + probe := successfulVAAPIProbe() + // The GPU that sorts first has no working driver; the second one does. + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi from the working device", got) + } + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + vaapi := info.DetectedBackends[len(info.DetectedBackends)-1] + if vaapi.Backend != "vaapi" || !vaapi.Verified { + t.Fatalf("vaapi entry = %+v, want verified", vaapi) + } + if device := filepath.Base(vaapi.Device); device != "renderD129" { + t.Fatalf("verified device = %q, want renderD129", device) + } +} + +func TestResolveHWAccelWithFFmpegSkipsNVIDIANodesAsVAAPIDevices(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x10de") + env.addRenderDevice(t, "renderD129", "0x1002") + probe := successfulVAAPIProbe() + // An NVIDIA render node has no libva driver; probing it would reject a + // backend the AMD card can drive. + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi from the AMD device", got) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), "vaapi=hw:"+filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("VAAPI probe used the NVIDIA render node; log:\n%s", logData) + } +} + +func TestResolveHWAccelWithFFmpegProbesTheConfiguredHWDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + probe := successfulQSVProbe() + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + // The operator pinned the working GPU, which is the device a transcode + // opens; auto resolution has to verify that one rather than renderD128. + pinned := filepath.Join(env.driDir, "renderD129") + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, pinned); got != "qsv" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv on the pinned device", got) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("probe touched an unconfigured device; log:\n%s", logData) + } +} + +func TestDetectHWAccelReportsHostInventoryBehindAPinnedDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, filepath.Join(env.driDir, "renderD129")) + if info.Resolved != "vaapi" { + t.Fatalf("Resolved = %q, want vaapi from the pinned AMD device", info.Resolved) + } + // Pinning a device narrows what is probed, never what is reported. + if !info.IntelDetected { + t.Fatal("IntelDetected = false, want the host's Intel GPU still reported") + } + if len(info.RenderDevices) != 2 { + t.Fatalf("RenderDevices = %v, want the full host inventory", info.RenderDevices) + } +} + +func TestResolveHWAccelWithFFmpegReturnsNoneWhenVAAPISmokeEncodeFails(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + probe := successfulVAAPIProbe() + probe.smokeFailures = []string{"h264_vaapi"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != HWAccelNone { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) + } +} + func TestResolveHWAccelWithFFmpegReturnsNoneWhenNVENCProbeFailsWithoutFallback(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "none" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) } } @@ -81,16 +207,138 @@ func TestResolveHWAccelWithFFmpegUsesNVIDIADeviceNodesWithoutDRM(t *testing.T) { env.addNVIDIADevice(t, "nvidia0") ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) } } -func TestExplicitNVENCBypassesFFmpegProbe(t *testing.T) { +func TestResolveHWAccelPassesThroughConfiguredBackends(t *testing.T) { setupHWAccelTest(t) - if got := ResolveHWAccelWithFFmpeg("nvenc", "/does/not/exist/ffmpeg"); got != "nvenc" { - t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) + for _, configured := range []string{"nvenc", "qsv", "vaapi", "none", "custom"} { + t.Run(configured, func(t *testing.T) { + if got := ResolveHWAccelWithFFmpeg(configured, "/does/not/exist/ffmpeg", ""); got != configured { + t.Fatalf("ResolveHWAccelWithFFmpeg(%q) = %q, want unchanged", configured, got) + } + }) + } +} + +func TestResolveHWAccelAutoIsNoneOffLinux(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + currentGOOS = "darwin" + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != HWAccelNone { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) + } + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + if info.Resolved != HWAccelNone { + t.Fatalf("DetectHWAccelWithFFmpeg().Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) != 0 { + t.Fatalf("DetectHWAccelWithFFmpeg().DetectedBackends = %+v, want empty off Linux", info.DetectedBackends) + } + if _, err := os.Stat(ffmpeg.logPath); !os.IsNotExist(err) { + t.Fatalf("off-Linux detection ran FFmpeg probes (stat err = %v)", err) + } +} + +func TestDetectHWAccelReportsEveryCandidateBackend(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x10de") + probe := fullyCapableProbe() + probe.h264NVENC = false + probe.smokeFailures = []string{"h264_vaapi"} + ffmpeg := writeFakeFFmpeg(t, probe) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + if info.Resolved != "qsv" { + t.Fatalf("Resolved = %q, want qsv", info.Resolved) + } + if len(info.DetectedBackends) != 3 { + t.Fatalf("DetectedBackends = %+v, want one entry per candidate backend", info.DetectedBackends) + } + // The NVIDIA render node carries no libva driver, so it is not a VAAPI + // candidate even though it is a render device. + want := []DetectedBackend{ + {Backend: "nvenc", Verified: false, Devices: []string{"/dev/dri/renderD129"}}, + {Backend: "qsv", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "vaapi", Verified: false, Devices: []string{"/dev/dri/renderD128"}}, + } + for i, expected := range want { + got := info.DetectedBackends[i] + if got.Backend != expected.Backend || got.Verified != expected.Verified { + t.Fatalf("DetectedBackends[%d] = %+v, want backend %q verified=%v", i, got, expected.Backend, expected.Verified) + } + if !slices.Equal(stripDevicePrefix(got.Devices), stripDevicePrefix(expected.Devices)) { + t.Fatalf("DetectedBackends[%d].Devices = %v, want %v", i, got.Devices, expected.Devices) + } + if expected.Verified && got.Reason != "" { + t.Fatalf("DetectedBackends[%d].Reason = %q, want empty for a verified backend", i, got.Reason) + } + if !expected.Verified && got.Reason == "" { + t.Fatalf("DetectedBackends[%d].Reason is empty, want a failure explanation", i) + } + } + if device := filepath.Base(info.DetectedBackends[1].Device); device != "renderD128" { + t.Fatalf("qsv verified device = %q, want the Intel render node", device) + } + if reason := info.DetectedBackends[0].Reason; reason != "h264_nvenc encoder unavailable" { + t.Fatalf("nvenc reason = %q, want the missing encoder", reason) + } + if reason := info.DetectedBackends[2].Reason; !strings.HasPrefix(reason, "h264_vaapi smoke encode failed") { + t.Fatalf("vaapi reason = %q, want the failed smoke encode", reason) + } +} + +func TestDetectHWAccelOmitsBackendsWithoutCandidateHardware(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + backends := make([]string, 0, len(info.DetectedBackends)) + for _, entry := range info.DetectedBackends { + backends = append(backends, entry.Backend) + } + if !slices.Equal(backends, []string{"qsv", "vaapi"}) { + t.Fatalf("detected backends = %v, want qsv and vaapi only", backends) + } + if !info.IntelDetected { + t.Fatal("IntelDetected = false, want true") + } +} + +func TestDetectedBackendJSONShape(t *testing.T) { + encoded, err := json.Marshal(HWAccelInfo{ + Resolved: "qsv", + DetectedBackends: []DetectedBackend{ + {Backend: "qsv", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "nvenc", Verified: false, Reason: "h264_nvenc encoder unavailable"}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"detected_backends":[`, + `{"backend":"qsv","verified":true,"devices":["/dev/dri/renderD128"]}`, + `{"backend":"nvenc","verified":false,"reason":"h264_nvenc encoder unavailable"}`, + } { + if !strings.Contains(string(encoded), want) { + t.Fatalf("HWAccelInfo JSON = %s, missing %s", encoded, want) + } + } + + empty, err := json.Marshal(HWAccelInfo{Resolved: HWAccelNone}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(empty), "detected_backends") { + t.Fatalf("HWAccelInfo JSON = %s, want detected_backends omitted when empty", empty) } } @@ -107,7 +355,7 @@ func TestResolveHWAccelWithFFmpegContextHonorsCallerDeadline(t *testing.T) { // below distinguishes it from. ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) started := time.Now() - got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path) + got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path, "") cancel() if got != HWAccelNone { t.Fatalf("ResolveHWAccelWithFFmpegContext() = %q, want none", got) @@ -120,7 +368,7 @@ func TestResolveHWAccelWithFFmpegContextHonorsCallerDeadline(t *testing.T) { } retryCtx, retryCancel := context.WithTimeout(context.Background(), 60*time.Millisecond) - _ = ResolveHWAccelWithFFmpegContext(retryCtx, "auto", ffmpeg.path) + _ = ResolveHWAccelWithFFmpegContext(retryCtx, "auto", ffmpeg.path, "") retryCancel() logData, err := os.ReadFile(ffmpeg.logPath) if err != nil { @@ -208,24 +456,117 @@ func TestFFmpegSupportsNVENCRequiresCUDAEncodersFiltersAndSmoke(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - resetNVENCProbeCacheForTest() + resetHWProbeCacheForTest() ffmpeg := writeFakeFFmpeg(t, tt.probe) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); ok { - t.Fatalf("ffmpegSupportsNVENC() = true, want false") + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); ok { + t.Fatalf("ffmpegSupportsBackend(nvenc) = true, want false") } else if reason == "" { - t.Fatalf("ffmpegSupportsNVENC() reason is empty") + t.Fatalf("ffmpegSupportsBackend(nvenc) reason is empty") } }) } } +func TestFFmpegSupportsQSVRequiresListingsAndSmoke(t *testing.T) { + setupHWAccelTest(t) + tests := []struct { + name string + probe fakeFFmpegProbe + want string + }{ + { + name: "missing qsv and vaapi hwaccels", + probe: fakeFFmpegProbe{h264QSV: true, hevcQSV: true, smokeOK: true}, + want: "qsv and vaapi hwaccels unavailable", + }, + { + name: "missing h264 qsv encoder", + probe: fakeFFmpegProbe{qsvHWAccel: true, hevcQSV: true, smokeOK: true}, + want: "h264_qsv encoder unavailable", + }, + { + name: "missing hevc qsv encoder", + probe: fakeFFmpegProbe{qsvHWAccel: true, h264QSV: true, smokeOK: true}, + want: "hevc_qsv encoder unavailable", + }, + { + name: "smoke encode failure", + probe: fakeFFmpegProbe{vaapiHWAccel: true, h264QSV: true, hevcQSV: true}, + want: "h264_qsv smoke encode failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, tt.probe) + ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, "/dev/dri/renderD128") + if ok { + t.Fatal("ffmpegSupportsBackend(qsv) = true, want false") + } + if !strings.HasPrefix(reason, tt.want) { + t.Fatalf("reason = %q, want prefix %q", reason, tt.want) + } + }) + } + + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + if ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("ffmpegSupportsBackend(qsv) = false, want true (reason=%q)", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + logText := string(logData) + for _, want := range []string{ + "vaapi=va:/dev/dri/renderD128,driver=iHD,kernel_driver=i915,vendor_id=0x8086", + "qsv=qs@va", + "testsrc2=size=640x360:rate=1", + } { + if !strings.Contains(logText, want) { + t.Fatalf("QSV smoke command missing %q; log:\n%s", want, logText) + } + } +} + +func TestFFmpegSupportsVAAPIRequiresEncoderAndSmoke(t *testing.T) { + setupHWAccelTest(t) + + resetHWProbeCacheForTest() + missing := writeFakeFFmpeg(t, fakeFFmpegProbe{vaapiHWAccel: true, smokeOK: true}) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, missing.path, "/dev/dri/renderD128"); ok { + t.Fatal("ffmpegSupportsBackend(vaapi) = true, want false") + } else if reason != "h264_vaapi encoder unavailable" { + t.Fatalf("reason = %q, want the missing encoder", reason) + } + + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("ffmpegSupportsBackend(vaapi) = false, want true (reason=%q)", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + logText := string(logData) + if !strings.Contains(logText, "vaapi=hw:/dev/dri/renderD128") { + t.Fatalf("VAAPI smoke command missing its init chain; log:\n%s", logText) + } + if strings.Count(logText, "\n") != 2 { + t.Fatalf("VAAPI probe ran %d commands, want an encoders listing and one smoke encode; log:\n%s", + strings.Count(logText, "\n"), logText) + } +} + func TestFFmpegSupportsNVENCCachesByFFmpegPath(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) for i := 0; i < 2; i++ { - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() call %d = %q, want nvenc", i+1, got) } } @@ -239,6 +580,50 @@ func TestFFmpegSupportsNVENCCachesByFFmpegPath(t *testing.T) { } } +func TestHWProbeCacheSeparatesBackendsAndDevices(t *testing.T) { + setupHWAccelTest(t) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + keys := map[string]string{ + "nvenc": hwProbeCacheKey(ffmpeg.path, transcodeHWNVENC, ""), + "qsv-128": hwProbeCacheKey(ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD128"), + "qsv-129": hwProbeCacheKey(ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD129"), + "vaapi-128": hwProbeCacheKey(ffmpeg.path, transcodeHWVAAPI, "/dev/dri/renderD128"), + "identity-eq": hwProbeCacheKey(ffmpeg.path, transcodeHWNVENC, ""), + } + if keys["nvenc"] != keys["identity-eq"] { + t.Fatal("identical backend and device produced different cache keys") + } + seen := map[string]string{} + for name, key := range keys { + if name == "identity-eq" { + continue + } + if other, ok := seen[key]; ok { + t.Fatalf("cache keys for %s and %s collided", name, other) + } + seen[key] = name + } + + // Each distinct key runs its own probe command set: 3 for QSV on two + // devices, 2 for VAAPI. + for _, device := range []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} { + if ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, device); !ok { + t.Fatalf("QSV probe on %s failed: %s", device, reason) + } + } + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("VAAPI probe failed: %s", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(string(logData), "\n"); got != 8 { + t.Fatalf("probe command count = %d, want 8 across three distinct cache keys; log:\n%s", got, logData) + } +} + func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") @@ -254,7 +639,7 @@ func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { go func() { defer wg.Done() <-start - results <- ResolveHWAccelWithFFmpeg("auto", ffmpeg.path) + results <- ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, "") }() } close(start) @@ -278,7 +663,7 @@ func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { func TestFFmpegSupportsNVENCInvalidatesWhenBinaryChangesInPlace(t *testing.T) { setupHWAccelTest(t) ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); !ok { + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); !ok { t.Fatalf("initial NVENC probe failed: %s", reason) } @@ -290,12 +675,12 @@ func TestFFmpegSupportsNVENCInvalidatesWhenBinaryChangesInPlace(t *testing.T) { if err := os.Chtimes(ffmpeg.path, changedAt, changedAt); err != nil { t.Fatalf("advance replacement timestamp: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpeg.path); ok { + if ok, _ := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); ok { t.Fatal("replaced FFmpeg binary reused a stale positive NVENC result") } } -func TestNVENCProbeCacheKeyIncludesResolvedPATHIdentity(t *testing.T) { +func TestFFmpegIdentityKeyIncludesResolvedPATHIdentity(t *testing.T) { firstDir := t.TempDir() secondDir := t.TempDir() stamp := time.Unix(100, 0) @@ -310,29 +695,26 @@ func TestNVENCProbeCacheKeyIncludesResolvedPATHIdentity(t *testing.T) { } t.Setenv("PATH", firstDir) - firstKey := nvencProbeCacheKey("ffmpeg") + firstKey := ffmpegIdentityKey("ffmpeg") t.Setenv("PATH", secondDir) - secondKey := nvencProbeCacheKey("ffmpeg") + secondKey := ffmpegIdentityKey("ffmpeg") if firstKey == secondKey { t.Fatalf("PATH-resolved FFmpeg identities collided: %q", firstKey) } } -func TestFFmpegSupportsNVENCNegativeResultExpires(t *testing.T) { +func TestHWProbeNegativeResultExpires(t *testing.T) { setupHWAccelTest(t) - oldTTL := nvencProbeNegativeTTL - nvencProbeNegativeTTL = 20 * time.Millisecond - t.Cleanup(func() { nvencProbeNegativeTTL = oldTTL }) + clock := time.Now() + hwProbeNow = func() time.Time { return clock } dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - markerPath := filepath.Join(dir, "nvenc-ready") + markerPath := filepath.Join(dir, "vaapi-ready") script := fmt.Sprintf(`#!/bin/sh case "$*" in - *-hwaccels*) echo cuda; exit 0 ;; - *-encoders*) echo 'h264_nvenc hevc_nvenc'; exit 0 ;; - *-filters*) echo 'scale_cuda hwupload_cuda'; exit 0 ;; + *-encoders*) echo 'h264_vaapi'; exit 0 ;; *) test -e %q ;; esac `, markerPath) @@ -340,27 +722,37 @@ esac t.Fatalf("write marker-controlled FFmpeg: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpegPath); ok { - t.Fatal("initial NVENC probe unexpectedly succeeded") + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); ok { + t.Fatal("initial VAAPI probe unexpectedly succeeded") } if err := os.WriteFile(markerPath, []byte("ready"), 0o600); err != nil { - t.Fatalf("enable NVENC smoke probe: %v", err) + t.Fatalf("enable VAAPI smoke probe: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpegPath); ok { - t.Fatal("negative result was not retained during its short TTL") + clock = clock.Add(hwProbeNegativeTTL - time.Millisecond) + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); ok { + t.Fatal("negative result was not retained during its TTL") } - time.Sleep(2 * nvencProbeNegativeTTL) - if ok, reason := ffmpegSupportsNVENC(ffmpegPath); !ok { + clock = clock.Add(2 * time.Millisecond) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); !ok { t.Fatalf("expired negative result was not retried: %s", reason) } + // A positive result is kept for the process lifetime, so removing the + // marker after success must not change the answer. + if err := os.Remove(markerPath); err != nil { + t.Fatal(err) + } + clock = clock.Add(time.Hour) + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); !ok { + t.Fatal("positive probe result expired, want process-lifetime caching") + } } func TestFFmpegSupportsNVENCSmokeProbeUsesSafeFrameDimensions(t *testing.T) { setupHWAccelTest(t) ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); !ok { - t.Fatalf("ffmpegSupportsNVENC() = false, want true (reason=%q)", reason) + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); !ok { + t.Fatalf("ffmpegSupportsBackend(nvenc) = false, want true (reason=%q)", reason) } logData, err := os.ReadFile(ffmpeg.logPath) @@ -384,6 +776,49 @@ func successfulNVENCProbe() fakeFFmpegProbe { } } +func successfulQSVProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + qsvHWAccel: true, + h264QSV: true, + hevcQSV: true, + smokeOK: true, + } +} + +func successfulVAAPIProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + vaapiHWAccel: true, + h264VAAPI: true, + smokeOK: true, + } +} + +func fullyCapableProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + cuda: true, + qsvHWAccel: true, + vaapiHWAccel: true, + h264NVENC: true, + hevcNVENC: true, + h264QSV: true, + hevcQSV: true, + h264VAAPI: true, + scaleCUDA: true, + uploadCUDA: true, + smokeOK: true, + } +} + +// stripDevicePrefix compares device lists by basename so expectations stay +// readable against the test's temporary /dev/dri stand-in. +func stripDevicePrefix(devices []string) []string { + names := make([]string, 0, len(devices)) + for _, device := range devices { + names = append(names, filepath.Base(device)) + } + return names +} + func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { t.Helper() @@ -392,8 +827,9 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { oldNVIDIADeviceGlob := defaultNVIDIADeviceGlob oldSysClassDRMDir := sysClassDRMDir oldGOOS := currentGOOS - oldProbeTimeout := nvencProbeCommandTimeout - resetNVENCProbeCacheForTest() + oldProbeTimeout := hwProbeCommandTimeout + oldProbeNow := hwProbeNow + resetHWProbeCacheForTest() tmp := t.TempDir() env := &hwAccelTestEnv{ @@ -406,7 +842,7 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { defaultNVIDIADeviceGlob = filepath.Join(env.devDir, "nvidia[0-9]*") sysClassDRMDir = env.sysDir currentGOOS = "linux" - nvencProbeCommandTimeout = 200 * time.Millisecond + hwProbeCommandTimeout = 200 * time.Millisecond if err := os.MkdirAll(env.driDir, 0o755); err != nil { t.Fatalf("create test dri dir: %v", err) @@ -421,8 +857,9 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { defaultNVIDIADeviceGlob = oldNVIDIADeviceGlob sysClassDRMDir = oldSysClassDRMDir currentGOOS = oldGOOS - nvencProbeCommandTimeout = oldProbeTimeout - resetNVENCProbeCacheForTest() + hwProbeCommandTimeout = oldProbeTimeout + hwProbeNow = oldProbeNow + resetHWProbeCacheForTest() }) return env @@ -469,6 +906,12 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary { if probe.cuda { script += " echo 'cuda'\n" } + if probe.qsvHWAccel { + script += " echo 'qsv'\n" + } + if probe.vaapiHWAccel { + script += " echo 'vaapi'\n" + } script += " exit 0 ;;\n" script += " *-encoders*)\n" if probe.h264NVENC { @@ -477,6 +920,15 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary { if probe.hevcNVENC { script += " echo ' V..... hevc_nvenc NVIDIA NVENC hevc encoder'\n" } + if probe.h264QSV { + script += " echo ' V..... h264_qsv H.264 QSV encoder'\n" + } + if probe.hevcQSV { + script += " echo ' V..... hevc_qsv HEVC QSV encoder'\n" + } + if probe.h264VAAPI { + script += " echo ' V..... h264_vaapi H.264 VAAPI encoder'\n" + } script += " exit 0 ;;\n" script += " *-filters*)\n" if probe.scaleCUDA { @@ -486,13 +938,31 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary { script += " echo ' ... hwupload_cuda V->V upload CUDA frames'\n" } script += " exit 0 ;;\n" - script += " *)\n" - if probe.smokeOK { - script += " exit 0 ;;\n" - } else { - script += " echo 'no capable devices found' >&2\n" - script += " exit 1 ;;\n" + for _, encoder := range []string{"h264_nvenc", "h264_qsv", "h264_vaapi"} { + script += fmt.Sprintf(" *%s*)\n", encoder) + if probe.smokeOK && !slices.Contains(probe.smokeFailures, encoder) { + // The init chain carries the device path, so a broken GPU is modeled + // by matching the command rather than by ignoring the argument. + if len(probe.smokeDeviceFailures) > 0 { + patterns := make([]string, 0, len(probe.smokeDeviceFailures)) + for _, device := range probe.smokeDeviceFailures { + patterns = append(patterns, "*"+device+"*") + } + script += " case \"$*\" in\n" + script += fmt.Sprintf(" %s)\n", strings.Join(patterns, "|")) + script += fmt.Sprintf(" echo 'no capable devices found for %s' >&2\n", encoder) + script += " exit 1 ;;\n" + script += " esac\n" + } + script += " exit 0 ;;\n" + } else { + script += fmt.Sprintf(" echo 'no capable devices found for %s' >&2\n", encoder) + script += " exit 1 ;;\n" + } } + script += " *)\n" + script += " echo 'unexpected probe command' >&2\n" + script += " exit 1 ;;\n" script += "esac\n" if err := os.WriteFile(path, []byte(script), 0o755); err != nil { @@ -501,8 +971,8 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary { return fakeFFmpegBinary{path: path, logPath: logPath} } -func resetNVENCProbeCacheForTest() { - nvencProbeCache.Lock() - defer nvencProbeCache.Unlock() - nvencProbeCache.byPath = make(map[string]nvencProbeCacheEntry) +func resetHWProbeCacheForTest() { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + hwProbeCache.entries = make(map[string]hwProbeCacheEntry) } diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index 05124526d..04b33d57d 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -143,6 +143,9 @@ const ( transcodeHWQSV = "qsv" transcodeHWVAAPI = "vaapi" transcodeHWNVENC = "nvenc" + // vaapiHWDeviceAlias names the VAAPI device every non-QSV hardware command + // line declares; filter graphs and probes reference it by this alias. + vaapiHWDeviceAlias = "hw" ) // TranscodeSession manages a running ffmpeg HLS transcode process. @@ -530,7 +533,7 @@ func ResolveToneMapExecutor(ctx context.Context, opts TranscodeOpts) (TranscodeO opts.ToneMapSourceRevision.IsZero() { return opts, fmt.Errorf("incomplete tone-map recipe") } - backend := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath) + backend := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath, opts.HWDevice) capabilities, err := tonemap.Probe(ctx, ResolveFFmpegPath(opts.FFmpegPath), backend, opts.HWDevice) if err != nil { return opts, fmt.Errorf("%w: probe tone-map executor: %w", ErrToneMapExecutorUnavailable, err) @@ -736,7 +739,7 @@ func buildFFmpegArgs(opts TranscodeOpts) []string { // resolveEffectiveTranscodeHWAccel returns the backend that will actually execute the recipe. func resolveEffectiveTranscodeHWAccel(opts TranscodeOpts) string { - hwAccel := ResolveHWAccelWithFFmpeg(opts.HWAccel, opts.FFmpegPath) + hwAccel := ResolveHWAccelWithFFmpeg(opts.HWAccel, opts.FFmpegPath, opts.HWDevice) if hwAccel == "" { return "" } @@ -866,10 +869,7 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { hwDevice = "/dev/dri/renderD128" // last-resort fallback } // VAAPI→QSV hardware pipeline: derive QSV from VAAPI device. - args = append(args, - "-init_hw_device", qsvVAAPIInitDevice(hwDevice), - "-init_hw_device", "qsv=qs@va", - ) + args = append(args, tonemap.QSVInitDeviceArgs(hwDevice)...) if opts.ToneMapMode == tonemap.ModeHardware && opts.ToneMapFilter == tonemap.HardwareFilterOpenCL { args = append(args, "-init_hw_device", "opencl=ocl@va") } @@ -883,10 +883,8 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { if vaapiDevice == "" { vaapiDevice = "/dev/dri/renderD128" // last-resort fallback } - args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=hw:%s", vaapiDevice), - "-filter_hw_device", "hw", - ) + args = append(args, tonemap.VAAPIInitDeviceArgs(vaapiHWDeviceAlias, vaapiDevice)...) + args = append(args, "-filter_hw_device", vaapiHWDeviceAlias) if !opts.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } @@ -903,10 +901,6 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { return args } -func qsvVAAPIInitDevice(device string) string { - return fmt.Sprintf("vaapi=va:%s,driver=iHD,kernel_driver=i915,vendor_id=0x8086", device) -} - // videoPreset returns an encoder-compatible preset. CPU encoders use a faster // fast-start preset for initial playback, while QSV stays on the fastest // preset family it supports. diff --git a/internal/proxy/server.go b/internal/proxy/server.go index aedaf47f8..de7a319d8 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -200,10 +200,14 @@ func (s *Server) Handler() http.Handler { // custom image) would fail at stream time rather than at selection time. func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { ffmpegPath := "" + hwAccel := playback.HWAccelNone + hwDevice := "" if cfg := s.watcher.Config(); cfg != nil { ffmpegPath = cfg.Playback.FFmpegPath + hwAccel = cfg.Playback.HWAccel + hwDevice = cfg.Playback.HWDevice } - info := playback.DetectHWAccelWithFFmpeg(ffmpegPath) + info := playback.DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice) info.Transformations = playback.ProbeTransformationRegistryV3(r.Context(), ffmpegPath).Advertised() w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(info); err != nil { diff --git a/internal/tonemap/preflight.go b/internal/tonemap/preflight.go index 303e0f8f3..2181509bc 100644 --- a/internal/tonemap/preflight.go +++ b/internal/tonemap/preflight.go @@ -441,7 +441,8 @@ func sourceConversionPreflightArgs(request SourcePreflightRequest, position floa if device == "" { device = defaultDRIRenderDevice } - args = append(args, "-init_hw_device", qsvVAAPIInitDevice(device), "-init_hw_device", "qsv=qs@va", "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va") + args = append(args, QSVInitDeviceArgs(device)...) + args = append(args, "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va") if !request.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } @@ -449,7 +450,8 @@ func sourceConversionPreflightArgs(request SourcePreflightRequest, position floa if device == "" { device = defaultDRIRenderDevice } - args = append(args, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va") + args = append(args, VAAPIInitDeviceArgs("va", device)...) + args = append(args, "-filter_hw_device", "va") if !request.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index 9393d1fa7..e4771ded4 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -18,8 +18,14 @@ const ( probeCommandTimeout = 5 * time.Second probeNegativeTTL = 15 * time.Second probeTimeoutSlack = time.Second - probeEndpointSlack = 20 * time.Second - probeRequestSlack = 5 * time.Second + // probeEndpointSlack covers what a capability endpoint spends around the + // tone-map matrix itself: one bounded hardware detection walk (30s in + // playback.hwAccelWalkTimeout), the transformation registry's three 3s + // commands, and response overhead. It cannot be derived from those + // constants — playback imports this package, not the other way round — so + // it is raised whenever either budget grows. + probeEndpointSlack = 45 * time.Second + probeRequestSlack = 5 * time.Second ) // One deterministic 64x64 Main 10 HEVC frame. Keeping the compressed fixture @@ -365,15 +371,15 @@ func hardwareSmokeArgs(fixturePath, backend, hardwareDevice string, kind SourceK base := []string{ffmpegHideBannerArg, ffmpegLogLevelArg, ffmpegErrorLogLevel} switch backend { case BackendQSV: + base = append(base, QSVInitDeviceArgs(device)...) base = append(base, - "-init_hw_device", qsvVAAPIInitDevice(device), - "-init_hw_device", "qsv=qs@va", "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI, ) case BackendVAAPI: - base = append(base, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI) + base = append(base, VAAPIInitDeviceArgs("va", device)...) + base = append(base, "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI) case BackendNVENC: cudaDevice := device if cudaDevice == "" { diff --git a/internal/tonemap/probe_test.go b/internal/tonemap/probe_test.go index 93fe988b3..a5f7b2c9f 100644 --- a/internal/tonemap/probe_test.go +++ b/internal/tonemap/probe_test.go @@ -53,15 +53,21 @@ func TestProbeTotalTimeoutCoversBoundedCommandMatrix(t *testing.T) { } func TestProbeEndpointTimeoutCoversDetectionAndProbeBudgets(t *testing.T) { - if got, want := ProbeEndpointTimeout(BackendQSV, "/dev/dri/renderD128"), 81*time.Second; got != want { + if got, want := ProbeEndpointTimeout(BackendQSV, "/dev/dri/renderD128"), 106*time.Second; got != want { t.Fatalf("ProbeEndpointTimeout() = %s, want %s", got, want) } - if got, want := ProbeEndpointTimeout("auto", "/dev/dri/renderD128,/dev/dri/renderD129"), 106*time.Second; got != want { + if got, want := ProbeEndpointTimeout("auto", "/dev/dri/renderD128,/dev/dri/renderD129"), 131*time.Second; got != want { t.Fatalf("ProbeEndpointTimeout(auto) = %s, want %s", got, want) } - if got, want := ProbeRequestTimeout(BackendQSV, "/dev/dri/renderD128"), 86*time.Second; got != want { + if got, want := ProbeRequestTimeout(BackendQSV, "/dev/dri/renderD128"), 111*time.Second; got != want { t.Fatalf("ProbeRequestTimeout() = %s, want %s", got, want) } + // The slack has to outlast a full hardware detection walk plus the + // transformation registry probe, or a node answers 503 while its own + // detection is still running. + if probeEndpointSlack < 30*time.Second+3*3*time.Second { + t.Fatalf("probeEndpointSlack = %s, too small for detection and registry probes", probeEndpointSlack) + } } // TestProbeEmptyCapabilitiesExpire verifies failed discovery is retried after a short interval. diff --git a/internal/tonemap/tonemap.go b/internal/tonemap/tonemap.go index 16eac4c3f..b90dc640a 100644 --- a/internal/tonemap/tonemap.go +++ b/internal/tonemap/tonemap.go @@ -665,6 +665,21 @@ func qsvVAAPIInitDevice(device string) string { return "vaapi=va:" + device + ",driver=iHD,kernel_driver=i915,vendor_id=0x8086" } +// QSVInitDeviceArgs declares the Intel VAAPI display and derives the QSV +// device from it. Every QSV command line in the server — transcode, encoder +// warmup, capability probes, tone-map smoke tests, chapter thumbnails — must +// initialize hardware through this chain, so a driver constraint is fixed in +// one place. +func QSVInitDeviceArgs(device string) []string { + return []string{"-init_hw_device", qsvVAAPIInitDevice(device), "-init_hw_device", "qsv=qs@va"} +} + +// VAAPIInitDeviceArgs declares one VAAPI device under the alias the caller's +// filter graph and encoder reference. +func VAAPIInitDeviceArgs(alias, device string) []string { + return []string{"-init_hw_device", "vaapi=" + alias + ":" + device} +} + // HDRMetadataRemovalFilter removes side data that would otherwise incorrectly // label the converted SDR frames as HDR or Dolby Vision. func HDRMetadataRemovalFilter() string { diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index fa4a19abd..aacb385df 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -896,10 +896,11 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { } resolveCtx, cancel := context.WithTimeout(r.Context(), toneMapCapabilityResolveTimeout(configuredHWAccel, hwDevice)) defer cancel() - hwAccel := playback.ResolveHWAccelWithFFmpegContext(resolveCtx, configuredHWAccel, ffmpegPath) - info := playback.DetectHWAccelWithFFmpegContext(resolveCtx, ffmpegPath) + // One detection walk answers both questions: Resolved honors the configured + // backend's pass-through contract, and DetectedBackends explains it. + info := playback.DetectHWAccelWithFFmpegContext(resolveCtx, configuredHWAccel, ffmpegPath, hwDevice) info.ProbeRequestTimeoutMillis = tonemap.ProbeRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() - capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), hwAccel, hwDevice) + capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), info.Resolved, hwDevice) if err != nil { http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) return From ae708bd53bc51067b45d0569c18d6659100f7831 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:59:52 -0400 Subject: [PATCH 002/163] feat(nodepool): persist per-node gpu capability inventory Nodes now compute a deterministic capability hash over their canonicalized hardware report (resolved backend, render devices with PCI address and NVIDIA uuid, host boot_id, probed backends, transformations, tone-map executors) and advertise it in the 30s health response without ever running probes there. On a hash change the health sweep fetches /hw-capabilities detached from the sweep (deduplicated per node, bounded at 2m to cover a cold node's probe budget), persists the payload to new nullable stream_nodes columns, updates the in-memory pools copy-on-write, refreshes the v3 capability cache, and logs capability drift. Nodes without a hash keep today's behavior exactly. GET /admin/nodes now returns the stored inventory plus derived physical_gpu_keys (gpu_uuid, else boot_id|pci_address) for shared-GPU detection, and the admin Nodes page gains a GPU column with verified/failed backend badges, device summaries, and a staleness indicator tied to the health-check clock. /admin/nodes and /admin/system/hw-accel are now documented in docs/admin-api.md. Phase 2 of the node GPU observability plan. Related issue: #780 Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 40 +- docs/admin-api.md | 148 +++++ internal/api/handlers/nodes.go | 80 ++- internal/api/handlers/nodes_test.go | 142 +++++ internal/api/handlers/playback.go | 25 +- internal/api/handlers/playback_v3.go | 97 +++- internal/api/handlers/playback_v3_test.go | 57 ++ internal/api/router.go | 5 + internal/nodepool/health.go | 228 +++++++- internal/nodepool/health_test.go | 508 ++++++++++++++++++ internal/nodepool/proxy_pool.go | 8 + internal/nodepool/repository.go | 50 +- .../nodepool/repository_capabilities_test.go | 95 ++++ internal/nodepool/transcode_pool.go | 26 + internal/playback/capabilityhash.go | 178 ++++++ internal/playback/capabilityhash_test.go | 129 +++++ internal/playback/gpudetect.go | 143 ++++- internal/playback/gpuidentity_test.go | 185 +++++++ internal/proxy/capability_snapshot_test.go | 128 +++++ internal/proxy/server.go | 120 ++++- .../transcodenode/capability_snapshot_test.go | 139 +++++ internal/transcodenode/server.go | 128 ++++- .../20260826225109_node_gpu_capabilities.sql | 34 ++ web/src/api/types.ts | 51 ++ web/src/pages/AdminNodes.tsx | 55 +- web/src/pages/adminNodesPresentation.test.ts | 295 ++++++++++ web/src/pages/adminNodesPresentation.ts | 205 +++++++ 27 files changed, 3234 insertions(+), 65 deletions(-) create mode 100644 internal/api/handlers/nodes_test.go create mode 100644 internal/nodepool/health_test.go create mode 100644 internal/nodepool/repository_capabilities_test.go create mode 100644 internal/playback/capabilityhash.go create mode 100644 internal/playback/capabilityhash_test.go create mode 100644 internal/playback/gpuidentity_test.go create mode 100644 internal/proxy/capability_snapshot_test.go create mode 100644 internal/transcodenode/capability_snapshot_test.go create mode 100644 migrations/sql/20260826225109_node_gpu_capabilities.sql create mode 100644 web/src/pages/adminNodesPresentation.test.ts create mode 100644 web/src/pages/adminNodesPresentation.ts diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 31ec0f1aa..79d171e5b 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -138,6 +138,37 @@ func resolveNodeIdentity() string { return h } +// nodeCapabilityFetcher adapts the authenticated node capability client to the +// node health sweep, which stores capability reports opaquely. The stored +// payload is this server's re-marshaling of the decoded report rather than the +// node's bytes, so what is persisted is exactly what the API understood; the +// hash comes out of the payload itself, because only the node knows what it +// hashed. A report without one is refused rather than given a synthetic hash, +// which would make an old node look like it had capability tracking. +func nodeCapabilityFetcher(jwtSecret string) nodepool.CapabilityFetcher { + client := &http.Client{Timeout: nodeCapabilityRequestTimeout} + return func(ctx context.Context, nodeURL string) ([]byte, string, error) { + info, status, err := transcodenode.FetchHWCapabilities(ctx, client, nodeURL, jwtSecret) + if err != nil { + return nil, "", err + } + if status != http.StatusOK { + return nil, "", fmt.Errorf("node capability request returned status %d", status) + } + payload, err := json.Marshal(info) + if err != nil { + return nil, "", err + } + return payload, info.CapabilityHash, nil + } +} + +// nodeCapabilityRequestTimeout bounds one capability request. A cold node runs +// ffmpeg probes to answer and advertises a probe budget of up to ~2 minutes; +// the fetch runs detached from the health sweep, so matching that budget is +// safe and lets a cold node's first report land instead of timing out. +const nodeCapabilityRequestTimeout = 2 * time.Minute + func clientIPResolverFromConfig(cfg *config.Config) (*clientip.Resolver, error) { if cfg == nil { return nil, fmt.Errorf("config is not loaded") @@ -838,6 +869,9 @@ func main() { watcher.Config, nil, )) + // Keep the capability hash /health advertises current, so the API's + // sweep refetches this proxy's inventory only when it actually changed. + srv.StartCapabilitySnapshots(appCtx) handler = srv.Handler() } else { srv := transcodenode.NewServer(watcher, tracker) @@ -848,7 +882,9 @@ func main() { // restart (the node hop token is recipe-less). Shares the offload Redis. srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0)) srv.SetStreamTelemetry(streamTelemetryRegistry) - srv.StartHardwareEncoderWarmup(appCtx) + // The first capability snapshot waits on warmup so it measures a + // primed encoder rather than racing it into a probe failure. + srv.StartCapabilitySnapshots(appCtx, srv.StartHardwareEncoderWarmup(appCtx)) // Reclaim orphaned transcode dirs at boot and hourly thereafter, bound // to appCtx so it stops on shutdown. srv.StartOrphanSweeper(appCtx) @@ -1067,6 +1103,8 @@ func main() { deps.NodePlanner = nodepool.NewPlanner(proxyPool, transcodePool) healthChecker := nodepool.NewHealthChecker(proxyPool, transcodePool, nodeRepo) + healthChecker.SetCapabilityFetcher(nodeCapabilityFetcher(cfg.Auth.JWTSecret)) + deps.NodeHealthChecker = healthChecker healthChecker.Start(appCtx) slog.Info("node pools initialized", "proxy_nodes", len(proxyNodes), "transcode_nodes", len(transcodeNodes)) diff --git a/docs/admin-api.md b/docs/admin-api.md index f29fb8b46..b505b6b29 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -9,6 +9,154 @@ This document is new and covers only the routes listed below. The rest of the admin surface predates it and is currently documented by the code and by the design documents under `docs/design/`. +## `GET /api/v1/admin/nodes` + +Lists every registered stream node — proxy and transcode alike — with its +configuration, last health result, and last stored hardware inventory. + +Always `200 OK` with a JSON array. + +| Field | Type | Meaning | +|---|---|---| +| `id`, `name`, `type`, `url` | int, string, string, string | Identity. `type` is `proxy` or `transcode`. | +| `enabled` | bool | Whether the node is eligible for selection at all. | +| `healthy` | bool | Result of the last health check. | +| `active_jobs`, `egress_kbps` | int | Last health-reported load. `egress_kbps` is a rolling average and is currently non-zero for proxy nodes only. | +| `group` | string \| null | Co-location group. A group is only eligible while every enabled member is healthy. | +| `max_jobs`, `max_bandwidth_kbps` | int \| null | Capacity caps. `null` means unlimited. | +| `last_health_check` | RFC3339 string \| null | When the node was last checked. | +| `created_at` | RFC3339 string | When the node was registered. | +| `capabilities` | object | The node's last stored capability report — the same body `GET /hw-capabilities` returns on the node. Omitted until one has been stored. | +| `capabilities_hash` | string | Identity of that report, as computed by the node. Omitted with `capabilities`. | +| `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | +| `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | + +Capability reports are refreshed by the background health sweep, not by this +read: a node advertises a `capabilities_hash` in its own health response, and +only a hash that differs from the stored one triggers a refetch. A node running +a build from before capability snapshots advertises no hash and therefore +carries none of the four fields above. A failed refetch keeps the previous +report rather than clearing it — a node that cannot be reached is not evidence +that its hardware changed. The refetch itself runs outside the sweep's own wait, +one at a time per node, so a slow capability probe cannot delay the health +cadence of the other nodes; a new report can therefore land shortly after the +check that noticed the change rather than with it. + +### `physical_gpu_keys` + +One key per render device in the stored report, deduplicated and sorted: + +- the device's `gpu_uuid` when present (NVIDIA's permanent GPU identity, which + follows the card between slots and hosts), otherwise +- `|`, because a PCI slot only means the same hardware + within one boot of one kernel. + +A device with neither contributes no key rather than a synthetic one. Two nodes +sharing a key are backed by the same physical GPU — the case that makes +per-node capacity accounting wrong, and which no single node's report can +express. + +## `POST /api/v1/admin/nodes` + +Registers a node. Body: `name`, `type` (`proxy` or `transcode`), `url`, and the +optional `group`, `max_jobs`, `max_bandwidth_kbps`. A non-positive cap and an +empty group mean "unlimited" and "ungrouped". + +`201 Created` with the created node in the same shape as one list entry (with +no capability fields yet — nothing has been fetched). `400 Bad Request` when a +required field is missing or `type` is not one of the two allowed values. The +node pools are reloaded afterwards. + +## `PUT /api/v1/admin/nodes/{id}` + +Updates a node's mutable fields. Every field is optional; an omitted field is +left unchanged. An empty-string `group` clears the group, and a non-positive +`max_jobs` or `max_bandwidth_kbps` clears that cap. + +`200 OK` with the updated node, `404 Not Found` for an unknown id. The node +pools are reloaded afterwards. + +Capability fields are not writable here. They are owned by the health sweep, +because only the node can say what hardware it has. + +## `DELETE /api/v1/admin/nodes/{id}` + +Removes a node. `204 No Content`, or `404 Not Found` for an unknown id. The +node pools are reloaded afterwards. Sessions already streaming from the node +are not torn down by this call. + +## `POST /api/v1/admin/nodes/{id}/check` + +Runs one health check against a node immediately and persists the result, for +an admin who does not want to wait for the next 30-second sweep. + +Always `200 OK`; an unreachable node is reported as `healthy: false` rather +than as an error status. `404 Not Found` for an unknown id. + +| Field | Type | Meaning | +|---|---|---| +| `healthy` | bool | The node answered its health endpoint. | +| `active_jobs`, `egress_kbps` | int | What it reported. Zero when unhealthy. | +| `capabilities_hash` | string | The hash the node advertised on this check. Omitted when the node reports none. | + +This is the node's *current* hash, not the stored one. A value here that +differs from the `capabilities_hash` in the list response means the background +sweep has a refetch pending; this route does not fetch capabilities itself. + +## `GET /api/v1/admin/system/hw-accel` + +Reports GPU hardware and acceleration capability. With healthy transcode nodes +registered it probes each of them; with none it probes this host. The top-level +fields are the first node that answered (or the local probe), and `nodes` +carries one entry per healthy node. + +`playback.hw_device` is one cluster-wide value, so the per-node inventories are +what an operator needs to see that a device path exists on every node before +pinning one. + +Always `200 OK`. A node that failed its probe is reported in `nodes` with an +`error` rather than dropped, so a hardware problem is visible instead of silent. + +Top-level (and each node's own report): + +| Field | Type | Meaning | +|---|---|---| +| `resolved` | string | The backend that would actually be used: `nvenc`, `qsv`, `vaapi`, or `none`. An explicitly configured backend wins even when its probe failed — read `detected_backends` for why. | +| `render_devices` | string[] | Every accessible `/dev/dri/renderD*` path. | +| `render_device_details` | object[] | One entry per device (see below). | +| `intel_detected` | bool | An Intel GPU is present in the inventory. | +| `detected_backends` | object[] | One entry per backend that had candidate hardware, with the outcome of its FFmpeg verification (see below). | +| `boot_id` | string | The host's kernel boot identity (Linux only). Pairs with a device's `pci_address` to distinguish the same GPU from the same slot after a reboot. | +| `capability_hash` | string | `sha256:` over this report's hardware identity and capability fields — not over `source`, `node_url`, the probe budget, or itself. Two reports of unchanged hardware hash identically regardless of probe order. | +| `source` | string | `local` for a probe of this host. | +| `node_url` | string | Set on a node's report. | +| `transformations`, `tone_map_capabilities` | object[] | What this host can execute, as advertised to the planner. | + +`render_device_details` entries: + +| Field | Type | Meaning | +|---|---|---| +| `path` | string | The `/dev/dri` path. Assigned by enumeration order, so it moves when hardware is added or removed. | +| `pci_address` | string | The device's PCI slot (e.g. `0000:03:00.0`), read from sysfs. Omitted when the device has no PCI identity. | +| `gpu_uuid` | string | NVIDIA's permanent GPU identity. Reported only for NVIDIA devices on hosts with `nvidia-smi` installed; omitted otherwise. | +| `description` | string | Short human label, e.g. `NVIDIA GPU (0x2204)`. | + +`detected_backends` entries: + +| Field | Type | Meaning | +|---|---|---| +| `backend` | string | `nvenc`, `qsv`, or `vaapi`. | +| `verified` | bool | At least one candidate device passed a real single-frame encode, not just an FFmpeg build-flag listing. | +| `devices` | string[] | Every candidate considered for this backend. | +| `device` | string | The candidate whose probe passed. Empty for NVENC, which addresses its GPU through CUDA rather than a render node. | +| `reason` | string | Why verification failed, attributed per device when several were tried. | + +Each entry in `nodes` carries `node_url` and `node_name` plus either that +node's `resolved`, `render_devices` and `render_device_details`, or an `error` +explaining why it could not be probed. The full report for one node — including +`detected_backends`, `boot_id` and `capability_hash` — is what +`GET /api/v1/admin/nodes` stores per node in `capabilities`. + ## `GET /api/v1/admin/stream-telemetry/parity` Returns the merged stream-telemetry view beside the two legacy live-session diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 79690c90b..81d3143ca 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "net/http" + "slices" "strconv" "sync" "time" @@ -71,6 +72,21 @@ type checkNodeResult struct { Healthy bool `json:"healthy"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash is what the node advertised on this check. It is not the + // stored hash: an unequal pair means the background sweep has a refetch to + // do, which is exactly what an operator checking a node wants to see. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` +} + +// nodeListItem is a stored node plus the fields derived from its capability +// payload. The row is embedded rather than copied so the response keeps every +// existing field automatically as the node model grows. +type nodeListItem struct { + *nodepool.Node + // PhysicalGPUKeys identifies the actual GPUs behind this node. Two nodes + // sharing a key are sharing hardware — the case that makes independent + // capacity accounting wrong — which no per-node field can express. + PhysicalGPUKeys []string `json:"physical_gpu_keys,omitempty"` } // HandleListNodes handles GET /admin/nodes. @@ -82,7 +98,60 @@ func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { return } - writeJSON(w, http.StatusOK, nodes) + items := make([]nodeListItem, 0, len(nodes)) + for _, node := range nodes { + items = append(items, nodeListItem{ + Node: node, + PhysicalGPUKeys: physicalGPUKeys(node.Capabilities), + }) + } + writeJSON(w, http.StatusOK, items) +} + +// nodeGPUIdentity is the minimal projection needed to identify a node's GPUs +// out of its stored capability payload. +type nodeGPUIdentity struct { + BootID string `json:"boot_id"` + RenderDeviceDetails []struct { + PCIAddress string `json:"pci_address"` + GPUUUID string `json:"gpu_uuid"` + } `json:"render_device_details"` +} + +// physicalGPUKeys derives one stable key per GPU a node can see. An NVIDIA +// uuid is preferred because it follows the card between slots and hosts; the +// PCI address falls back to it, scoped by boot id because a device path and +// slot only mean the same hardware within one boot of one kernel. A device with +// neither is unidentifiable and contributes no key rather than a fake one. +func physicalGPUKeys(capabilities []byte) []string { + if len(capabilities) == 0 { + return nil + } + var identity nodeGPUIdentity + if err := json.Unmarshal(capabilities, &identity); err != nil { + return nil + } + seen := make(map[string]struct{}, len(identity.RenderDeviceDetails)) + keys := make([]string, 0, len(identity.RenderDeviceDetails)) + for _, device := range identity.RenderDeviceDetails { + key := device.GPUUUID + if key == "" { + if device.PCIAddress == "" { + continue + } + key = identity.BootID + "|" + device.PCIAddress + } + if _, duplicate := seen[key]; duplicate { + continue + } + seen[key] = struct{}{} + keys = append(keys, key) + } + if len(keys) == 0 { + return nil + } + slices.Sort(keys) + return keys } // HandleCreateNode handles POST /admin/nodes. @@ -182,16 +251,17 @@ func (h *NodeHandler) HandleCheckNode(w http.ResponseWriter, r *http.Request) { return } - healthy, activeJobs, egressKbps := nodepool.CheckNode(r.Context(), node) + healthy, activeJobs, egressKbps, capabilitiesHash := nodepool.CheckNode(r.Context(), node) if err := h.repo.UpdateHealth(r.Context(), id, healthy, activeJobs, egressKbps); err != nil { slog.ErrorContext(r.Context(), "persisting health check result", "component", "api", "node_id", id, "error", err) } writeJSON(w, http.StatusOK, checkNodeResult{ - Healthy: healthy, - ActiveJobs: activeJobs, - EgressKbps: egressKbps, + Healthy: healthy, + ActiveJobs: activeJobs, + EgressKbps: egressKbps, + CapabilitiesHash: capabilitiesHash, }) } diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go new file mode 100644 index 000000000..73cba44dd --- /dev/null +++ b/internal/api/handlers/nodes_test.go @@ -0,0 +1,142 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodepool" +) + +// An NVIDIA uuid identifies a card wherever it is plugged in; a PCI address +// only identifies a slot, and only within one boot of one kernel. Deriving the +// key this way is what lets an admin see that two nodes are sharing one GPU. +func TestPhysicalGPUKeys(t *testing.T) { + tests := []struct { + name string + capabilities string + want []string + }{ + { + name: "prefers gpu uuid over slot identity", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`, + want: []string{"GPU-aaa"}, + }, + { + name: "falls back to boot-scoped pci address", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}]}`, + want: []string{"boot-1|0000:04:00.0"}, + }, + { + name: "mixed devices are deduped and sorted", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}, + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}, + {"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}]}`, + want: []string{"GPU-bbb", "boot-1|0000:05:00.0"}, + }, + { + name: "device with no identity contributes no key", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128"},{"path":"/dev/dri/renderD129","gpu_uuid":"GPU-ccc"}]}`, + want: []string{"GPU-ccc"}, + }, + {name: "no capabilities stored", capabilities: "", want: nil}, + {name: "unparseable payload", capabilities: `not json`, want: nil}, + {name: "no render devices", capabilities: `{"boot_id":"boot-1","render_device_details":[]}`, want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := physicalGPUKeys([]byte(tt.capabilities)) + if !slices.Equal(got, tt.want) { + t.Fatalf("physicalGPUKeys() = %v, want %v", got, tt.want) + } + }) + } +} + +type stubNodeRepository struct { + nodes []*nodepool.Node +} + +func (s *stubNodeRepository) List(context.Context) ([]*nodepool.Node, error) { return s.nodes, nil } + +func (s *stubNodeRepository) GetByID(context.Context, int) (*nodepool.Node, error) { + return nil, nodepool.ErrNodeNotFound +} + +func (s *stubNodeRepository) Create(context.Context, nodepool.CreateNodeInput) (*nodepool.Node, error) { + return nil, nodepool.ErrNodeNotFound +} + +func (s *stubNodeRepository) Update(context.Context, int, nodepool.UpdateNodeInput) (*nodepool.Node, error) { + return nil, nodepool.ErrNodeNotFound +} + +func (s *stubNodeRepository) Delete(context.Context, int) error { return nil } + +func (s *stubNodeRepository) UpdateHealth(context.Context, int, bool, int, int) error { return nil } + +// The node list is the admin's inventory view, so it must carry the stored +// capability report, its age, and the derived GPU identities beside the +// existing node fields. +func TestHandleListNodesIncludesCapabilities(t *testing.T) { + hash := "sha256:abc" + refreshedAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + { + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", Enabled: true, Healthy: true, + Capabilities: json.RawMessage(`{"boot_id":"boot-1","resolved":"nvenc","render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`), + CapabilitiesHash: &hash, + CapabilitiesRefreshedAt: &refreshedAt, + }, + {ID: 2, Name: "old-node", Type: nodepool.NodeTypeProxy, URL: "http://old", Enabled: true}, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var items []map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &items); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(items) != 2 { + t.Fatalf("returned %d nodes, want 2", len(items)) + } + // Existing fields must still be present: this response is embedded, not + // rebuilt, so a client reading it today keeps working. + if items[0]["name"] != "gpu-1" || items[0]["healthy"] != true { + t.Fatalf("node fields were lost: %v", items[0]) + } + if items[0]["capabilities_hash"] != "sha256:abc" { + t.Fatalf("capabilities_hash = %v", items[0]["capabilities_hash"]) + } + if items[0]["capabilities_refreshed_at"] == nil { + t.Fatalf("capabilities_refreshed_at missing: %v", items[0]) + } + if items[0]["capabilities"] == nil { + t.Fatalf("capabilities missing: %v", items[0]) + } + keys, ok := items[0]["physical_gpu_keys"].([]any) + if !ok || len(keys) != 1 || keys[0] != "GPU-aaa" { + t.Fatalf("physical_gpu_keys = %v", items[0]["physical_gpu_keys"]) + } + // A node that never reported capabilities carries none of the new fields + // rather than empty ones a client would have to special-case. + for _, field := range []string{"capabilities", "capabilities_hash", "capabilities_refreshed_at", "physical_gpu_keys"} { + if _, present := items[1][field]; present { + t.Fatalf("node without capabilities carries %q: %v", field, items[1]) + } + } +} diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 066931957..dd0f8a524 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -246,14 +246,23 @@ type PlaybackHandler struct { tm *playback.TranscodeManager // PlanStoreV3 owns the short-lived protocol-v3 control-plane state. Router // wiring replaces the in-memory default with PostgreSQL in integrated mode. - PlanStoreV3 playback.PlanStoreV3 - v3RegistryMu sync.Mutex - v3Registry *playback.TransformationRegistryV3 - v3RegistryProbe func(context.Context, string, tonemap.Capabilities) (*playback.TransformationRegistryV3, error) - v3ToneMapProbe func(context.Context, string, string, string) (tonemap.Capabilities, error) - v3NodeCapabilitiesMu sync.Mutex - v3NodeCapabilities map[string]v3NodeCapabilityCache - v3NodeCapabilityRefresh sync.Map + PlanStoreV3 playback.PlanStoreV3 + v3RegistryMu sync.Mutex + v3Registry *playback.TransformationRegistryV3 + v3RegistryProbe func(context.Context, string, tonemap.Capabilities) (*playback.TransformationRegistryV3, error) + v3ToneMapProbe func(context.Context, string, string, string) (tonemap.Capabilities, error) + v3NodeCapabilitiesMu sync.Mutex + v3NodeCapabilities map[string]v3NodeCapabilityCache + // v3NodeCapabilityInvalidations counts invalidations per node URL, guarded + // by v3NodeCapabilitiesMu. A probe that started before the count moved + // describes hardware the health sweep has already reported as changed, so + // its result must not be installed. See RefreshNodeCapabilitiesV3. + v3NodeCapabilityInvalidations map[string]uint64 + // v3NodeCapabilityRefresh holds the nodes with a background refresh in + // flight, one at a time each. Guarded by v3NodeCapabilitiesMu, the same + // lock as the invalidation counter, so a refresh cannot release its slot in + // between an invalidation and that invalidation's claim on it. + v3NodeCapabilityRefresh map[string]struct{} v3EventOnce sync.Once v3EventQueue chan playback.RouteEventRecordV3 v3StartEffectsOnce sync.Once diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 7b120fcde..70cdaa127 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -372,6 +372,9 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } + // Snapshot the invalidation count before probing: anything this fetch + // learns describes the node as it was when the request left. + invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) requestCtx, cancel := context.WithTimeout(ctx, h.remoteToneMapProbeTimeoutV3(nodeURL)) defer cancel() info, err := fetchRemoteTranscodeCapabilities(requestCtx, nodeURL, h.JWTSecret) @@ -381,6 +384,10 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR if h.v3NodeCapabilities == nil { h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) } + if h.v3NodeCapabilityInvalidations[nodeURL] != invalidations { + h.v3NodeCapabilitiesMu.Unlock() + return v3NodeCapabilityCache{}, err + } if current, currentOK := h.v3NodeCapabilities[nodeURL]; currentOK && current.err == nil && completedAt.Before(current.expiresAt) { h.v3NodeCapabilitiesMu.Unlock() return current, nil @@ -396,6 +403,14 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR probeRequestTimeout: playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout), } h.v3NodeCapabilitiesMu.Lock() + if h.v3NodeCapabilityInvalidations[nodeURL] != invalidations { + // The node's hardware changed while this probe was in flight. Hand the + // result to this caller, which has nothing better, but leave the cache + // empty so the next lookup re-probes instead of serving it for a full + // TTL to everyone else. + h.v3NodeCapabilitiesMu.Unlock() + return entry, nil + } if h.v3NodeCapabilities == nil { h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) } @@ -404,19 +419,87 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } -func (h *PlaybackHandler) refreshRemoteCapabilitiesV3(nodeURL string) { +func (h *PlaybackHandler) nodeCapabilityInvalidationsV3(nodeURL string) uint64 { + h.v3NodeCapabilitiesMu.Lock() + defer h.v3NodeCapabilitiesMu.Unlock() + return h.v3NodeCapabilityInvalidations[nodeURL] +} + +// RefreshNodeCapabilitiesV3 discards one node's cached capability inventory and +// re-probes it in the background. It exists for the node health sweep, which +// learns from a node's advertised capability hash that its hardware changed +// long before this cache's freshness window would expire — and a cache that +// outlives the hardware it describes plans transcodes onto a GPU that is gone. +func (h *PlaybackHandler) RefreshNodeCapabilitiesV3(nodeURL string) { if h == nil || nodeURL == "" { return } - if _, loaded := h.v3NodeCapabilityRefresh.LoadOrStore(nodeURL, struct{}{}); loaded { + h.v3NodeCapabilitiesMu.Lock() + delete(h.v3NodeCapabilities, nodeURL) + if h.v3NodeCapabilityInvalidations == nil { + h.v3NodeCapabilityInvalidations = make(map[string]uint64) + } + // Bumping the count is what makes the delete stick. A refresh already in + // flight fetched this node before its hardware changed, so it must neither + // re-install its answer over this delete nor be mistaken for the re-probe + // this invalidation is owed. + h.v3NodeCapabilityInvalidations[nodeURL]++ + claimed := h.claimCapabilityRefreshLockedV3(nodeURL) + h.v3NodeCapabilitiesMu.Unlock() + if claimed { + h.runCapabilityRefreshV3(nodeURL) + } +} + +func (h *PlaybackHandler) refreshRemoteCapabilitiesV3(nodeURL string) { + if h == nil || nodeURL == "" { return } + h.v3NodeCapabilitiesMu.Lock() + claimed := h.claimCapabilityRefreshLockedV3(nodeURL) + h.v3NodeCapabilitiesMu.Unlock() + if claimed { + h.runCapabilityRefreshV3(nodeURL) + } +} + +// claimCapabilityRefreshLockedV3 takes the node's single background-refresh +// slot. The caller must hold v3NodeCapabilitiesMu: taking the slot under the +// same lock as the invalidation counter is what guarantees that an invalidation +// either starts a refresh or is seen by the one already running. +func (h *PlaybackHandler) claimCapabilityRefreshLockedV3(nodeURL string) bool { + if _, inFlight := h.v3NodeCapabilityRefresh[nodeURL]; inFlight { + return false + } + if h.v3NodeCapabilityRefresh == nil { + h.v3NodeCapabilityRefresh = make(map[string]struct{}) + } + h.v3NodeCapabilityRefresh[nodeURL] = struct{}{} + return true +} + +// runCapabilityRefreshV3 probes one node in the background until its result is +// current: a probe that an invalidation overtook was discarded, so it repeats +// rather than leave the cache empty until the next viewer pays for a probe. +func (h *PlaybackHandler) runCapabilityRefreshV3(nodeURL string) { go func() { - defer h.v3NodeCapabilityRefresh.Delete(nodeURL) - ctx, cancel := context.WithTimeout(context.Background(), h.remoteToneMapProbeTimeoutV3(nodeURL)) - defer cancel() - if _, err := h.lookupRemoteCapabilitiesV3(ctx, nodeURL, false); err != nil { - slog.Debug("protocol v3 background node capability refresh failed", "component", "api", "node", logredact.SanitizeURL(nodeURL), "error", err) + for { + invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) + ctx, cancel := context.WithTimeout(context.Background(), h.remoteToneMapProbeTimeoutV3(nodeURL)) + _, err := h.lookupRemoteCapabilitiesV3(ctx, nodeURL, false) + cancel() + if err != nil { + slog.Debug("protocol v3 background node capability refresh failed", "component", "api", "node", logredact.SanitizeURL(nodeURL), "error", err) + } + h.v3NodeCapabilitiesMu.Lock() + current := h.v3NodeCapabilityInvalidations[nodeURL] == invalidations + if current { + delete(h.v3NodeCapabilityRefresh, nodeURL) + } + h.v3NodeCapabilitiesMu.Unlock() + if current { + return + } } }() } diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index d92ba55a9..cf3d387fc 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -5256,6 +5257,62 @@ func TestLookupRemoteCapabilitiesStartsCacheTTLAfterRequestCompletes(t *testing. } } +// waitForCachedNodeCapabilitiesV3 returns the first successful cache entry the +// background refresh installs for nodeURL. +func waitForCachedNodeCapabilitiesV3(t *testing.T, handler *PlaybackHandler, nodeURL string) v3NodeCapabilityCache { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + handler.v3NodeCapabilitiesMu.Lock() + entry, ok := handler.v3NodeCapabilities[nodeURL] + handler.v3NodeCapabilitiesMu.Unlock() + if ok && entry.err == nil { + return entry + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("no node capability report was cached before the deadline") + return v3NodeCapabilityCache{} +} + +// A report fetched before the health sweep reported the node's hardware changed +// must not be installed after the invalidation that change fired, and the +// invalidation still owes a re-probe: a cache that outlives the hardware it +// describes plans transcodes onto a GPU that is gone. +func TestRefreshNodeCapabilitiesDropsReportFetchedBeforeInvalidation(t *testing.T) { + var requests atomic.Int64 + firstRequest := make(chan struct{}) + release := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + transformation := "post_change" + if requests.Add(1) == 1 { + close(firstRequest) + <-release + transformation = "pre_change" + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + Transformations: []playback.TransformationV3{{Name: transformation}}, + }) + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.refreshRemoteCapabilitiesV3(remote.URL) + <-firstRequest + + // The sweep's callback lands while the earlier probe is still outstanding. + handler.RefreshNodeCapabilitiesV3(remote.URL) + close(release) + + entry := waitForCachedNodeCapabilitiesV3(t, handler, remote.URL) + if len(entry.transformations) != 1 || entry.transformations[0].Name != "post_change" { + t.Fatalf("cached transformations = %v, want the report fetched after the invalidation", entry.transformations) + } + if got := requests.Load(); got != 2 { + t.Fatalf("node was probed %d time(s), want the invalidation's own re-probe and no more", got) + } +} + func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { probeTimeoutMillis := int64(137000) remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/api/router.go b/internal/api/router.go index bd956b20c..bd4fdcced 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -123,6 +123,7 @@ type Dependencies struct { ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) + NodeHealthChecker *nodepool.HealthChecker // periodic node health/capability sweep (may be nil) SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger EventBus cache.EventBus AdminStatsProvider handlers.AdminStatsSource @@ -1032,6 +1033,10 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.SetProfileRefreshRequester(deps.RecWorker) } playbackHandler.StartCapabilityWarmupV3(deps.AppContext) + // The health sweep sees a node's capability hash change long before this + // cache would expire, so let it invalidate directly. Wired here rather + // than at checker construction because the handler does not exist yet. + deps.NodeHealthChecker.SetCapabilitiesChangedCallback(playbackHandler.RefreshNodeCapabilitiesV3) realtimeHub := deps.PlaybackRealtimeHub if realtimeHub == nil { diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 411263171..752b717e4 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log/slog" "net/http" + "slices" "sync" "time" ) @@ -14,38 +15,56 @@ type healthResponse struct { Status string `json:"status"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash identifies the node's current hardware capability + // snapshot. A node that predates capability snapshots reports none, which + // is how the sweep tells "nothing changed" from "cannot say". + CapabilitiesHash string `json:"capabilities_hash"` } // CheckNode pings a node's /health endpoint and returns its health status, -// active job count, and reported egress bandwidth. -func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int) { +// active job count, reported egress bandwidth, and capability hash. +func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int, capabilitiesHash string) { client := &http.Client{Timeout: 5 * time.Second} ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, n.URL+"/api/v1/health", nil) if err != nil { - return false, 0, 0 + return false, 0, 0, "" } resp, err := client.Do(req) if err != nil { - return false, 0, 0 + return false, 0, 0, "" } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return false, 0, 0 + return false, 0, 0, "" } var hr healthResponse if err := json.NewDecoder(resp.Body).Decode(&hr); err != nil { - return false, 0, 0 + return false, 0, 0, "" } - return true, hr.ActiveJobs, hr.EgressKbps + return true, hr.ActiveJobs, hr.EgressKbps, hr.CapabilitiesHash } +// CapabilityFetcher retrieves one node's full capability report and the hash +// the payload identifies itself by. The payload is stored opaquely, so this +// package never has to understand — or import — the playback capability model. +// An empty hash means the payload cannot be tracked for change and must not be +// persisted. +type CapabilityFetcher func(ctx context.Context, nodeURL string) (payload []byte, hash string, err error) + +// capabilityFetchTimeout bounds one capability fetch. Node-side capability +// answers can involve ffmpeg probes on a cold cache — the node's own advertised +// probe budget reaches ~2 minutes — and the fetch runs detached from the +// health sweep, so the bound covers a genuinely cold node rather than +// abandoning it every sweep. +const capabilityFetchTimeout = 2 * time.Minute + // HealthChecker runs periodic health checks on all nodes in both pools, // updating in-memory state and optionally persisting to the database. type HealthChecker struct { @@ -53,6 +72,22 @@ type HealthChecker struct { transcodePool *TranscodePool repo *Repository // may be nil (proxy/transcode modes have no DB) interval time.Duration + + // mu guards the two injected hooks. Both are wired after construction — + // the capability-change callback because the playback handler that consumes + // it is built later, during router assembly — while the sweep may already + // be running. + mu sync.RWMutex + capFetch CapabilityFetcher + onCapabilitiesChanged func(nodeURL string) + + // capabilityRefreshes tracks the detached capability fetches so shutdown — + // and tests — can wait for them. The sweep itself must never wait on one. + capabilityRefreshes sync.WaitGroup + // capabilityRefreshInFlight holds the node ids currently being fetched, so + // a fetch that outlives the sweep that started it is not started again by + // the next sweep. Node ids are unique across both pools (one table). + capabilityRefreshInFlight sync.Map } // NewHealthChecker creates a health checker for the given pools. @@ -65,6 +100,36 @@ func NewHealthChecker(proxyPool *ProxyPool, transcodePool *TranscodePool, repo * } } +// SetCapabilityFetcher wires how the sweep retrieves a node's capability report +// once the node reports a hash it has not stored. Without one the sweep behaves +// exactly as it did before capability tracking. +func (hc *HealthChecker) SetCapabilityFetcher(fetch CapabilityFetcher) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.capFetch = fetch +} + +// SetCapabilitiesChangedCallback wires the notification fired after a node's +// capabilities were refetched and stored, so caches keyed on node capability +// can be invalidated without waiting for their own TTL. +func (hc *HealthChecker) SetCapabilitiesChangedCallback(fn func(nodeURL string)) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.onCapabilitiesChanged = fn +} + +func (hc *HealthChecker) hooks() (CapabilityFetcher, func(nodeURL string)) { + hc.mu.RLock() + defer hc.mu.RUnlock() + return hc.capFetch, hc.onCapabilitiesChanged +} + // Start runs health checks in a background goroutine. Stops when ctx is cancelled. func (hc *HealthChecker) Start(ctx context.Context) { go func() { @@ -82,11 +147,14 @@ func (hc *HealthChecker) Start(ctx context.Context) { }() } +// applyCapabilitiesFunc is a pool's copy-on-write capability writer. +type applyCapabilitiesFunc func(id int, capabilities []byte, hash string, refreshedAt time.Time) + func (hc *HealthChecker) checkAll(ctx context.Context) { var wg sync.WaitGroup - check := func(n *Node, applyHealth func(int, bool, int, int, time.Time)) { + check := func(n *Node, applyHealth func(int, bool, int, int, time.Time), applyCapabilities applyCapabilitiesFunc) { wg.Go(func() { - healthy, activeJobs, egressKbps := CheckNode(ctx, n) + healthy, activeJobs, egressKbps, capabilitiesHash := CheckNode(ctx, n) // Publish the result through the pool lock so readers never see // a Node struct mutated in place (the pool swaps in a copy). @@ -103,13 +171,151 @@ func (hc *HealthChecker) checkAll(ctx context.Context) { slog.ErrorContext(ctx, "failed to persist node health", "component", "nodepool", "id", n.ID, "error", err) } } + + if healthy && capabilitiesHash != "" && capabilitiesHash != storedCapabilitiesHash(n) { + hc.startCapabilityRefresh(ctx, n, applyCapabilities) + } }) } for _, n := range hc.proxyPool.Nodes() { - check(n, hc.proxyPool.ApplyHealth) + check(n, hc.proxyPool.ApplyHealth, hc.proxyPool.ApplyCapabilities) } for _, n := range hc.transcodePool.Nodes() { - check(n, hc.transcodePool.ApplyHealth) + check(n, hc.transcodePool.ApplyHealth, hc.transcodePool.ApplyCapabilities) } wg.Wait() } + +// startCapabilityRefresh runs one node's capability fetch off the sweep's +// WaitGroup, deduplicated per node id. +// +// The fetch budget is larger than the sweep interval by design (a cold node +// runs ffmpeg probes to answer), so waiting for it inside the sweep would let +// one slow node stretch every other node's health cadence past that interval — +// and pool health is what routes streams away from a node that died. The +// in-flight guard is what keeps the detached fetches from stacking up: without +// it, a node that answers slower than the interval would collect one goroutine +// per sweep, since a fetch that has not completed cannot have moved the stored +// hash that triggered it. +func (hc *HealthChecker) startCapabilityRefresh(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) { + if fetch, _ := hc.hooks(); fetch == nil { + return + } + if _, loaded := hc.capabilityRefreshInFlight.LoadOrStore(n.ID, struct{}{}); loaded { + return + } + hc.capabilityRefreshes.Add(1) + go func() { + defer hc.capabilityRefreshes.Done() + defer hc.capabilityRefreshInFlight.Delete(n.ID) + hc.refreshCapabilities(ctx, n, applyCapabilities) + }() +} + +// waitForCapabilityRefreshes blocks until every detached capability fetch +// started so far has finished. Callers must not hold the sweep open on it. +func (hc *HealthChecker) waitForCapabilityRefreshes() { + hc.capabilityRefreshes.Wait() +} + +// refreshCapabilities fetches and stores one node's capability report. A +// failure leaves the stored row alone and is retried on the next sweep, because +// a fetch that failed is no evidence about what the node has. +func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) { + fetch, onChanged := hc.hooks() + if fetch == nil { + return + } + fetchCtx, cancel := context.WithTimeout(ctx, capabilityFetchTimeout) + defer cancel() + payload, hash, err := fetch(fetchCtx, n.URL) + if err != nil { + slog.WarnContext(ctx, "node capability fetch failed", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, "error", err) + return + } + if hash == "" || len(payload) == 0 { + // A hash is what makes the payload trackable; storing one without it + // would refetch every sweep forever. Never synthesize one here — the + // node is the only thing that knows what it hashed. + slog.WarnContext(ctx, "node capability report carries no hash", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + return + } + + refreshedAt := time.Now() + if hc.repo != nil { + if err := hc.repo.UpdateCapabilities(ctx, n.ID, payload, hash, refreshedAt); err != nil { + slog.WarnContext(ctx, "failed to persist node capabilities", "component", "nodepool", + "id", n.ID, "name", n.Name, "error", err) + return + } + } + logCapabilityChange(ctx, n, payload) + if applyCapabilities != nil { + applyCapabilities(n.ID, payload, hash, refreshedAt) + } + if onChanged != nil { + onChanged(n.URL) + } +} + +func storedCapabilitiesHash(n *Node) string { + if n == nil || n.CapabilitiesHash == nil { + return "" + } + return *n.CapabilitiesHash +} + +// capabilityDriftView is the minimal projection this package parses out of an +// otherwise opaque capability payload. It is deliberately local and partial: +// nodepool must not depend on playback, and drift only needs to know which +// backends were verified and which render devices existed. +type capabilityDriftView struct { + Resolved string `json:"resolved"` + DetectedBackends []struct { + Backend string `json:"backend"` + Verified bool `json:"verified"` + } `json:"detected_backends"` + RenderDevices []string `json:"render_devices"` +} + +// logCapabilityChange records what changed between the stored report and the +// new one. Losing a verified backend or a render device is the case worth +// waking an operator for: it means a node that was picked for hardware work +// silently became less capable, which otherwise only shows up as slow or +// failing transcodes. +func logCapabilityChange(ctx context.Context, n *Node, payload []byte) { + if len(n.Capabilities) == 0 { + slog.InfoContext(ctx, "node capabilities stored", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + return + } + var previous, current capabilityDriftView + if json.Unmarshal(n.Capabilities, &previous) != nil || json.Unmarshal(payload, ¤t) != nil { + return + } + verifiedNow := make(map[string]bool, len(current.DetectedBackends)) + for _, backend := range current.DetectedBackends { + verifiedNow[backend.Backend] = backend.Verified + } + var lostBackends []string + for _, backend := range previous.DetectedBackends { + if backend.Verified && !verifiedNow[backend.Backend] { + lostBackends = append(lostBackends, backend.Backend) + } + } + var lostDevices []string + for _, device := range previous.RenderDevices { + if !slices.Contains(current.RenderDevices, device) { + lostDevices = append(lostDevices, device) + } + } + if len(lostBackends) == 0 && len(lostDevices) == 0 { + return + } + slog.WarnContext(ctx, "node capability drift", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, + "lost_verified_backends", lostBackends, "lost_render_devices", lostDevices, + "previous_resolved", previous.Resolved, "resolved", current.Resolved) +} diff --git a/internal/nodepool/health_test.go b/internal/nodepool/health_test.go new file mode 100644 index 000000000..53b50cdb8 --- /dev/null +++ b/internal/nodepool/health_test.go @@ -0,0 +1,508 @@ +package nodepool + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeCapabilityFetcher records what the sweep asked for and answers with a +// canned report, standing in for a real node's /hw-capabilities. +type fakeCapabilityFetcher struct { + mu sync.Mutex + calls []string + payload []byte + hash string + err error +} + +func (f *fakeCapabilityFetcher) fetch(_ context.Context, nodeURL string) ([]byte, string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, nodeURL) + return f.payload, f.hash, f.err +} + +func (f *fakeCapabilityFetcher) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +// newHealthNode starts a stand-in node whose /health advertises capabilitiesHash. +func newHealthNode(t *testing.T, capabilitiesHash string) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(healthResponse{ + Status: "ok", + ActiveJobs: 2, + EgressKbps: 17, + CapabilitiesHash: capabilitiesHash, + }) + })) + t.Cleanup(server.Close) + return server.URL +} + +func stringPtr(s string) *string { return &s } + +// capabilityCheckerFixture wires one transcode node into a checker with a fake +// fetcher and a callback recorder, which is the whole capability flow minus the +// database (repo stays nil, as it is in proxy/transcode modes). +type capabilityCheckerFixture struct { + checker *HealthChecker + pool *TranscodePool + fetcher *fakeCapabilityFetcher + node *Node + + mu sync.Mutex + notified []string +} + +func newCapabilityCheckerFixture(t *testing.T, node *Node, fetcher *fakeCapabilityFetcher) *capabilityCheckerFixture { + t.Helper() + transcodePool := NewTranscodePool() + transcodePool.SetNodes([]*Node{node}) + fixture := &capabilityCheckerFixture{ + checker: NewHealthChecker(NewProxyPool(), transcodePool, nil), + pool: transcodePool, + fetcher: fetcher, + node: node, + } + fixture.checker.SetCapabilityFetcher(fetcher.fetch) + fixture.checker.SetCapabilitiesChangedCallback(func(nodeURL string) { + fixture.mu.Lock() + defer fixture.mu.Unlock() + fixture.notified = append(fixture.notified, nodeURL) + }) + return fixture +} + +// sweep runs one health sweep and then waits for the capability fetches it +// detached, which is what a test needs to observe. Production deliberately does +// not wait: see startCapabilityRefresh. +func (f *capabilityCheckerFixture) sweep() { + f.checker.checkAll(context.Background()) + f.checker.waitForCapabilityRefreshes() +} + +func (f *capabilityCheckerFixture) notifications() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.notified...) +} + +// storedNode returns the pool's current copy, which is what the next sweep +// compares against. +func (f *capabilityCheckerFixture) storedNode(t *testing.T) *Node { + t.Helper() + nodes := f.pool.Nodes() + if len(nodes) != 1 { + t.Fatalf("pool holds %d nodes, want 1", len(nodes)) + } + return nodes[0] +} + +const testCapabilityPayload = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}],"capability_hash":"sha256:new"}` + +// A node whose hardware changed advertises a new hash; the sweep must fetch the +// report once, publish it to the pool, and tell the capability cache to drop +// what it had. +func TestHealthCheckerFetchesCapabilitiesOnHashChange(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 1 { + t.Fatalf("capability fetches = %d, want exactly 1 per sweep", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("pool hash = %v, want sha256:new", stored.CapabilitiesHash) + } + if string(stored.Capabilities) != testCapabilityPayload { + t.Fatalf("pool payload = %s", stored.Capabilities) + } + if stored.CapabilitiesRefreshedAt == nil || stored.CapabilitiesRefreshedAt.IsZero() { + t.Fatal("pool copy carries no capability refresh time") + } + if got := fixture.notifications(); len(got) != 1 || got[0] != url { + t.Fatalf("capability change notifications = %v, want one for %s", got, url) + } +} + +// Nothing changed: the sweep must cost one health request, not a probe of every +// node every 30 seconds. +func TestHealthCheckerSkipsFetchWhenHashUnchanged(t *testing.T) { + url := newHealthNode(t, "sha256:same") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:same"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:same"), + }, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for an unchanged hash", got) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none", got) + } +} + +// A node from before capability snapshots reports no hash. It gets exactly the +// old behavior: no fetch, and nothing invented on its behalf. +func TestHealthCheckerSkipsFetchWhenNodeReportsNoHash(t *testing.T) { + url := newHealthNode(t, "") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "old-node", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for a node that reports no hash", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash != nil || stored.Capabilities != nil { + t.Fatalf("stored capabilities were synthesized: hash=%v payload=%s", stored.CapabilitiesHash, stored.Capabilities) + } + if !stored.Healthy { + t.Fatal("node without a capability hash was not marked healthy") + } +} + +// A node that is down tells us nothing about its hardware, so its stored +// inventory must survive rather than be cleared or refetched. +func TestHealthCheckerSkipsFetchWhenNodeUnhealthy(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "down", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: server.URL, Enabled: true, Healthy: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for an unhealthy node", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:old" { + t.Fatalf("stored hash = %v, want the previous report kept", stored.CapabilitiesHash) + } +} + +// A failed fetch is not evidence about the hardware: keep what is stored and +// retry on the next sweep. +func TestHealthCheckerKeepsStoredCapabilitiesOnFetchFailure(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{err: errors.New("connection refused")} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:old" { + t.Fatalf("stored hash = %v, want the previous report kept", stored.CapabilitiesHash) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none after a failed fetch", got) + } + + // The next sweep retries rather than backing off forever. + fixture.sweep() + if got := fetcher.callCount(); got != 2 { + t.Fatalf("capability fetches = %d, want a retry on the next sweep", got) + } +} + +// A payload without its own hash cannot be tracked for change; storing it would +// refetch every sweep forever. +func TestHealthCheckerRefusesUnhashedCapabilityPayload(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(`{"resolved":"nvenc"}`), hash: ""} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilitiesHash != nil || stored.Capabilities != nil { + t.Fatalf("unhashed payload was stored: hash=%v payload=%s", stored.CapabilitiesHash, stored.Capabilities) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none", got) + } +} + +// Proxy nodes carry capabilities too (they execute remux recipes), so the sweep +// must publish through the proxy pool's copy-on-write path as well. +func TestHealthCheckerAppliesCapabilitiesToProxyPool(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + proxyPool := NewProxyPool() + proxyPool.SetNodes([]*Node{{ID: 9, Name: "proxy-1", URL: url, Enabled: true}}) + checker := NewHealthChecker(proxyPool, NewTranscodePool(), nil) + checker.SetCapabilityFetcher(fetcher.fetch) + + checker.checkAll(context.Background()) + checker.waitForCapabilityRefreshes() + + stored := proxyPool.Nodes()[0] + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("proxy pool hash = %v, want sha256:new", stored.CapabilitiesHash) + } +} + +// Without a fetcher wired (proxy/transcode modes, or before wiring) the sweep +// must behave exactly as it did before capability tracking. +func TestHealthCheckerWithoutFetcherStillChecksHealth(t *testing.T) { + url := newHealthNode(t, "sha256:new") + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, Name: "gpu-1", URL: url, Enabled: true}}) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + + checker.checkAll(context.Background()) + + stored := pool.Nodes()[0] + if !stored.Healthy || stored.ActiveJobs != 2 || stored.EgressKbps != 17 { + t.Fatalf("health was not applied: %+v", stored) + } + if stored.Capabilities != nil { + t.Fatalf("capabilities = %s, want none without a fetcher", stored.Capabilities) + } +} + +// Losing a verified backend or a render device is the case worth warning about: +// the node still answers health, so nothing else would surface the loss. +func TestHealthCheckerWarnsOnCapabilityDrift(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + degraded := `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false}],"capability_hash":"sha256:degraded"}` + fetcher := &fakeCapabilityFetcher{payload: []byte(degraded), hash: "sha256:degraded"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + logged := captureSlog(t, slog.LevelWarn) + fixture.sweep() + + output := logged.String() + if !bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("no drift warning logged; log was:\n%s", output) + } + for _, want := range []string{"nvenc", "/dev/dri/renderD128"} { + if !bytes.Contains(logged.Bytes(), []byte(want)) { + t.Fatalf("drift warning does not name %q; log was:\n%s", want, output) + } + } +} + +// The first report is not drift — there is nothing to compare it against — so +// it must not warn. +func TestHealthCheckerDoesNotWarnOnFirstCapabilityStore(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + logged := captureSlog(t, slog.LevelInfo) + fixture.sweep() + + if bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("first capability store logged drift:\n%s", logged.String()) + } + if !bytes.Contains(logged.Bytes(), []byte("node capabilities stored")) { + t.Fatalf("first capability store was not logged:\n%s", logged.String()) + } +} + +// Gaining hardware is not drift either: only a loss is worth an operator's +// attention. +func TestHealthCheckerDoesNotWarnWhenCapabilitiesImprove(t *testing.T) { + url := newHealthNode(t, "sha256:better") + improved := `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128","/dev/dri/renderD129"],` + + `"detected_backends":[{"backend":"nvenc","verified":true},{"backend":"vaapi","verified":true}],` + + `"capability_hash":"sha256:better"}` + fetcher := &fakeCapabilityFetcher{payload: []byte(improved), hash: "sha256:better"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + logged := captureSlog(t, slog.LevelWarn) + fixture.sweep() + + if bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("added hardware logged as drift:\n%s", logged.String()) + } +} + +// captureSlog redirects the default logger for one test and returns its output. +func captureSlog(t *testing.T, level slog.Level) *lockedBuffer { + t.Helper() + buffer := &lockedBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(buffer, &slog.HandlerOptions{Level: level}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return buffer +} + +// lockedBuffer collects log output written from the sweep's per-node goroutines. +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(p) +} + +func (b *lockedBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + return append([]byte(nil), b.buffer.Bytes()...) +} + +func (b *lockedBuffer) String() string { return string(b.Bytes()) } + +// The fetch budget must not inherit the 5s health timeout: a cold node runs +// ffmpeg probes to answer a capability request. +func TestCapabilityFetchTimeoutExceedsHealthTimeout(t *testing.T) { + if capabilityFetchTimeout <= 5*time.Second { + t.Fatalf("capabilityFetchTimeout = %s, want more than the health request budget", capabilityFetchTimeout) + } +} + +// blockedSweepTimeout only bounds a failure: on a correct sweep the wait below +// is satisfied by the sweep returning, not by elapsed time. +const blockedSweepTimeout = 30 * time.Second + +// blockingFetcher answers only after it is released, so a test can hold one +// node's capability fetch open and observe what the sweep does meanwhile. +type blockingFetcher struct { + started chan struct{} + release chan struct{} + calls atomic.Int64 +} + +func newBlockingFetcher() *blockingFetcher { + return &blockingFetcher{started: make(chan struct{}, 1), release: make(chan struct{})} +} + +func (f *blockingFetcher) fetch(ctx context.Context, _ string) ([]byte, string, error) { + f.calls.Add(1) + select { + case f.started <- struct{}{}: + default: + } + select { + case <-f.release: + return []byte(testCapabilityPayload), "sha256:new", nil + case <-ctx.Done(): + return nil, "", ctx.Err() + } +} + +// The capability fetch budget is larger than the sweep interval, so a fetch +// that ran inside the sweep would stretch every other node's liveness update +// past that interval — and pool health is what routes streams away from a node +// that just died. The sweep must return while the fetch is still outstanding. +func TestHealthCheckerSweepDoesNotWaitForCapabilityFetch(t *testing.T) { + slowURL := newHealthNode(t, "sha256:new") + // The second node advertises no hash, so its own health is all the sweep + // has to do for it. + steadyURL := newHealthNode(t, "") + pool := NewTranscodePool() + pool.SetNodes([]*Node{ + {ID: 1, Name: "slow-gpu", URL: slowURL, Enabled: true}, + {ID: 2, Name: "steady-gpu", URL: steadyURL, Enabled: true}, + }) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + fetcher := newBlockingFetcher() + checker.SetCapabilityFetcher(fetcher.fetch) + + sweepDone := make(chan struct{}) + go func() { + defer close(sweepDone) + checker.checkAll(context.Background()) + }() + + <-fetcher.started + select { + case <-sweepDone: + case <-time.After(blockedSweepTimeout): + t.Fatal("sweep is still running while one node's capability fetch is outstanding") + } + + for _, n := range pool.Nodes() { + if !n.Healthy || n.LastHealthCheck == nil { + t.Fatalf("node %s health was not applied during the blocked fetch: %+v", n.Name, n) + } + } + + close(fetcher.release) + checker.waitForCapabilityRefreshes() + stored := pool.Nodes()[0] + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("detached fetch did not publish its report: hash = %v", stored.CapabilitiesHash) + } +} + +// A fetch that outlives its sweep has not moved the stored hash, so the next +// sweep sees the same mismatch. Without a per-node guard that would start a +// second fetch of the same node on every sweep until the first one answers. +func TestHealthCheckerDoesNotStackCapabilityFetchesForOneNode(t *testing.T) { + url := newHealthNode(t, "sha256:new") + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, Name: "slow-gpu", URL: url, Enabled: true}}) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + fetcher := newBlockingFetcher() + checker.SetCapabilityFetcher(fetcher.fetch) + + checker.checkAll(context.Background()) + <-fetcher.started + checker.checkAll(context.Background()) + + if got := fetcher.calls.Load(); got != 1 { + t.Fatalf("capability fetches = %d while the first is still outstanding, want 1", got) + } + + close(fetcher.release) + checker.waitForCapabilityRefreshes() + + // Once the report is stored the hash matches, so no further fetch is due. + checker.checkAll(context.Background()) + checker.waitForCapabilityRefreshes() + if got := fetcher.calls.Load(); got != 1 { + t.Fatalf("capability fetches = %d after the report was stored, want 1", got) + } +} diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index 95c0480ad..6e4d0e1c2 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -62,3 +62,11 @@ func (p *ProxyPool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int defer p.mu.Unlock() applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) } + +// ApplyCapabilities records a freshly fetched capability report by swapping the +// node for an updated copy, keeping published *Node values immutable. +func (p *ProxyPool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt) +} diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 21a48e9a6..be98cadb7 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -2,6 +2,7 @@ package nodepool import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -33,6 +34,16 @@ type Node struct { EgressKbps int `json:"egress_kbps"` // health-reported rolling egress average LastHealthCheck *time.Time `json:"last_health_check"` CreatedAt time.Time `json:"created_at"` + // Capabilities is the node's last stored capability report, verbatim as the + // node served it. Kept opaque here: nodepool must not depend on playback, + // and readers that need fields parse the ones they need. + Capabilities json.RawMessage `json:"capabilities,omitempty"` + // CapabilitiesHash identifies Capabilities. The health sweep compares it + // against the hash a node reports to decide whether to refetch. + CapabilitiesHash *string `json:"capabilities_hash,omitempty"` + // CapabilitiesRefreshedAt is when Capabilities was last fetched — the age of + // the inventory, not of the last health check. + CapabilitiesRefreshedAt *time.Time `json:"capabilities_refreshed_at,omitempty"` } // CreateNodeInput holds the fields for creating a new node. @@ -99,37 +110,39 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at` +const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at` func scanNode(row pgx.Row) (*Node, error) { var n Node + // jsonb is scanned as raw bytes rather than into json.RawMessage directly so + // a NULL column stays nil instead of decoding through the JSON codec. + var capabilities []byte err := row.Scan( &n.ID, &n.Name, &n.Type, &n.URL, &n.Enabled, &n.Healthy, &n.ActiveJobs, &n.Group, &n.MaxJobs, &n.MaxBandwidthKbps, &n.EgressKbps, &n.LastHealthCheck, &n.CreatedAt, + &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, ) if err != nil { return nil, err } + if len(capabilities) > 0 { + n.Capabilities = json.RawMessage(capabilities) + } return &n, nil } func scanNodes(rows pgx.Rows) ([]*Node, error) { var nodes []*Node for rows.Next() { - var n Node - if err := rows.Scan( - &n.ID, &n.Name, &n.Type, &n.URL, - &n.Enabled, &n.Healthy, &n.ActiveJobs, - &n.Group, &n.MaxJobs, - &n.MaxBandwidthKbps, &n.EgressKbps, - &n.LastHealthCheck, &n.CreatedAt, - ); err != nil { + // pgx.Rows satisfies pgx.Row, so both paths share one column list. + n, err := scanNode(rows) + if err != nil { return nil, err } - nodes = append(nodes, &n) + nodes = append(nodes, n) } return nodes, rows.Err() } @@ -252,5 +265,22 @@ func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, act return nil } +// UpdateCapabilities persists a freshly fetched capability report together with +// the hash that identifies it. The three columns are written in one statement +// so a reader never sees a payload beside a hash from a different report. +func (r *Repository) UpdateCapabilities(ctx context.Context, id int, capabilities []byte, hash string, refreshedAt time.Time) error { + tag, err := r.pool.Exec(ctx, + `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4 + WHERE id = $1`, + id, capabilities, hash, refreshedAt) + if err != nil { + return fmt.Errorf("update node capabilities: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrNodeNotFound + } + return nil +} + // Sentinel errors. var ErrNodeNotFound = errors.New("stream node not found") diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go new file mode 100644 index 000000000..1acdc58eb --- /dev/null +++ b/internal/nodepool/repository_capabilities_test.go @@ -0,0 +1,95 @@ +package nodepool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// newNodeTestPool follows the repository-wide convention for tests that need a +// real database: skip unless one is configured, and skip again if it predates +// the migration under test rather than failing on a missing column. +func newNodeTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + var column *string + if err := pool.QueryRow(ctx, + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'stream_nodes' AND column_name = 'capabilities_hash'`).Scan(&column); err != nil { + t.Skip("test database has not applied the node capabilities migration") + } + return pool +} + +// Stored capabilities are what makes GPU inventory survive an API restart, so +// the payload, its hash, and its age must all come back through an ordinary +// List — the same read the pools and the admin API use. +func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("capability-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://capability-test-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + if node.Capabilities != nil || node.CapabilitiesHash != nil || node.CapabilitiesRefreshedAt != nil { + t.Fatalf("new node already carries capabilities: %+v", node) + } + + payload := json.RawMessage(`{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"]}`) + refreshedAt := time.Now().UTC().Truncate(time.Millisecond) + if err := repo.UpdateCapabilities(ctx, node.ID, payload, "sha256:abc", refreshedAt); err != nil { + t.Fatalf("update capabilities: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + var stored, want map[string]any + if err := json.Unmarshal(reloaded.Capabilities, &stored); err != nil { + t.Fatalf("stored capabilities are not json: %v", err) + } + if err := json.Unmarshal(payload, &want); err != nil { + t.Fatal(err) + } + if fmt.Sprint(stored) != fmt.Sprint(want) { + t.Fatalf("stored capabilities = %v, want %v", stored, want) + } + if reloaded.CapabilitiesHash == nil || *reloaded.CapabilitiesHash != "sha256:abc" { + t.Fatalf("stored hash = %v", reloaded.CapabilitiesHash) + } + if reloaded.CapabilitiesRefreshedAt == nil || !reloaded.CapabilitiesRefreshedAt.UTC().Equal(refreshedAt) { + t.Fatalf("stored refresh time = %v, want %v", reloaded.CapabilitiesRefreshedAt, refreshedAt) + } +} + +func TestRepositoryUpdateCapabilitiesUnknownNode(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + err := repo.UpdateCapabilities(context.Background(), -1, []byte(`{}`), "sha256:abc", time.Now()) + if !errors.Is(err, ErrNodeNotFound) { + t.Fatalf("err = %v, want ErrNodeNotFound", err) + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 2367fa334..75758f19f 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -1,6 +1,7 @@ package nodepool import ( + "encoding/json" "strings" "sync" "time" @@ -87,6 +88,14 @@ func (p *TranscodePool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) } +// ApplyCapabilities records a freshly fetched capability report by swapping the +// node for an updated copy, keeping published *Node values immutable. +func (p *TranscodePool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt) +} + // applyNodeHealth replaces the slice entry for id with an updated copy. func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { for i, n := range nodes { @@ -102,3 +111,20 @@ func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps return } } + +// applyNodeCapabilities replaces the slice entry for id with a copy carrying +// the new capability report. The payload is cloned because the caller's buffer +// (a decoded HTTP response) is not ours to publish. +func applyNodeCapabilities(nodes []*Node, id int, capabilities []byte, hash string, refreshedAt time.Time) { + for i, n := range nodes { + if n.ID != id { + continue + } + clone := *n + clone.Capabilities = append(json.RawMessage(nil), capabilities...) + clone.CapabilitiesHash = &hash + clone.CapabilitiesRefreshedAt = &refreshedAt + nodes[i] = &clone + return + } +} diff --git a/internal/playback/capabilityhash.go b/internal/playback/capabilityhash.go new file mode 100644 index 000000000..198dbbbd3 --- /dev/null +++ b/internal/playback/capabilityhash.go @@ -0,0 +1,178 @@ +package playback + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "slices" + "strings" + + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// ComputeCapabilityHash summarizes what a host has and can do into one +// comparable token, so a reader (the node health sweep, an operator) detects +// change without diffing a whole report or re-probing. +// +// Only hardware identity and capability are hashed. Fields that describe the +// report rather than the host — Source, NodeURL, ProbeRequestTimeoutMillis, and +// the hash itself — are excluded, because a report of unchanged hardware must +// keep its hash no matter who asked for it or how. IntelDetected is excluded as +// well: it is derived from the render devices already covered. +// +// Every slice is ordered here rather than trusted from the input: probe and +// filesystem enumeration order is incidental, so two reports of the same host +// hash identically regardless of it. +func ComputeCapabilityHash(info HWAccelInfo) string { + payload, err := json.Marshal(canonicalCapabilities(info)) + if err != nil { + // Unreachable: every field below is a string, bool, or slice of them. + return "" + } + sum := sha256.Sum256(payload) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// canonicalCapability is the hashed projection of an HWAccelInfo. It is a +// distinct type rather than a reordered copy of the input so that adding a +// field to HWAccelInfo never silently changes every node's hash. +type canonicalCapability struct { + Resolved string `json:"resolved"` + BootID string `json:"boot_id"` + RenderDevices []string `json:"render_devices"` + RenderDeviceDetails []canonicalRenderDevice `json:"render_device_details"` + DetectedBackends []canonicalDetectedBackend `json:"detected_backends"` + Transformations []canonicalTransformation `json:"transformations"` + ToneMapCapabilities []canonicalToneMap `json:"tone_map_capabilities"` +} + +// canonicalRenderDevice serializes a device strongest identity first: the GPU +// uuid identifies the card anywhere, the PCI address identifies a slot, and the +// path only identifies an enumeration position. +type canonicalRenderDevice struct { + GPUUUID string `json:"gpu_uuid"` + PCIAddress string `json:"pci_address"` + Path string `json:"path"` + Description string `json:"description"` +} + +type canonicalDetectedBackend struct { + Backend string `json:"backend"` + Verified bool `json:"verified"` + Devices []string `json:"devices"` + Device string `json:"device"` + Reason string `json:"reason"` +} + +type canonicalTransformation struct { + Name string `json:"name"` + Executor string `json:"executor"` + RecipeVersion string `json:"recipe_version"` + ValidatedClaims []string `json:"validated_claims"` +} + +type canonicalToneMap struct { + Mode string `json:"mode"` + Backend string `json:"backend"` + Filter string `json:"filter"` + SourceKinds []string `json:"source_kinds"` +} + +func canonicalCapabilities(info HWAccelInfo) canonicalCapability { + return canonicalCapability{ + Resolved: info.Resolved, + BootID: info.BootID, + RenderDevices: sortedStrings(info.RenderDevices), + RenderDeviceDetails: canonicalRenderDevices(info.RenderDeviceDetails), + DetectedBackends: canonicalDetectedBackends(info.DetectedBackends), + Transformations: canonicalTransformations(info.Transformations), + ToneMapCapabilities: canonicalToneMaps(info.ToneMapCapabilities), + } +} + +func canonicalRenderDevices(details []RenderDeviceInfo) []canonicalRenderDevice { + out := make([]canonicalRenderDevice, 0, len(details)) + for _, detail := range details { + out = append(out, canonicalRenderDevice{ + GPUUUID: detail.GPUUUID, + PCIAddress: detail.PCIAddress, + Path: detail.Path, + Description: detail.Description, + }) + } + slices.SortFunc(out, func(a, b canonicalRenderDevice) int { + return strings.Compare(a.Path, b.Path) + }) + return out +} + +func canonicalDetectedBackends(backends []DetectedBackend) []canonicalDetectedBackend { + out := make([]canonicalDetectedBackend, 0, len(backends)) + for _, backend := range backends { + out = append(out, canonicalDetectedBackend{ + Backend: backend.Backend, + Verified: backend.Verified, + Devices: sortedStrings(backend.Devices), + Device: backend.Device, + Reason: backend.Reason, + }) + } + slices.SortFunc(out, func(a, b canonicalDetectedBackend) int { + return strings.Compare(a.Backend, b.Backend) + }) + return out +} + +func canonicalTransformations(transformations []TransformationV3) []canonicalTransformation { + out := make([]canonicalTransformation, 0, len(transformations)) + for _, transformation := range transformations { + out = append(out, canonicalTransformation{ + Name: transformation.Name, + Executor: transformation.Executor, + RecipeVersion: transformation.RecipeVersion, + ValidatedClaims: sortedStrings(transformation.ValidatedClaims), + }) + } + slices.SortFunc(out, func(a, b canonicalTransformation) int { + if byName := strings.Compare(a.Name, b.Name); byName != 0 { + return byName + } + return strings.Compare(a.RecipeVersion, b.RecipeVersion) + }) + return out +} + +func canonicalToneMaps(capabilities tonemap.Capabilities) []canonicalToneMap { + out := make([]canonicalToneMap, 0, len(capabilities)) + for _, capability := range capabilities { + kinds := make([]string, 0, len(capability.SourceKinds)) + for _, kind := range capability.SourceKinds { + kinds = append(kinds, string(kind)) + } + out = append(out, canonicalToneMap{ + Mode: string(capability.Mode), + Backend: capability.Backend, + Filter: capability.Filter, + SourceKinds: sortedStrings(kinds), + }) + } + slices.SortFunc(out, func(a, b canonicalToneMap) int { + if byMode := strings.Compare(a.Mode, b.Mode); byMode != 0 { + return byMode + } + if byBackend := strings.Compare(a.Backend, b.Backend); byBackend != 0 { + return byBackend + } + return strings.Compare(a.Filter, b.Filter) + }) + return out +} + +// sortedStrings returns an ordered copy, leaving the caller's slice alone: the +// input is a live report another goroutine may still be serving. +func sortedStrings(values []string) []string { + out := make([]string, 0, len(values)) + out = append(out, values...) + slices.Sort(out) + return out +} diff --git a/internal/playback/capabilityhash_test.go b/internal/playback/capabilityhash_test.go new file mode 100644 index 000000000..e08a37da3 --- /dev/null +++ b/internal/playback/capabilityhash_test.go @@ -0,0 +1,129 @@ +package playback + +import ( + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +func sampleCapabilityInfo() HWAccelInfo { + return HWAccelInfo{ + Resolved: "nvenc", + BootID: "5b2c1f0e-1111-4a2b-9c3d-2f6e7a8b9c0d", + RenderDevices: []string{"/dev/dri/renderD128", "/dev/dri/renderD129"}, + RenderDeviceDetails: []RenderDeviceInfo{ + {Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", GPUUUID: "GPU-aaa", Description: "NVIDIA GPU (0x2204)"}, + {Path: "/dev/dri/renderD129", PCIAddress: "0000:04:00.0", Description: "Intel GPU (0x9a49)"}, + }, + DetectedBackends: []DetectedBackend{ + {Backend: "nvenc", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "vaapi", Verified: false, Devices: []string{"/dev/dri/renderD129", "/dev/dri/renderD128"}, Reason: "h264_vaapi encoder unavailable"}, + }, + Transformations: []TransformationV3{ + {Name: "tone_map", Executor: "node", RecipeVersion: "v2", ValidatedClaims: []string{"hdr10", "hlg"}}, + {Name: "audio_to_aac", Executor: "node", RecipeVersion: "v1"}, + }, + ToneMapCapabilities: tonemap.Capabilities{ + {Mode: tonemap.ModeHardware, Backend: "nvenc", Filter: "tonemap_cuda", SourceKinds: []tonemap.SourceKind{"hdr10", "hlg"}}, + {Mode: tonemap.ModeSoftware, Backend: "software", Filter: "tonemap", SourceKinds: []tonemap.SourceKind{"hdr10"}}, + }, + } +} + +// A node reports the same hardware twice with its slices enumerated in a +// different order — probe order and directory listing order are incidental. If +// that moved the hash, every sweep would look like a hardware change and refetch +// the whole inventory forever. +func TestComputeCapabilityHashIgnoresSliceOrder(t *testing.T) { + info := sampleCapabilityInfo() + shuffled := sampleCapabilityInfo() + shuffled.RenderDevices = []string{"/dev/dri/renderD129", "/dev/dri/renderD128"} + shuffled.RenderDeviceDetails = []RenderDeviceInfo{shuffled.RenderDeviceDetails[1], shuffled.RenderDeviceDetails[0]} + shuffled.DetectedBackends = []DetectedBackend{shuffled.DetectedBackends[1], shuffled.DetectedBackends[0]} + shuffled.DetectedBackends[0].Devices = []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} + shuffled.Transformations = []TransformationV3{shuffled.Transformations[1], shuffled.Transformations[0]} + shuffled.Transformations[1].ValidatedClaims = []string{"hlg", "hdr10"} + shuffled.ToneMapCapabilities = tonemap.Capabilities{shuffled.ToneMapCapabilities[1], shuffled.ToneMapCapabilities[0]} + shuffled.ToneMapCapabilities[1].SourceKinds = []tonemap.SourceKind{"hlg", "hdr10"} + + if got, want := ComputeCapabilityHash(shuffled), ComputeCapabilityHash(info); got != want { + t.Fatalf("reordered report hashed differently:\n got %s\nwant %s", got, want) + } +} + +// Fields that describe the report rather than the host must not move the hash: +// otherwise the same node hashes differently depending on who asked. +func TestComputeCapabilityHashIgnoresReportMetadata(t *testing.T) { + info := sampleCapabilityInfo() + want := ComputeCapabilityHash(info) + + info.Source = "remote" + info.NodeURL = "http://node-7:8080" + info.ProbeRequestTimeoutMillis = 42_000 + info.CapabilityHash = "sha256:stale" + + if got := ComputeCapabilityHash(info); got != want { + t.Fatalf("report metadata changed the hash:\n got %s\nwant %s", got, want) + } +} + +func TestComputeCapabilityHashIsPrefixedSHA256(t *testing.T) { + hash := ComputeCapabilityHash(sampleCapabilityInfo()) + if !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("hash = %q, want a sha256: prefix", hash) + } + if len(hash) != len("sha256:")+64 { + t.Fatalf("hash = %q, want 64 hex digits after the prefix", hash) + } +} + +// Every hardware or capability change must move the hash, since the hash is the +// only thing the health sweep looks at before deciding nothing changed. +func TestComputeCapabilityHashDetectsRealChanges(t *testing.T) { + base := ComputeCapabilityHash(sampleCapabilityInfo()) + tests := []struct { + name string + mutate func(*HWAccelInfo) + }{ + {"resolved backend", func(i *HWAccelInfo) { i.Resolved = "vaapi" }}, + {"boot id", func(i *HWAccelInfo) { i.BootID = "0000ffff-2222-4a2b-9c3d-2f6e7a8b9c0d" }}, + {"render device removed", func(i *HWAccelInfo) { + i.RenderDevices = i.RenderDevices[:1] + i.RenderDeviceDetails = i.RenderDeviceDetails[:1] + }}, + {"pci address", func(i *HWAccelInfo) { i.RenderDeviceDetails[0].PCIAddress = "0000:07:00.0" }}, + {"gpu uuid", func(i *HWAccelInfo) { i.RenderDeviceDetails[0].GPUUUID = "GPU-bbb" }}, + {"device description", func(i *HWAccelInfo) { i.RenderDeviceDetails[1].Description = "AMD GPU" }}, + {"backend verification lost", func(i *HWAccelInfo) { i.DetectedBackends[0].Verified = false }}, + {"backend failure reason", func(i *HWAccelInfo) { i.DetectedBackends[1].Reason = "no driver" }}, + {"verified device", func(i *HWAccelInfo) { i.DetectedBackends[0].Device = "/dev/dri/renderD129" }}, + {"transformation recipe version", func(i *HWAccelInfo) { i.Transformations[0].RecipeVersion = "v3" }}, + {"validated claims", func(i *HWAccelInfo) { i.Transformations[0].ValidatedClaims = []string{"hdr10"} }}, + {"tone map filter", func(i *HWAccelInfo) { i.ToneMapCapabilities[0].Filter = "tonemap_opencl" }}, + {"tone map source kinds", func(i *HWAccelInfo) { + i.ToneMapCapabilities[0].SourceKinds = []tonemap.SourceKind{"hdr10"} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := sampleCapabilityInfo() + tt.mutate(&info) + if got := ComputeCapabilityHash(info); got == base { + t.Fatalf("%s did not change the hash (%s)", tt.name, got) + } + }) + } +} + +// An empty report still hashes, so a node with no hardware advertises a stable +// identity rather than an empty one the sweep would read as "cannot say". +func TestComputeCapabilityHashOfEmptyReport(t *testing.T) { + hash := ComputeCapabilityHash(HWAccelInfo{}) + if hash == "" { + t.Fatal("empty report produced no hash") + } + if hash == ComputeCapabilityHash(sampleCapabilityInfo()) { + t.Fatal("empty report hashed the same as a populated one") + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 5e47dcda2..460f471f6 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -9,6 +9,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "sync" "time" @@ -22,6 +23,7 @@ var ( defaultNVIDIAControlDevice = "/dev/nvidiactl" defaultNVIDIADeviceGlob = "/dev/nvidia[0-9]*" sysClassDRMDir = "/sys/class/drm" + procBootIDPath = "/proc/sys/kernel/random/boot_id" currentGOOS = runtime.GOOS hwProbeCommandTimeout = 3 * time.Second hwProbeNegativeTTL = 15 * time.Second @@ -75,6 +77,14 @@ type HWAccelInfo struct { NodeURL string `json:"node_url,omitempty"` Transformations []TransformationV3 `json:"transformations,omitempty"` ToneMapCapabilities tonemap.Capabilities `json:"tone_map_capabilities,omitempty"` + // BootID is this host's kernel boot identity (Linux only). Paired with a + // render device's PCI address it distinguishes "same GPU, same boot" from + // "same device path on a host that rebooted or was replaced". + BootID string `json:"boot_id,omitempty"` + // CapabilityHash summarizes every hardware-identity and capability field + // below, so a reader can detect change without diffing the whole report. + // Set by the node that serves the report; see ComputeCapabilityHash. + CapabilityHash string `json:"capability_hash,omitempty"` // ProbeRequestTimeoutMillis is the caller-side budget for this node's // effective tone-map probe matrix, including endpoint and transport slack. ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms,omitempty"` @@ -131,6 +141,7 @@ func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hw RenderDeviceDetails: renderDeviceDetails(candidates.renderDevices), IntelDetected: candidates.intelPresent, DetectedBackends: detected, + BootID: detectBootID(), Source: "local", } } @@ -656,7 +667,15 @@ func detectRenderDevice(driDir string) string { // RenderDeviceInfo describes one render device for operator-facing surfaces. type RenderDeviceInfo struct { - Path string `json:"path"` + Path string `json:"path"` + // PCIAddress is the device's sysfs PCI slot (e.g. 0000:03:00.0). It is + // stable across reboots for a card that stays in its slot, which /dev/dri + // paths are not, so it — not Path — identifies the hardware. + PCIAddress string `json:"pci_address,omitempty"` + // GPUUUID is NVIDIA's own permanent GPU identity, reported only when + // nvidia-smi is installed. It survives a card moving between slots and + // hosts, so it outranks PCIAddress wherever both are present. + GPUUUID string `json:"gpu_uuid,omitempty"` Description string `json:"description"` } @@ -696,10 +715,132 @@ func readSysfsID(path string) string { func renderDeviceDetails(devices []string) []RenderDeviceInfo { details := make([]RenderDeviceInfo, 0, len(devices)) for _, device := range devices { + pciAddress := renderDevicePCIAddress(device) details = append(details, RenderDeviceInfo{ Path: device, + PCIAddress: pciAddress, + GPUUUID: renderDeviceGPUUUID(device, pciAddress), Description: describeRenderDevice(device), }) } return details } + +// renderDevicePCIAddress resolves the sysfs device symlink behind a render node +// and returns its PCI slot. Best effort: an unresolvable link (a virtual or +// non-PCI device, a restricted sysfs) yields an empty address rather than an +// error, because a missing identity only weakens inventory, never breaks it. +func renderDevicePCIAddress(renderDevPath string) string { + name := filepath.Base(renderDevPath) + devicePath := filepath.Join(sysClassDRMDir, name, "device") + // sysfs exposes this as a symlink into the PCI tree. Anything else is a + // device with no PCI identity, and its own directory name would be "device". + info, err := os.Lstat(devicePath) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + return "" + } + resolved, err := filepath.EvalSymlinks(devicePath) + if err != nil { + return "" + } + return filepath.Base(resolved) +} + +// renderDeviceGPUUUID returns NVIDIA's permanent GPU identity for a render +// device. Only NVIDIA-vendor devices are looked up: no other vendor publishes +// such an id, so querying for them would only cost a subprocess. +func renderDeviceGPUUUID(renderDevPath, pciAddress string) string { + if pciAddress == "" || !isNVIDIADevice(renderDevPath) { + return "" + } + return nvidiaGPUUUIDsByPCIAddress()[normalizePCIAddress(pciAddress)] +} + +// nvidiaSMIQueryTimeout bounds the single nvidia-smi invocation. A wedged +// driver makes nvidia-smi hang, and hardware inventory must not inherit that. +var nvidiaSMIQueryTimeout = 3 * time.Second + +// nvidiaSMIQuery is the execution seam for the GPU uuid listing; tests replace +// it rather than installing a fake binary on PATH. +var nvidiaSMIQuery = runNVIDIASMIQuery + +// nvidiaGPUUUIDs caches the one nvidia-smi listing this process makes. GPU +// identities cannot change without a reboot, so a second query could only cost +// a subprocess to learn the same answer. +var nvidiaGPUUUIDs struct { + once sync.Once + byPCI map[string]string +} + +func runNVIDIASMIQuery(ctx context.Context) ([]byte, error) { + path, err := exec.LookPath("nvidia-smi") + if err != nil { + return nil, err + } + return exec.CommandContext(ctx, path, "--query-gpu=uuid,pci.bus_id", "--format=csv,noheader").Output() +} + +func nvidiaGPUUUIDsByPCIAddress() map[string]string { + nvidiaGPUUUIDs.once.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), nvidiaSMIQueryTimeout) + defer cancel() + output, err := nvidiaSMIQuery(ctx) + if err != nil { + // Expected on every host without the NVIDIA toolkit installed, so + // this stays at debug: the report is complete without it. + slog.Debug("nvidia-smi gpu identity query unavailable", "component", "playback", "error", err) + return + } + nvidiaGPUUUIDs.byPCI = parseNVIDIAGPUUUIDs(output) + }) + return nvidiaGPUUUIDs.byPCI +} + +// parseNVIDIAGPUUUIDs reads "csv,noheader" rows of ", " and +// keys them by normalized PCI address. Malformed rows are skipped. +func parseNVIDIAGPUUUIDs(output []byte) map[string]string { + byPCI := make(map[string]string) + for line := range strings.Lines(string(output)) { + uuid, address, ok := strings.Cut(line, ",") + if !ok { + continue + } + uuid = strings.TrimSpace(uuid) + address = normalizePCIAddress(address) + if uuid == "" || address == "" { + continue + } + byPCI[address] = uuid + } + return byPCI +} + +// normalizePCIAddress makes sysfs and nvidia-smi addresses comparable: +// sysfs prints a 16-bit domain (0000:03:00.0) and nvidia-smi a 32-bit one +// (00000000:03:00.0), and neither guarantees a case. +func normalizePCIAddress(address string) string { + address = strings.ToLower(strings.TrimSpace(address)) + domain, rest, ok := strings.Cut(address, ":") + if !ok { + return address + } + value, err := strconv.ParseUint(domain, 16, 64) + if err != nil { + return address + } + return fmt.Sprintf("%04x:%s", value, rest) +} + +// detectBootID reads the kernel's per-boot identity. It is Linux-only and +// best effort: an empty value simply means device identities cannot be scoped +// to a boot on this host. +func detectBootID() string { + if currentGOOS != "linux" { + return "" + } + data, err := os.ReadFile(procBootIDPath) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} diff --git a/internal/playback/gpuidentity_test.go b/internal/playback/gpuidentity_test.go new file mode 100644 index 000000000..4aaabaa2e --- /dev/null +++ b/internal/playback/gpuidentity_test.go @@ -0,0 +1,185 @@ +package playback + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "testing" +) + +// addPCIRenderDevice models the sysfs layout a real render node has: the drm +// entry's "device" is a symlink into the PCI tree, and the vendor id lives on +// the far side of it. +func (e *hwAccelTestEnv) addPCIRenderDevice(t *testing.T, name, vendor, pciAddress string) { + t.Helper() + if err := os.WriteFile(filepath.Join(e.driDir, name), []byte{}, 0o600); err != nil { + t.Fatalf("create render device: %v", err) + } + pciDir := filepath.Join(e.sysDir, "pci", pciAddress) + if err := os.MkdirAll(pciDir, 0o755); err != nil { + t.Fatalf("create pci dir: %v", err) + } + if err := os.WriteFile(filepath.Join(pciDir, "vendor"), []byte(vendor+"\n"), 0o644); err != nil { + t.Fatalf("write vendor file: %v", err) + } + drmDir := filepath.Join(e.sysDir, name) + if err := os.MkdirAll(drmDir, 0o755); err != nil { + t.Fatalf("create drm dir: %v", err) + } + if err := os.Symlink(pciDir, filepath.Join(drmDir, "device")); err != nil { + t.Fatalf("link drm device: %v", err) + } +} + +func (e *hwAccelTestEnv) setBootID(t *testing.T, bootID string) { + t.Helper() + path := filepath.Join(t.TempDir(), "boot_id") + if err := os.WriteFile(path, []byte(bootID+"\n"), 0o600); err != nil { + t.Fatalf("write boot id: %v", err) + } + previous := procBootIDPath + procBootIDPath = path + t.Cleanup(func() { procBootIDPath = previous }) +} + +// stubNVIDIASMI replaces the nvidia-smi invocation and clears its process-wide +// cache, which is otherwise computed once and would leak between tests. +func stubNVIDIASMI(t *testing.T, output string, err error) { + t.Helper() + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + if err != nil { + return nil, err + } + return []byte(output), nil + } + resetNVIDIAUUIDCacheForTest() + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAUUIDCacheForTest() + }) +} + +func resetNVIDIAUUIDCacheForTest() { + nvidiaGPUUUIDs.once = sync.Once{} + nvidiaGPUUUIDs.byPCI = nil +} + +func renderDeviceDetail(t *testing.T, info HWAccelInfo, path string) RenderDeviceInfo { + t.Helper() + for _, detail := range info.RenderDeviceDetails { + if detail.Path == path { + return detail + } + } + t.Fatalf("no render device detail for %s in %+v", path, info.RenderDeviceDetails) + return RenderDeviceInfo{} +} + +// The device paths under /dev/dri are assigned by enumeration order and move +// when hardware is added or removed. PCI address, GPU uuid and boot id are what +// let an operator (and the node inventory) say the GPU behind a path is still +// the same GPU. +func TestDetectHWAccelReportsHardwareIdentity(t *testing.T) { + env := setupHWAccelTest(t) + env.addPCIRenderDevice(t, "renderD128", "0x10de", "0000:03:00.0") + env.addPCIRenderDevice(t, "renderD129", "0x8086", "0000:04:00.0") + env.setBootID(t, "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111") + // nvidia-smi prints a 32-bit PCI domain in upper case; sysfs prints 16 bits. + stubNVIDIASMI(t, "GPU-11112222-3333-4444-5555-666677778888, 00000000:03:00.0\n"+ + "GPU-99990000-1111-2222-3333-444455556666, 00000000:04:00.0\n", nil) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + + if info.BootID != "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111" { + t.Fatalf("BootID = %q", info.BootID) + } + nvidia := renderDeviceDetail(t, info, filepath.Join(env.driDir, "renderD128")) + if nvidia.PCIAddress != "0000:03:00.0" { + t.Fatalf("nvidia PCIAddress = %q, want 0000:03:00.0", nvidia.PCIAddress) + } + if nvidia.GPUUUID != "GPU-11112222-3333-4444-5555-666677778888" { + t.Fatalf("nvidia GPUUUID = %q", nvidia.GPUUUID) + } + intel := renderDeviceDetail(t, info, filepath.Join(env.driDir, "renderD129")) + if intel.PCIAddress != "0000:04:00.0" { + t.Fatalf("intel PCIAddress = %q, want 0000:04:00.0", intel.PCIAddress) + } + // nvidia-smi listed this address, but the device is Intel: attributing an + // NVIDIA uuid to it would merge two distinct GPUs into one inventory entry. + if intel.GPUUUID != "" { + t.Fatalf("intel GPUUUID = %q, want empty", intel.GPUUUID) + } +} + +// A host without the NVIDIA toolkit is the common case, not a failure: the +// report must still describe the hardware it can see. +func TestDetectHWAccelOmitsGPUUUIDWhenNVIDIASMIUnavailable(t *testing.T) { + env := setupHWAccelTest(t) + env.addPCIRenderDevice(t, "renderD128", "0x10de", "0000:03:00.0") + stubNVIDIASMI(t, "", errors.New("exec: \"nvidia-smi\": executable file not found in $PATH")) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + detail := renderDeviceDetail(t, DetectHWAccelWithFFmpeg("auto", ffmpeg.path, ""), filepath.Join(env.driDir, "renderD128")) + + if detail.PCIAddress != "0000:03:00.0" { + t.Fatalf("PCIAddress = %q, want the sysfs address even without nvidia-smi", detail.PCIAddress) + } + if detail.GPUUUID != "" { + t.Fatalf("GPUUUID = %q, want empty", detail.GPUUUID) + } +} + +// A device with no PCI symlink (virtual, or a restricted sysfs) reports no +// address rather than the literal directory name behind the lookup. +func TestDetectHWAccelOmitsPCIAddressWithoutSysfsLink(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + detail := renderDeviceDetail(t, DetectHWAccelWithFFmpeg("auto", ffmpeg.path, ""), filepath.Join(env.driDir, "renderD128")) + + if detail.PCIAddress != "" { + t.Fatalf("PCIAddress = %q, want empty", detail.PCIAddress) + } +} + +func TestDetectHWAccelOmitsBootIDOffLinux(t *testing.T) { + env := setupHWAccelTest(t) + env.setBootID(t, "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111") + currentGOOS = "darwin" + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "").BootID; got != "" { + t.Fatalf("BootID = %q, want empty off Linux", got) + } +} + +func TestNormalizePCIAddress(t *testing.T) { + tests := []struct{ in, want string }{ + {"0000:03:00.0", "0000:03:00.0"}, + {"00000000:03:00.0", "0000:03:00.0"}, + {"00000000:0A:00.0", "0000:0a:00.0"}, + {" 0000:03:00.0 ", "0000:03:00.0"}, + {"not-an-address", "not-an-address"}, + {"", ""}, + } + for _, tt := range tests { + if got := normalizePCIAddress(tt.in); got != tt.want { + t.Fatalf("normalizePCIAddress(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestParseNVIDIAGPUUUIDsSkipsMalformedRows(t *testing.T) { + parsed := parseNVIDIAGPUUUIDs([]byte("GPU-aaa, 00000000:03:00.0\n\nmissing-separator\n, 00000000:05:00.0\nGPU-bbb, \n")) + if len(parsed) != 1 { + t.Fatalf("parsed = %v, want exactly the one well-formed row", parsed) + } + if parsed["0000:03:00.0"] != "GPU-aaa" { + t.Fatalf("parsed = %v", parsed) + } +} diff --git a/internal/proxy/capability_snapshot_test.go b/internal/proxy/capability_snapshot_test.go new file mode 100644 index 000000000..fd9676852 --- /dev/null +++ b/internal/proxy/capability_snapshot_test.go @@ -0,0 +1,128 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +func decodeProxyHealth(t *testing.T, server *Server) healthResponse { + t.Helper() + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var health healthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &health); err != nil { + t.Fatalf("decode health: %v", err) + } + return health +} + +// A proxy executes remux recipes, so the API tracks its capabilities the same +// way it tracks a transcode node's: by the hash health advertises. Until the +// first snapshot the field must be absent rather than a hash of nothing, which +// the sweep would treat as a real inventory. +func TestProxyHealthPublishesCapabilityHashOnlyAfterSnapshot(t *testing.T) { + server := newDownloadProxyServer(t, "capability-secret") + + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q before any snapshot, want empty", got) + } + + server.refreshCapabilitySnapshot(context.Background()) + + hash := decodeProxyHealth(t, server).CapabilitiesHash + if hash == "" { + t.Fatal("capabilities_hash is still empty after a snapshot") + } + // A second snapshot of unchanged hardware must not move the hash, or the + // sweep would refetch this proxy's inventory forever. + server.refreshCapabilitySnapshot(context.Background()) + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != hash { + t.Fatalf("capabilities_hash changed without hardware changing: %q then %q", hash, got) + } +} + +// The endpoint and the background snapshot share one assembly, so a served +// report carries the same hash health publishes. +func TestProxyCapabilitiesPublishCapabilityHash(t *testing.T) { + const secret = "capability-secret" + server := newDownloadProxyServer(t, secret) + + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash == "" { + t.Fatal("served capability report carries no capability_hash") + } + served := info + served.CapabilityHash = "" + if want := playback.ComputeCapabilityHash(served); want != info.CapabilityHash { + t.Fatalf("capability_hash = %s, want %s for the served payload", info.CapabilityHash, want) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != info.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the just-served %q", got, info.CapabilityHash) + } +} + +// A probe that did not finish hashes differently from the same hardware probed +// successfully, so publishing it would announce a hardware change that never +// happened — and cost the API a full capability refetch plus a planning-cache +// drop. A caller that gives up must leave the published hash alone. +func TestProxyCapabilitiesRejectsIncompleteProbeWithoutPublishing(t *testing.T) { + const secret = "capability-secret" + server := newDownloadProxyServer(t, secret) + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published before the cancelled request") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil).WithContext(canceled) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusServiceUnavailable, recorder.Body.String()) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished probe, want the previous %q", got, published) + } +} + +// The background snapshot has the same duty: a probe it could not finish is not +// evidence the proxy lost hardware. +func TestProxySnapshotKeepsPreviousHashWhenProbeCannotFinish(t *testing.T) { + server := newDownloadProxyServer(t, "capability-secret") + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published by the first snapshot") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + server.refreshCapabilitySnapshot(canceled) + + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished snapshot, want the previous %q", got, published) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index de7a319d8..8f85c5eca 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/go-chi/chi/v5" @@ -53,6 +54,11 @@ type Server struct { downloadBandwidth *downloads.BandwidthManager downloadServerBPS int64 downloadUserBPS int64 + + // capabilityHash is the last computed capability snapshot's hash, published + // by /health without probing. Nil until the first snapshot or capability + // request completes. + capabilityHash atomic.Pointer[string] } type remoteArtifactMissReporter interface { @@ -199,6 +205,34 @@ func (s *Server) Handler() http.Handler { // a pool whose proxies carry a different ffmpeg build (a rolling upgrade, a // custom image) would fail at stream time rather than at selection time. func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { + info, err := s.buildCapabilitySnapshot(r.Context()) + if err != nil { + // An incomplete probe would hash differently from the same hardware + // probed successfully, so serving it would announce a hardware change + // that did not happen. + slog.WarnContext(r.Context(), "proxy capability probe incomplete", "component", "proxy", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return + } + // A served report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + slog.WarnContext(r.Context(), "encode proxy capabilities", "component", "proxy", "error", err) + } +} + +// buildCapabilitySnapshot assembles this proxy's capability report and its +// identity hash. It is the single assembly used by both the capability endpoint +// and the background snapshot, so the hash a health response advertises always +// describes the payload the endpoint would serve. +// +// An error means the probes did not finish — a caller that gave up, or an +// ffmpeg slower than a probe deadline — not that the proxy lost hardware. The +// caller must keep the previous hash rather than publish the partial report, +// exactly as a transcode node does. +func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { ffmpegPath := "" hwAccel := playback.HWAccelNone hwDevice := "" @@ -207,18 +241,87 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { hwAccel = cfg.Playback.HWAccel hwDevice = cfg.Playback.HWDevice } - info := playback.DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice) - info.Transformations = playback.ProbeTransformationRegistryV3(r.Context(), ffmpegPath).Advertised() - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(info); err != nil { - slog.WarnContext(r.Context(), "encode proxy capabilities", "component", "proxy", "error", err) + info := playback.DetectHWAccelWithFFmpegContext(ctx, hwAccel, ffmpegPath, hwDevice) + if err := ctx.Err(); err != nil { + // The hardware walk has no error return: it degrades to unverified + // backends when its context ends mid-probe. + return playback.HWAccelInfo{}, err + } + registry, err := playback.ProbeTransformationRegistryWithToneMapV3Result(ctx, ffmpegPath, nil) + if err != nil { + return playback.HWAccelInfo{}, err + } + info.Transformations = registry.Advertised() + info.CapabilityHash = playback.ComputeCapabilityHash(info) + return info, nil +} + +// capabilitySnapshotInterval is how often the proxy recomputes its capability +// snapshot. It exists to notice hardware or ffmpeg changing underneath a +// long-running proxy without waiting for a restart. The hardware walk is +// cached, but the transformation registry re-execs ffmpeg every time, which is +// the other reason a snapshot that did not finish must not be published. +const capabilitySnapshotInterval = 15 * time.Minute + +// StartCapabilitySnapshots keeps the capability hash published by /health +// current, in the background, until ctx is canceled. +func (s *Server) StartCapabilitySnapshots(ctx context.Context) { + if s == nil || ctx == nil { + return } + go func() { + s.refreshCapabilitySnapshot(ctx) + ticker := time.NewTicker(capabilitySnapshotInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.refreshCapabilitySnapshot(ctx) + } + } + }() +} + +func (s *Server) refreshCapabilitySnapshot(ctx context.Context) { + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + // Keep the previous hash: a failed probe is not evidence the hardware + // changed, and republishing a degraded one would make the API refetch + // this proxy's inventory and store a report it did not lose anything to. + slog.WarnContext(ctx, "proxy capability snapshot incomplete", "component", "proxy", "error", err) + return + } + if previous := s.storedCapabilityHash(); previous != "" && previous != info.CapabilityHash { + slog.InfoContext(ctx, "proxy capabilities changed", "component", "proxy", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + } + s.storeCapabilityHash(info.CapabilityHash) +} + +// storedCapabilityHash returns the last published capability hash, or empty +// when none has been computed yet. +func (s *Server) storedCapabilityHash() string { + if hash := s.capabilityHash.Load(); hash != nil { + return *hash + } + return "" +} + +func (s *Server) storeCapabilityHash(hash string) { + s.capabilityHash.Store(&hash) } type healthResponse struct { Status string `json:"status"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash identifies this proxy's last computed capability + // snapshot. It is read from the stored snapshot only — health must stay a + // cheap liveness answer, so it never triggers a probe — and is empty until + // the first background snapshot completes. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { @@ -228,9 +331,10 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(healthResponse{ - Status: "ok", - ActiveJobs: activeJobs, - EgressKbps: s.egress.RateKbps(), + Status: "ok", + ActiveJobs: activeJobs, + EgressKbps: s.egress.RateKbps(), + CapabilitiesHash: s.storedCapabilityHash(), }) } diff --git a/internal/transcodenode/capability_snapshot_test.go b/internal/transcodenode/capability_snapshot_test.go new file mode 100644 index 000000000..278c14285 --- /dev/null +++ b/internal/transcodenode/capability_snapshot_test.go @@ -0,0 +1,139 @@ +package transcodenode + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// newCapabilityTestServer builds a node whose configured ffmpeg is a script +// that records every invocation, so a test can assert nothing probed. +func newCapabilityTestServer(t *testing.T) (*Server, string) { + t.Helper() + dir := t.TempDir() + logPath := filepath.Join(dir, "ffmpeg-invocations.log") + ffmpegPath := filepath.Join(dir, "ffmpeg") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + logPath + "\nexit 1\n" + if err := os.WriteFile(ffmpegPath, []byte(script), 0o700); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + + watcher := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = testSecret + cfg.Playback.TranscodeDir = t.TempDir() + cfg.Playback.FFmpegPath = ffmpegPath + watcher.SetConfigForTest(cfg) + return &Server{watcher: watcher, sessions: make(map[string]*playback.TranscodeSession)}, logPath +} + +func decodeHealth(t *testing.T, server *Server) HealthResponse { + t.Helper() + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var health HealthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &health); err != nil { + t.Fatalf("decode health: %v", err) + } + return health +} + +// Health is polled every 30 seconds per node and is the liveness signal the +// pools act on. It must publish the last snapshot's hash and nothing more: if +// it probed, a slow or wedged ffmpeg would make a live node look dead. +func TestHealthPublishesStoredCapabilityHashWithoutProbing(t *testing.T) { + server, ffmpegLog := newCapabilityTestServer(t) + + if got := decodeHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q before any snapshot, want empty", got) + } + + server.storeCapabilityHash("sha256:abc123") + + if got := decodeHealth(t, server).CapabilitiesHash; got != "sha256:abc123" { + t.Fatalf("capabilities_hash = %q, want the stored snapshot hash", got) + } + if _, err := os.Stat(ffmpegLog); !os.IsNotExist(err) { + contents, _ := os.ReadFile(ffmpegLog) + t.Fatalf("health ran ffmpeg probes:\n%s", contents) + } +} + +// The capability endpoint and the background snapshot must agree, so a served +// report carries its hash and health starts advertising it immediately — +// otherwise the API would refetch a report it already has. +func TestHWCapabilitiesPublishesCapabilityHash(t *testing.T) { + server := newTestServer(t) + + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+testSecret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code == http.StatusServiceUnavailable { + t.Skip("this host's ffmpeg cannot answer a capability probe") + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash == "" { + t.Fatal("served capability report carries no capability_hash") + } + // The hash must describe this payload, not some earlier one. + served := info + served.CapabilityHash = "" + if want := playback.ComputeCapabilityHash(served); want != info.CapabilityHash { + t.Fatalf("capability_hash = %s, want %s for the served payload", info.CapabilityHash, want) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != info.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the just-served %q", got, info.CapabilityHash) + } +} + +// The first snapshot waits on encoder warmup so it measures a primed encoder, +// and a canceled node must not leave that wait running. +func TestStartCapabilitySnapshotsWaitsForReadyChannel(t *testing.T) { + server, ffmpegLog := newCapabilityTestServer(t) + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + + server.StartCapabilitySnapshots(ctx, ready) + cancel() + <-ctx.Done() + + if _, err := os.Stat(ffmpegLog); !os.IsNotExist(err) { + contents, _ := os.ReadFile(ffmpegLog) + t.Fatalf("snapshot probed before warmup completed:\n%s", contents) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q, want empty while the snapshot is gated", got) + } +} + +// A failed probe is not evidence the hardware changed, so the previously +// published hash must survive it. +func TestRefreshCapabilitySnapshotKeepsHashOnProbeFailure(t *testing.T) { + server, _ := newCapabilityTestServer(t) + server.storeCapabilityHash("sha256:previous") + + server.refreshCapabilitySnapshot(context.Background()) + + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous hash kept after a failed probe", got) + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index aacb385df..cc2159b73 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -116,6 +116,11 @@ func ValidateAudioRecipeAttestation(req TranscodeStartRequest, response Transcod type HealthResponse struct { Status string `json:"status"` ActiveJobs int32 `json:"active_jobs"` + // CapabilitiesHash identifies this node's last computed hardware capability + // snapshot. It is read from the stored snapshot only — health must stay a + // cheap liveness answer, so it never triggers a probe — and is empty until + // the first background snapshot completes. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` } // sessionIdleTTL is how long a job may go without a manifest or segment @@ -193,6 +198,24 @@ type Server struct { // recipeStore is the control-plane recipe store consulted when a forwarded // token carries no recipe (the jellycompat node hop). Nil disables that path. recipeStore recipeStore + + // capabilityHash is the last computed capability snapshot's hash, published + // by /health without probing. Nil until the first snapshot or capability + // request completes. + capabilityHash atomic.Pointer[string] +} + +// storedCapabilityHash returns the last published capability hash, or empty +// when none has been computed yet. +func (s *Server) storedCapabilityHash() string { + if hash := s.capabilityHash.Load(); hash != nil { + return *hash + } + return "" +} + +func (s *Server) storeCapabilityHash(hash string) { + s.capabilityHash.Store(&hash) } func (s *Server) resolveToneMapRecipe(ctx context.Context, opts *playback.TranscodeOpts) error { @@ -309,7 +332,7 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv } // StartOrphanSweeper runs the age-guarded orphan-transcode sweep immediately and -// then hourly until ctx is cancelled. It never blocks (a slow network-filesystem +// then hourly until ctx is canceled. It never blocks (a slow network-filesystem // delete runs in its own goroutine), so it is safe to call before the node binds // its listener. This is the node's only filesystem-level reclaimer of dirs left // behind by a session that was dropped without its output dir being removed — the @@ -331,21 +354,29 @@ func (s *Server) StartOrphanSweeper(ctx context.Context) { } // StartHardwareEncoderWarmup primes the configured hardware encoder behind -// node startup. It is best effort and never delays the health listener. -func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) { +// node startup. It is best effort and never delays the health listener. The +// returned channel closes once warmup has settled (including when there was +// nothing to warm), so work that wants a primed encoder — the first capability +// snapshot — can wait for it instead of racing it. +func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} { + done := make(chan struct{}) if s == nil || s.watcher == nil || ctx == nil { - return + close(done) + return done } cfg := s.watcher.Config() if cfg == nil { - return + close(done) + return done } playbackCfg := cfg.Playback go func() { + defer close(done) if err := playback.WarmHardwareEncoder(ctx, playbackCfg.FFmpegPath, playbackCfg.HWAccel, playbackCfg.HWDevice); err != nil { slog.DebugContext(ctx, "transcode node hardware encoder warmup failed", "component", "transcodenode", "error", err) } }() + return done } // activeSessionIDs snapshots the ids of currently registered jobs so the orphan @@ -879,13 +910,21 @@ func (s *Server) trackDownloadPrepare(ctx context.Context, info nodesessions.Ses func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(HealthResponse{ - Status: "ok", - ActiveJobs: s.activeJobs.Load(), + Status: "ok", + ActiveJobs: s.activeJobs.Load(), + CapabilitiesHash: s.storedCapabilityHash(), }) } -// handleHWCapabilities reports live smoke-tested node capabilities. -func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { +// buildCapabilitySnapshot runs the node's full capability detection: hardware +// walk, tone-map probe, and transformation registry, hashed into one identity. +// It is the single assembly used by both the capability endpoint and the +// background snapshot, so the hash a health response advertises always +// describes the payload the endpoint would serve. +// +// The probes behind it are individually cached, so repeating this is cheap once +// the first pass has run. +func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { ffmpegPath := "" configuredHWAccel := playback.HWAccelNone hwDevice := "" @@ -894,7 +933,7 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { configuredHWAccel = cfg.Playback.HWAccel hwDevice = cfg.Playback.HWDevice } - resolveCtx, cancel := context.WithTimeout(r.Context(), toneMapCapabilityResolveTimeout(configuredHWAccel, hwDevice)) + resolveCtx, cancel := context.WithTimeout(ctx, toneMapCapabilityResolveTimeout(configuredHWAccel, hwDevice)) defer cancel() // One detection walk answers both questions: Resolved honors the configured // backend's pass-through contract, and DetectedBackends explains it. @@ -902,20 +941,83 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { info.ProbeRequestTimeoutMillis = tonemap.ProbeRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), info.Resolved, hwDevice) if err != nil { - http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) - return + return playback.HWAccelInfo{}, err } info.ToneMapCapabilities = capabilities registry, err := playback.ProbeTransformationRegistryWithToneMapV3Result(resolveCtx, ffmpegPath, info.ToneMapCapabilities) + if err != nil { + return playback.HWAccelInfo{}, err + } + info.Transformations = registry.Advertised() + info.CapabilityHash = playback.ComputeCapabilityHash(info) + return info, nil +} + +// handleHWCapabilities reports live smoke-tested node capabilities. +func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { + info, err := s.buildCapabilitySnapshot(r.Context()) if err != nil { http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) return } - info.Transformations = registry.Advertised() + // A served report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(info) } +// capabilitySnapshotInterval is how often the node recomputes its capability +// snapshot. The probes behind it are cached, so this mostly re-reads sysfs and +// re-hashes; it exists to notice hardware or ffmpeg changing underneath a +// long-running node without waiting for a restart. +const capabilitySnapshotInterval = 15 * time.Minute + +// StartCapabilitySnapshots keeps the capability hash published by /health +// current, in the background, until ctx is canceled. ready gates the first +// snapshot so it observes a primed encoder rather than racing warmup; a nil +// channel means snapshot immediately. +func (s *Server) StartCapabilitySnapshots(ctx context.Context, ready <-chan struct{}) { + if s == nil || ctx == nil { + return + } + go func() { + if ready != nil { + select { + case <-ctx.Done(): + return + case <-ready: + } + } + s.refreshCapabilitySnapshot(ctx) + ticker := time.NewTicker(capabilitySnapshotInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.refreshCapabilitySnapshot(ctx) + } + } + }() +} + +func (s *Server) refreshCapabilitySnapshot(ctx context.Context) { + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + // Keep the previous hash: a failed probe is not evidence the hardware + // changed, and clearing it would look like a downgrade to the API. + slog.DebugContext(ctx, "transcode node capability snapshot failed", "component", "transcodenode", "error", err) + return + } + if previous := s.storedCapabilityHash(); previous != "" && previous != info.CapabilityHash { + slog.InfoContext(ctx, "transcode node capabilities changed", "component", "transcodenode", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + } + s.storeCapabilityHash(info.CapabilityHash) +} + func toneMapCapabilityResolveTimeout(hardwareBackend, hardwareDevice string) time.Duration { return tonemap.ProbeEndpointTimeout(hardwareBackend, hardwareDevice) } diff --git a/migrations/sql/20260826225109_node_gpu_capabilities.sql b/migrations/sql/20260826225109_node_gpu_capabilities.sql new file mode 100644 index 000000000..7e6bdbc7d --- /dev/null +++ b/migrations/sql/20260826225109_node_gpu_capabilities.sql @@ -0,0 +1,34 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capabilities jsonb, + ADD COLUMN capabilities_hash text, + ADD COLUMN capabilities_refreshed_at timestamptz; + +COMMENT ON COLUMN stream_nodes.capabilities IS + 'Last capability report fetched from this node (the /hw-capabilities ' + 'payload: resolved backend, render devices with PCI address and GPU uuid, ' + 'probed backends, transformations, tone-map executors). NULL until the ' + 'node has advertised a capability hash and the fetch succeeded. Durable ' + 'so GPU inventory survives an API restart and so a node that goes ' + 'unhealthy still reports what hardware it had.'; + +COMMENT ON COLUMN stream_nodes.capabilities_hash IS + 'Identity of the stored capabilities payload, as computed by the node. ' + 'The health sweep refetches only when the node reports a hash different ' + 'from this one, so an unchanged node costs one health request. NULL until ' + 'the first successful fetch.'; + +COMMENT ON COLUMN stream_nodes.capabilities_refreshed_at IS + 'When the stored capabilities were last fetched and persisted. This is the ' + 'age of the inventory, not of the last health check; a node checked every ' + '30s may keep capabilities from hours ago because nothing changed.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capabilities, + DROP COLUMN IF EXISTS capabilities_hash, + DROP COLUMN IF EXISTS capabilities_refreshed_at; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 7782ac3a9..a697cfdb2 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3784,6 +3784,49 @@ export interface UpdatePluginSettingsRequest { } // Stream Nodes + +/** One render device in a node's stored capability report. */ +export interface NodeRenderDevice { + path: string; + /** sysfs PCI slot (e.g. "0000:03:00.0"); absent for a non-PCI device. */ + pci_address?: string; + /** NVIDIA's permanent GPU identity; only present with nvidia-smi installed. */ + gpu_uuid?: string; + description: string; +} + +/** One hardware backend with candidate devices, plus its probe outcome. */ +export interface NodeDetectedBackend { + backend: string; + /** A real single-frame encode passed, not just an FFmpeg build-flag listing. */ + verified: boolean; + devices?: string[]; + /** The candidate that passed. Empty for NVENC, which selects via CUDA. */ + device?: string; + /** Why verification failed, attributed per device when several were tried. */ + reason?: string; +} + +/** + * A node's stored hardware capability report — the body its /hw-capabilities + * endpoint served. The payload also carries the node's transformation and + * tone-map advertisements, which no admin surface reads yet. + */ +export interface NodeCapabilities { + /** Backend that would actually be used: nvenc, qsv, vaapi, or none. */ + resolved?: string; + render_devices?: string[] | null; + render_device_details?: NodeRenderDevice[] | null; + intel_detected?: boolean; + detected_backends?: NodeDetectedBackend[]; + /** Kernel boot identity (Linux only); scopes pci_address to one boot. */ + boot_id?: string; + /** "sha256:" over the report's hardware identity and capabilities. */ + capability_hash?: string; + source?: string; + node_url?: string; +} + export interface StreamNode { id: number; name: string; @@ -3798,6 +3841,14 @@ export interface StreamNode { egress_kbps: number; last_health_check: string | null; created_at: string; + // Capability fields are owned by the background health sweep and are absent + // until one report has been stored — and on every server predating them. + capabilities?: NodeCapabilities | null; + capabilities_hash?: string; + /** When `capabilities` was fetched: the age of the inventory, not the health check. */ + capabilities_refreshed_at?: string; + /** Stable per-GPU identities; two nodes sharing one share hardware. */ + physical_gpu_keys?: string[]; } export interface CreateNodeRequest { diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index ff4de4897..457b378f6 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -26,6 +26,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import { Plus, Pencil, Trash2, RefreshCw, Info, AlertTriangle } from "lucide-react"; import { ConfirmDialog } from "@/components/ConfirmDialog"; import { formatDateTime } from "@/lib/datetime"; +import { describeNodeGPU } from "./adminNodesPresentation"; type NodeType = "proxy" | "transcode"; @@ -33,6 +34,54 @@ function formatMbps(kbps: number): string { return (Math.round(kbps / 100) / 10).toString(); } +function NodeGPUCell({ node }: { node: StreamNode }) { + const gpu = describeNodeGPU(node); + if (gpu.kind === "awaiting") { + return ( + + {gpu.label} + + ); + } + + return ( +
+
+ + {gpu.backend.label} + + {gpu.failures.length > 0 && ( + `${failure.label}: ${failure.reason}`).join("\n")} + > + + + )} + {gpu.stale && ( + + stale + + )} +
+ {gpu.deviceSummary && ( +
+ {gpu.deviceSummary} +
+ )} +
+ ); +} + interface NodeSectionProps { type: NodeType; nodes: StreamNode[]; @@ -59,7 +108,7 @@ function NodeSection({ checkingHealthId, }: NodeSectionProps) { const label = type === "proxy" ? "Proxy" : "Transcode"; - const colCount = (showJobs ? 8 : 7) + (type === "proxy" ? 1 : 0); + const colCount = (showJobs ? 9 : 8) + (type === "proxy" ? 1 : 0); return (
@@ -84,6 +133,7 @@ function NodeSection({ Group Status Health + GPU {showJobs && {type === "proxy" ? "Streams" : "Jobs"}} {type === "proxy" && Egress} Last Check @@ -139,6 +189,9 @@ function NodeSection({ + + + {showJobs && ( {node.active_jobs} diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts new file mode 100644 index 000000000..7ddac0f63 --- /dev/null +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -0,0 +1,295 @@ +import { describe, expect, it } from "vitest"; +import type { StreamNode } from "@/api/types"; +import { CAPABILITY_STALE_AFTER_MS, describeNodeGPU } from "./adminNodesPresentation"; + +const NOW = Date.parse("2026-08-26T12:00:00Z"); + +function makeNode(overrides: Partial = {}): StreamNode { + return { + id: 1, + name: "transcode-1", + type: "transcode", + url: "http://10.0.0.5:8082", + enabled: true, + healthy: true, + active_jobs: 0, + group: null, + max_jobs: null, + max_bandwidth_kbps: null, + egress_kbps: 0, + last_health_check: "2026-08-26T11:59:50Z", + created_at: "2026-08-01T00:00:00Z", + ...overrides, + }; +} + +describe("describeNodeGPU", () => { + it("reports a node with no stored capabilities as awaiting its first report", () => { + expect(describeNodeGPU(makeNode(), NOW)).toEqual({ + kind: "awaiting", + label: "Awaiting first report", + title: "No hardware capability report has been stored for this node yet.", + }); + }); + + it("treats an explicit null payload the same as an absent one", () => { + expect(describeNodeGPU(makeNode({ capabilities: null }), NOW).kind).toBe("awaiting"); + }); + + it("marks the resolved backend verified when its probe passed", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "qsv", + detected_backends: [ + { + backend: "qsv", + verified: true, + devices: ["/dev/dri/renderD128"], + device: "/dev/dri/renderD128", + }, + ], + }, + capabilities_refreshed_at: "2026-08-26T11:59:00Z", + }), + NOW, + ); + + expect(presentation).toMatchObject({ + kind: "reported", + backend: { + label: "QSV", + state: "verified", + badgeClass: "bg-success/10 text-success border-success/15", + title: "QSV verified by FFmpeg probe on /dev/dri/renderD128.", + }, + failures: [], + stale: false, + }); + }); + + it("omits the device from an NVENC title, which has no render node", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "nvenc", + detected_backends: [{ backend: "nvenc", verified: true }], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.backend.title).toBe( + "NVENC verified by FFmpeg probe.", + ); + }); + + it("warns with the failure reason when the resolved backend failed its probe", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "qsv", + detected_backends: [ + { + backend: "qsv", + verified: false, + reason: "h264_qsv smoke encode failed: device busy", + }, + ], + }, + }), + NOW, + ); + + expect(presentation).toMatchObject({ + kind: "reported", + backend: { + label: "QSV", + state: "failed", + badgeClass: "bg-warning/10 text-warning border-warning/15", + title: "QSV probe failed: h264_qsv smoke encode failed: device busy", + }, + failures: [], + }); + }); + + it("names a failed backend with no reason rather than showing an empty title", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "vaapi", + detected_backends: [{ backend: "vaapi", verified: false }], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.backend.title).toBe( + "VAAPI probe failed: no reason reported", + ); + }); + + it("lists failed backends other than the resolved one", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "vaapi", + detected_backends: [ + { backend: "qsv", verified: false, reason: "h264_qsv encoder unavailable" }, + { backend: "vaapi", verified: true, device: "/dev/dri/renderD128" }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.failures).toEqual([ + { label: "QSV", reason: "h264_qsv encoder unavailable" }, + ]); + }); + + it("falls back to software with no hardware backend resolved", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { resolved: "none", render_devices: [], render_device_details: [] }, + }), + NOW, + ); + + expect(presentation).toMatchObject({ + kind: "reported", + backend: { label: "SW", state: "none" }, + deviceSummary: null, + deviceTitle: null, + }); + }); + + it("treats a configured backend with no probe entry as unverified, not failed", () => { + const presentation = describeNodeGPU(makeNode({ capabilities: { resolved: "qsv" } }), NOW); + + expect(presentation).toMatchObject({ + kind: "reported", + backend: { + label: "QSV", + state: "unverified", + title: "QSV is in use but this node reported no verification probe for it.", + }, + }); + }); + + it("collapses identical device descriptions and keeps full paths in the title", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "qsv", + render_device_details: [ + { path: "/dev/dri/renderD128", description: "Intel GPU", pci_address: "0000:00:02.0" }, + { path: "/dev/dri/renderD129", description: "Intel GPU" }, + { path: "/dev/dri/renderD130", description: "NVIDIA GPU (0x2204)" }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.deviceSummary).toBe( + "2× Intel GPU, NVIDIA GPU (0x2204)", + ); + expect(presentation.kind === "reported" && presentation.deviceTitle).toBe( + [ + "/dev/dri/renderD128 — Intel GPU (0000:00:02.0)", + "/dev/dri/renderD129 — Intel GPU", + "/dev/dri/renderD130 — NVIDIA GPU (0x2204)", + ].join("\n"), + ); + }); + + it("counts render device paths when a report carries no details", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "vaapi", + render_devices: ["/dev/dri/renderD128", "/dev/dri/renderD129"], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.deviceSummary).toBe("2 render devices"); + expect(presentation.kind === "reported" && presentation.deviceTitle).toBe( + "/dev/dri/renderD128\n/dev/dri/renderD129", + ); + }); + + it("marks a node stale once the health checks that confirm its report stop", () => { + const node = makeNode({ + capabilities: { resolved: "qsv" }, + capabilities_refreshed_at: new Date(NOW - 6 * 60 * 60 * 1000).toISOString(), + last_health_check: new Date(NOW - CAPABILITY_STALE_AFTER_MS - 1000).toISOString(), + }); + + expect(describeNodeGPU(node, NOW)).toMatchObject({ stale: true }); + // The same node read earlier was still being checked: the clock decides. + expect(describeNodeGPU(node, NOW - CAPABILITY_STALE_AFTER_MS)).toMatchObject({ stale: false }); + }); + + // The sweep refetches only when a node advertises a changed hash, so an + // untouched GPU keeps its original report forever by design. Calling that + // stale would light the warning on every steady-state node. + it("does not call an old report stale while health checks keep confirming it", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { resolved: "qsv" }, + capabilities_refreshed_at: new Date(NOW - 6 * 60 * 60 * 1000).toISOString(), + last_health_check: new Date(NOW - 20 * 1000).toISOString(), + }), + NOW, + ); + + expect(presentation).toMatchObject({ stale: false }); + }); + + it("does not call an unhealthy node's report stale", () => { + const presentation = describeNodeGPU( + makeNode({ + healthy: false, + capabilities: { resolved: "qsv" }, + capabilities_refreshed_at: new Date(NOW - 24 * 60 * 60 * 1000).toISOString(), + last_health_check: new Date(NOW - 24 * 60 * 60 * 1000).toISOString(), + }), + NOW, + ); + + expect(presentation).toMatchObject({ stale: false }); + }); + + it("is not stale when the server sent no refresh timestamp", () => { + expect(describeNodeGPU(makeNode({ capabilities: { resolved: "qsv" } }), NOW)).toMatchObject({ + stale: false, + }); + }); + + it("is not stale when the server sent no health check timestamp", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { resolved: "qsv" }, + capabilities_refreshed_at: new Date(NOW - 6 * 60 * 60 * 1000).toISOString(), + last_health_check: null, + }), + NOW, + ); + + expect(presentation).toMatchObject({ stale: false }); + }); + + it("tolerates physical_gpu_keys without letting it change the presentation", () => { + const capabilities = { + resolved: "nvenc", + detected_backends: [{ backend: "nvenc", verified: true }], + }; + + expect( + describeNodeGPU(makeNode({ capabilities, physical_gpu_keys: ["GPU-abc"] }), NOW), + ).toEqual(describeNodeGPU(makeNode({ capabilities }), NOW)); + }); +}); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts new file mode 100644 index 000000000..ddd715feb --- /dev/null +++ b/web/src/pages/adminNodesPresentation.ts @@ -0,0 +1,205 @@ +import type { NodeCapabilities, NodeRenderDevice, StreamNode } from "@/api/types"; + +/** + * How long a stored capability report may go unconfirmed before it is called + * out. The health sweep only refetches when a node advertises a changed hash, + * so an unchanged node legitimately keeps a report from hours ago — its age + * says nothing. What does tick is the sweep itself: every check that finds the + * advertised hash unchanged re-confirms the stored report. So the report is + * only doubtful once the checks stop, which at a 30s cadence this allows twenty + * of before saying so. + */ +export const CAPABILITY_STALE_AFTER_MS = 10 * 60 * 1000; + +/** Verification state of the backend a node resolved to. */ +export type NodeGPUBackendState = "verified" | "failed" | "unverified" | "none"; + +export interface NodeGPUBackendBadge { + /** Uppercase backend name, or "SW" when no hardware backend is in use. */ + label: string; + state: NodeGPUBackendState; + /** Tinted badge classes, shared with the activity-method tags. */ + badgeClass: string; + /** Hover text: the probe outcome, including a failure reason. */ + title: string; +} + +/** A backend that had candidate hardware but failed its FFmpeg probe. */ +export interface NodeGPUFailure { + label: string; + reason: string; +} + +export type NodeGPUPresentation = + | { + kind: "awaiting"; + label: string; + title: string; + } + | { + kind: "reported"; + backend: NodeGPUBackendBadge; + /** Failed backends other than the resolved one, whose reason is in its title. */ + failures: NodeGPUFailure[]; + /** Compact device list, e.g. "2× Intel GPU"; null when none were reported. */ + deviceSummary: string | null; + /** Full device paths, one per line, for the summary's tooltip. */ + deviceTitle: string | null; + /** No health check has re-confirmed this report recently. */ + stale: boolean; + }; + +const BACKEND_BADGE_CLASS: Record = { + verified: "bg-success/10 text-success border-success/15", + failed: "bg-warning/10 text-warning border-warning/15", + unverified: "bg-surface text-muted-foreground border-border", + none: "bg-surface text-muted-foreground border-border", +}; + +/** + * Describe a node's GPU column. `now` is injected so staleness is testable and + * so a rendered table can be pinned to one clock reading. + */ +export function describeNodeGPU(node: StreamNode, now: number = Date.now()): NodeGPUPresentation { + const capabilities = node.capabilities; + if (!capabilities) { + return { + kind: "awaiting", + label: "Awaiting first report", + title: "No hardware capability report has been stored for this node yet.", + }; + } + + const devices = summarizeRenderDevices(capabilities); + const resolved = capabilities.resolved?.trim().toLowerCase() ?? ""; + return { + kind: "reported", + backend: describeBackend(resolved, capabilities.detected_backends ?? []), + failures: otherFailures(resolved, capabilities.detected_backends ?? []), + deviceSummary: devices.summary, + deviceTitle: devices.title, + stale: isCapabilityReportStale(node, now), + }; +} + +function describeBackend(resolved: string, detected: readonly NodeDetected[]): NodeGPUBackendBadge { + if (resolved === "" || resolved === "none") { + return badge("SW", "none", "No hardware backend verified — encoding in software."); + } + + const label = resolved.toUpperCase(); + const entry = detected.find((candidate) => candidate.backend?.trim().toLowerCase() === resolved); + if (!entry) { + // A configured backend wins resolution even with no candidate hardware to + // probe, so absence of an entry is unknown, not failure. + return badge( + label, + "unverified", + `${label} is in use but this node reported no verification probe for it.`, + ); + } + if (!entry.verified) { + return badge(label, "failed", `${label} probe failed: ${failureReason(entry)}`); + } + const device = entry.device?.trim(); + return badge( + label, + "verified", + device + ? `${label} verified by FFmpeg probe on ${device}.` + : `${label} verified by FFmpeg probe.`, + ); +} + +function badge(label: string, state: NodeGPUBackendState, title: string): NodeGPUBackendBadge { + return { label, state, badgeClass: BACKEND_BADGE_CLASS[state], title }; +} + +type NodeDetected = NonNullable[number]; + +function otherFailures(resolved: string, detected: readonly NodeDetected[]): NodeGPUFailure[] { + return detected + .filter((entry) => { + const backend = entry.backend?.trim().toLowerCase() ?? ""; + return !entry.verified && backend !== "" && backend !== resolved; + }) + .map((entry) => ({ + label: (entry.backend?.trim() ?? "").toUpperCase(), + reason: failureReason(entry), + })); +} + +function failureReason(entry: NodeDetected): string { + return entry.reason?.trim() || "no reason reported"; +} + +function isCapabilityReportStale(node: StreamNode, now: number): boolean { + // An unhealthy node cannot refresh its report; calling that stale would blame + // the inventory for the outage the Health column already shows. + if (!node.healthy) { + return false; + } + if (Number.isNaN(Date.parse(node.capabilities_refreshed_at ?? ""))) { + return false; + } + // Measured against the health check, not against the report's own age: the + // check is what re-confirms the report, and it is the only one of the two + // that moves on a node whose hardware never changes. + const lastCheck = Date.parse(node.last_health_check ?? ""); + if (Number.isNaN(lastCheck)) { + return false; + } + return now - lastCheck > CAPABILITY_STALE_AFTER_MS; +} + +function summarizeRenderDevices(capabilities: NodeCapabilities): { + summary: string | null; + title: string | null; +} { + const details = capabilities.render_device_details ?? []; + if (details.length > 0) { + return { + summary: countedDescriptions(details), + title: details.map(describeDeviceLine).join("\n"), + }; + } + + // A report with paths but no details is still worth a count. + const paths = (capabilities.render_devices ?? []).filter((path) => path.trim() !== ""); + if (paths.length === 0) { + return { summary: null, title: null }; + } + return { + summary: paths.length === 1 ? "1 render device" : `${paths.length} render devices`, + title: paths.join("\n"), + }; +} + +/** "2× Intel GPU, NVIDIA GPU (0x2204)" — identical descriptions collapse. */ +function countedDescriptions(details: readonly NodeRenderDevice[]): string { + const counted: { label: string; count: number }[] = []; + for (const device of details) { + const label = deviceLabel(device); + const existing = counted.find((entry) => entry.label === label); + if (existing) { + existing.count += 1; + } else { + counted.push({ label, count: 1 }); + } + } + return counted + .map((entry) => (entry.count > 1 ? `${entry.count}× ${entry.label}` : entry.label)) + .join(", "); +} + +function deviceLabel(device: NodeRenderDevice): string { + return device.description?.trim() || device.path?.trim() || "GPU"; +} + +function describeDeviceLine(device: NodeRenderDevice): string { + const path = device.path?.trim() || "(unknown path)"; + const parts = [path, device.description?.trim()].filter((part): part is string => !!part); + const line = parts.join(" — "); + const address = device.pci_address?.trim(); + return address ? `${line} (${address})` : line; +} From 6c565edead8cd36acc24687fb1fb65b8ce379df7 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:34:27 -0400 Subject: [PATCH 003/163] feat(web): rebuild admin dashboard as customizable widget grid Registry-driven 12-column widget grid with customize mode (drag reorder, drag-to-resize with keyboard alternatives, add/remove sheet), layout persisted to localStorage. Old dashboard sections extracted into widgets; Trakt card replaced by a compact sync strip. Co-Authored-By: Claude Fable 5 --- web/src/app.css | 37 +- .../admin/dashboard/DashboardGrid.tsx | 385 ++++++++ .../components/admin/dashboard/feedback.tsx | 41 + web/src/components/admin/dashboard/format.ts | 18 + .../components/admin/dashboard/registry.tsx | 133 +++ web/src/components/admin/dashboard/types.ts | 28 + .../dashboard/useDashboardLayout.test.ts | 221 +++++ .../admin/dashboard/useDashboardLayout.ts | 185 ++++ .../dashboard/widgets/LibrariesWidget.tsx | 157 ++++ .../dashboard/widgets/NowPlayingWidget.tsx | 184 ++++ .../widgets/RecentActivityWidget.tsx | 87 ++ .../dashboard/widgets/SessionProfilePill.tsx | 7 + .../dashboard/widgets/TraktSyncWidget.tsx | 92 ++ .../admin/dashboard/widgets/UsersWidget.tsx | 86 ++ .../admin/dashboard/widgets/statTiles.tsx | 125 +++ web/src/pages/AdminDashboard.tsx | 856 +----------------- 16 files changed, 1832 insertions(+), 810 deletions(-) create mode 100644 web/src/components/admin/dashboard/DashboardGrid.tsx create mode 100644 web/src/components/admin/dashboard/feedback.tsx create mode 100644 web/src/components/admin/dashboard/format.ts create mode 100644 web/src/components/admin/dashboard/registry.tsx create mode 100644 web/src/components/admin/dashboard/types.ts create mode 100644 web/src/components/admin/dashboard/useDashboardLayout.test.ts create mode 100644 web/src/components/admin/dashboard/useDashboardLayout.ts create mode 100644 web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/SessionProfilePill.tsx create mode 100644 web/src/components/admin/dashboard/widgets/TraktSyncWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/UsersWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/statTiles.tsx diff --git a/web/src/app.css b/web/src/app.css index df8acbf02..ab8011fcb 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -1905,12 +1905,45 @@ padding-inline: 0; } - .admin-dashboard-stats { + /* Admin dashboard widget grid: single column on small screens, two columns + on tablets (where .admin-widget-wide keeps the full row), and a 12-column + grid from lg up where each widget spans var(--widget-span) columns. */ + .admin-widget-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(min(100%, 10.5rem), 1fr)); + grid-template-columns: 1fr; gap: 0.875rem; } + .admin-widget { + grid-column: 1 / -1; + position: relative; + min-width: 0; + } + + @media (min-width: 640px) and (max-width: 1023px) { + .admin-widget-grid { + grid-template-columns: repeat(2, 1fr); + } + + .admin-widget { + grid-column: span 1; + } + + .admin-widget-wide { + grid-column: 1 / -1; + } + } + + @media (min-width: 1024px) { + .admin-widget-grid { + grid-template-columns: repeat(12, 1fr); + } + + .admin-widget { + grid-column: span var(--widget-span) / span var(--widget-span); + } + } + @media (max-width: 639px) { .admin-shell .page-header { align-items: stretch; diff --git a/web/src/components/admin/dashboard/DashboardGrid.tsx b/web/src/components/admin/dashboard/DashboardGrid.tsx new file mode 100644 index 000000000..518dc1726 --- /dev/null +++ b/web/src/components/admin/dashboard/DashboardGrid.tsx @@ -0,0 +1,385 @@ +import type { + CSSProperties, + DragEvent, + KeyboardEvent as ReactKeyboardEvent, + PointerEvent as ReactPointerEvent, +} from "react"; +import { useCallback, useRef, useState } from "react"; +import { GripVertical, Plus, X } from "lucide-react"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; +import { cn } from "@/lib/utils"; +import { getDashboardWidget } from "./registry"; +import type { WidgetId } from "./types"; +import type { DashboardLayout } from "./useDashboardLayout"; + +/** Must match the `gap` of `.admin-widget-grid` in app.css (0.875rem). */ +const GRID_GAP_PX = 14; +const GRID_COLUMNS = 12; + +interface DropIndicator { + id: WidgetId; + edge: "before" | "after"; +} + +interface ResizePreview { + id: WidgetId; + span: number; +} + +export function DashboardGrid({ + layout, + isAddPanelOpen, + onAddPanelOpenChange, +}: { + layout: DashboardLayout; + isAddPanelOpen: boolean; + onAddPanelOpenChange: (open: boolean) => void; +}) { + const { + entries, + hiddenWidgets, + isCustomizing, + moveWidget, + resizeWidget, + removeWidget, + addWidget, + } = layout; + + const gridRef = useRef(null); + const [draggedId, setDraggedId] = useState(null); + const [dropIndicator, setDropIndicator] = useState(null); + const [resizePreview, setResizePreview] = useState(null); + const [liveMessage, setLiveMessage] = useState(""); + const resizeSessionRef = useRef<{ + id: WidgetId; + startX: number; + startSpan: number; + unit: number; + minSpan: number; + maxSpan: number; + latestSpan: number; + } | null>(null); + + const findWidgetIdFromEvent = useCallback((event: DragEvent): WidgetId | null => { + const target = event.target as HTMLElement | null; + const host = target?.closest("[data-widget-id]"); + return (host?.dataset.widgetId as WidgetId | undefined) ?? null; + }, []); + + const handleDragStart = useCallback( + (event: DragEvent) => { + if (!isCustomizing) return; + const id = findWidgetIdFromEvent(event); + if (!id) return; + event.dataTransfer.effectAllowed = "move"; + event.dataTransfer.setData("text/plain", id); + setDraggedId(id); + }, + [findWidgetIdFromEvent, isCustomizing], + ); + + const handleDragEnd = useCallback(() => { + setDraggedId(null); + setDropIndicator(null); + }, []); + + const handleDragOver = useCallback( + (event: DragEvent) => { + if (!draggedId) return; + event.preventDefault(); + event.dataTransfer.dropEffect = "move"; + const overId = findWidgetIdFromEvent(event); + if (!overId || overId === draggedId) { + setDropIndicator(null); + return; + } + const host = (event.target as HTMLElement).closest("[data-widget-id]"); + if (!host) { + setDropIndicator(null); + return; + } + const rect = host.getBoundingClientRect(); + const edge = event.clientX < rect.left + rect.width / 2 ? "before" : "after"; + setDropIndicator((prev) => + prev?.id === overId && prev.edge === edge ? prev : { id: overId, edge }, + ); + }, + [draggedId, findWidgetIdFromEvent], + ); + + const handleDrop = useCallback( + (event: DragEvent) => { + if (!draggedId) return; + event.preventDefault(); + const overId = findWidgetIdFromEvent(event); + if (overId && overId !== draggedId) { + const host = (event.target as HTMLElement).closest("[data-widget-id]"); + const rect = host?.getBoundingClientRect(); + const before = rect ? event.clientX < rect.left + rect.width / 2 : true; + if (before) { + moveWidget(draggedId, overId); + } else { + const overIndex = entries.findIndex((entry) => entry.id === overId); + const nextEntry = overIndex === -1 ? undefined : entries[overIndex + 1]; + moveWidget(draggedId, nextEntry ? nextEntry.id : null); + } + } + setDraggedId(null); + setDropIndicator(null); + }, + [draggedId, entries, findWidgetIdFromEvent, moveWidget], + ); + + const handleResizePointerDown = useCallback( + (event: ReactPointerEvent, id: WidgetId, currentSpan: number) => { + if (!isCustomizing) return; + // Only start on a primary-button press: a right-click opens the context + // menu and never delivers the matching pointerup, which would leave the + // resize session stuck. + if (!event.isPrimary || event.button !== 0) return; + const widget = getDashboardWidget(id); + event.preventDefault(); + event.stopPropagation(); + event.currentTarget.setPointerCapture(event.pointerId); + const gridWidth = gridRef.current?.getBoundingClientRect().width ?? 0; + const unit = gridWidth > 0 ? (gridWidth + GRID_GAP_PX) / GRID_COLUMNS : 1; + resizeSessionRef.current = { + id, + startX: event.clientX, + startSpan: currentSpan, + unit, + minSpan: widget.minSpan, + maxSpan: widget.maxSpan, + latestSpan: currentSpan, + }; + setResizePreview({ id, span: currentSpan }); + }, + [isCustomizing], + ); + + const handleResizePointerMove = useCallback((event: ReactPointerEvent) => { + const session = resizeSessionRef.current; + if (!session) return; + const raw = session.startSpan + (event.clientX - session.startX) / session.unit; + const next = Math.min(session.maxSpan, Math.max(session.minSpan, Math.round(raw))); + if (next !== session.latestSpan) { + session.latestSpan = next; + setResizePreview({ id: session.id, span: next }); + } + }, []); + + const handleResizePointerEnd = useCallback( + (event: ReactPointerEvent) => { + const session = resizeSessionRef.current; + if (!session) return; + resizeSessionRef.current = null; + setResizePreview(null); + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + resizeWidget(session.id, session.latestSpan); + }, + [resizeWidget], + ); + + const handleMoveKeyDown = useCallback( + (event: ReactKeyboardEvent, id: WidgetId) => { + const backward = event.key === "ArrowLeft" || event.key === "ArrowUp"; + const forward = event.key === "ArrowRight" || event.key === "ArrowDown"; + if (!backward && !forward) return; + event.preventDefault(); + const index = entries.findIndex((entry) => entry.id === id); + if (index === -1) return; + const nextIndex = backward ? index - 1 : index + 1; + const neighbor = entries[nextIndex]; + if (!neighbor) return; + if (backward) { + moveWidget(id, neighbor.id); + } else { + const after = entries[index + 2]; + moveWidget(id, after ? after.id : null); + } + setLiveMessage( + `${getDashboardWidget(id).title} moved to position ${nextIndex + 1} of ${entries.length}`, + ); + }, + [entries, moveWidget], + ); + + const handleResizeKeyDown = useCallback( + (event: ReactKeyboardEvent, id: WidgetId, currentSpan: number) => { + const shrink = event.key === "ArrowLeft" || event.key === "ArrowDown"; + const grow = event.key === "ArrowRight" || event.key === "ArrowUp"; + if (!shrink && !grow) return; + event.preventDefault(); + const widget = getDashboardWidget(id); + const next = Math.min( + widget.maxSpan, + Math.max(widget.minSpan, currentSpan + (grow ? 1 : -1)), + ); + if (next === currentSpan) return; + resizeWidget(id, next); + setLiveMessage(`${widget.title} resized to ${next} of ${GRID_COLUMNS} columns`); + }, + [resizeWidget], + ); + + const isResizing = resizePreview !== null; + + return ( + <> + + {liveMessage} + +
+ {entries.map((entry) => { + const widget = getDashboardWidget(entry.id); + const span = resizePreview?.id === entry.id ? resizePreview.span : entry.span; + const canResize = widget.minSpan !== widget.maxSpan; + const isWidgetResizing = resizePreview?.id === entry.id; + const WidgetComponent = widget.Component; + + return ( +
= 6 && "admin-widget-wide", + isCustomizing && "rounded-2xl", + draggedId === entry.id && "opacity-40", + )} + style={{ "--widget-span": span } as CSSProperties} + draggable={isCustomizing && !isResizing} + > + + + {isCustomizing && ( + <> + + + + event.preventDefault()} + > + + Add widget + + Widgets you've removed or haven't placed yet. + + +
+ {hiddenWidgets.length === 0 ? ( +

+ Everything is on the dashboard already. +

+ ) : ( + hiddenWidgets.map((widget) => ( + + )) + )} +
+ + + + ); +} diff --git a/web/src/components/admin/dashboard/feedback.tsx b/web/src/components/admin/dashboard/feedback.tsx new file mode 100644 index 000000000..40311e217 --- /dev/null +++ b/web/src/components/admin/dashboard/feedback.tsx @@ -0,0 +1,41 @@ +import { Skeleton } from "@/components/ui/skeleton"; + +export function SectionError({ message }: { message: string }) { + return
{message}
; +} + +export function LibrarySkeletonRows() { + return ( + <> + {Array.from({ length: 3 }).map((_, i) => ( + + ))} + + ); +} + +export function UserSkeletonRows() { + return ( +
+ {Array.from({ length: 4 }).map((_, i) => ( + + ))} +
+ ); +} + +export function ActivitySkeletonRows() { + return ( +
+ {Array.from({ length: 4 }).map((_, i) => ( +
+ +
+ + +
+
+ ))} +
+ ); +} diff --git a/web/src/components/admin/dashboard/format.ts b/web/src/components/admin/dashboard/format.ts new file mode 100644 index 000000000..45d1358a9 --- /dev/null +++ b/web/src/components/admin/dashboard/format.ts @@ -0,0 +1,18 @@ +import type { ScanRun } from "@/api/types"; +import { formatActiveScanMode, formatActiveScanProgress } from "@/lib/scanRuns"; + +export function formatFileCount(count: number | null | undefined) { + if (count == null) { + return "—"; + } + return count === 1 ? "1 file" : `${count.toLocaleString()} files`; +} + +export function formatDashboardLibraryScanProgress(scan: ScanRun, activeScanCount: number) { + const status = scan.status === "running" ? "Scanning" : "Queued"; + const progress = formatActiveScanProgress(scan); + const detail = + progress || (scan.status === "running" ? formatActiveScanMode(scan) : "Waiting for capacity"); + const extraScans = activeScanCount > 1 ? ` + ${activeScanCount - 1} more` : ""; + return `${status}: ${detail}${extraScans}`; +} diff --git a/web/src/components/admin/dashboard/registry.tsx b/web/src/components/admin/dashboard/registry.tsx new file mode 100644 index 000000000..5d9f918c6 --- /dev/null +++ b/web/src/components/admin/dashboard/registry.tsx @@ -0,0 +1,133 @@ +import type { DashboardLayoutEntry, DashboardWidgetDefinition, WidgetId } from "./types"; +import { + ActiveStreamsStatWidget, + MoviesStatWidget, + ShowsStatWidget, + StorageStatWidget, + UsersStatWidget, +} from "./widgets/statTiles"; +import { TraktSyncWidget } from "./widgets/TraktSyncWidget"; +import { NowPlayingWidget } from "./widgets/NowPlayingWidget"; +import { LibrariesWidget } from "./widgets/LibrariesWidget"; +import { UsersWidget } from "./widgets/UsersWidget"; +import { RecentActivityWidget } from "./widgets/RecentActivityWidget"; + +export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ + { + id: "stat-active-streams", + title: "Active streams", + description: "Live count of playback sessions", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + Component: ActiveStreamsStatWidget, + }, + { + id: "stat-movies", + title: "Movies", + description: "Total movies and movie files", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + Component: MoviesStatWidget, + }, + { + id: "stat-shows", + title: "Shows", + description: "Total series and episode files", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + Component: ShowsStatWidget, + }, + { + id: "stat-users", + title: "User count", + description: "Registered accounts on the server", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + Component: UsersStatWidget, + }, + { + id: "stat-storage", + title: "Storage", + description: "Used space across all libraries", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + Component: StorageStatWidget, + }, + { + id: "trakt-sync", + title: "Trakt sync", + description: "Watch provider connection and 24h sync status", + minSpan: 6, + maxSpan: 12, + defaultSpan: 9, + Component: TraktSyncWidget, + }, + { + id: "now-playing", + title: "Now playing", + description: "Active streams with client, method, and progress", + minSpan: 6, + maxSpan: 12, + defaultSpan: 12, + Component: NowPlayingWidget, + }, + { + id: "libraries", + title: "Libraries", + description: "Library list with scan controls and progress", + minSpan: 6, + maxSpan: 12, + defaultSpan: 7, + Component: LibrariesWidget, + }, + { + id: "users", + title: "Users", + description: "Recent user accounts with role and status", + minSpan: 4, + maxSpan: 8, + defaultSpan: 5, + Component: UsersWidget, + }, + { + id: "recent-activity", + title: "Recent activity", + description: "Feed of recently started playback sessions", + minSpan: 6, + maxSpan: 12, + defaultSpan: 12, + Component: RecentActivityWidget, + }, +]; + +const WIDGETS_BY_ID = new Map(DASHBOARD_WIDGETS.map((widget) => [widget.id, widget])); + +export function getDashboardWidget(id: WidgetId): DashboardWidgetDefinition { + const widget = WIDGETS_BY_ID.get(id); + if (!widget) { + throw new Error(`Unknown dashboard widget: ${id}`); + } + return widget; +} + +export function findDashboardWidget(id: string): DashboardWidgetDefinition | undefined { + return WIDGETS_BY_ID.get(id as WidgetId); +} + +export const DEFAULT_LAYOUT: DashboardLayoutEntry[] = [ + { id: "stat-active-streams", span: 3 }, + { id: "stat-movies", span: 3 }, + { id: "stat-shows", span: 3 }, + { id: "stat-users", span: 3 }, + { id: "stat-storage", span: 3 }, + { id: "trakt-sync", span: 9 }, + { id: "now-playing", span: 12 }, + { id: "libraries", span: 7 }, + { id: "users", span: 5 }, + { id: "recent-activity", span: 12 }, +]; diff --git a/web/src/components/admin/dashboard/types.ts b/web/src/components/admin/dashboard/types.ts new file mode 100644 index 000000000..a7c5d11f4 --- /dev/null +++ b/web/src/components/admin/dashboard/types.ts @@ -0,0 +1,28 @@ +import type React from "react"; + +export type WidgetId = + | "stat-active-streams" + | "stat-movies" + | "stat-shows" + | "stat-users" + | "stat-storage" + | "trakt-sync" + | "now-playing" + | "libraries" + | "users" + | "recent-activity"; + +export interface DashboardWidgetDefinition { + id: WidgetId; + title: string; + description: string; + minSpan: number; + maxSpan: number; + defaultSpan: number; + Component: React.ComponentType; +} + +export interface DashboardLayoutEntry { + id: WidgetId; + span: number; +} diff --git a/web/src/components/admin/dashboard/useDashboardLayout.test.ts b/web/src/components/admin/dashboard/useDashboardLayout.test.ts new file mode 100644 index 000000000..3de522eb1 --- /dev/null +++ b/web/src/components/admin/dashboard/useDashboardLayout.test.ts @@ -0,0 +1,221 @@ +// @vitest-environment jsdom + +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { DASHBOARD_WIDGETS, DEFAULT_LAYOUT } from "./registry"; +import { DASHBOARD_LAYOUT_STORAGE_KEY, useDashboardLayout } from "./useDashboardLayout"; +import type { DashboardLayoutEntry } from "./types"; + +function readStored(): { version: number; entries: DashboardLayoutEntry[] } { + const raw = window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY); + if (raw === null) { + throw new Error("expected a persisted layout"); + } + return JSON.parse(raw) as { version: number; entries: DashboardLayoutEntry[] }; +} + +function writeStored(entries: unknown) { + window.localStorage.setItem( + DASHBOARD_LAYOUT_STORAGE_KEY, + JSON.stringify({ version: 1, entries }), + ); +} + +describe("useDashboardLayout", () => { + beforeEach(() => { + window.localStorage.clear(); + }); + + it("uses the default layout when storage is empty", () => { + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + expect(result.current.hiddenWidgets).toEqual([]); + expect(result.current.isCustomizing).toBe(false); + }); + + it("falls back to the default layout on corrupt JSON", () => { + window.localStorage.setItem(DASHBOARD_LAYOUT_STORAGE_KEY, "{not json"); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + }); + + it("falls back to the default layout on an unexpected shape", () => { + window.localStorage.setItem( + DASHBOARD_LAYOUT_STORAGE_KEY, + JSON.stringify({ version: 2, entries: [{ id: "libraries", span: 7 }] }), + ); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + }); + + it("drops unknown widget ids on load", () => { + writeStored([ + { id: "libraries", span: 7 }, + { id: "not-a-widget", span: 6 }, + { id: "users", span: 5 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "libraries", span: 7 }, + { id: "users", span: 5 }, + ]); + }); + + it("clamps spans to the widget's [minSpan, maxSpan] on load", () => { + writeStored([ + { id: "stat-movies", span: 1 }, // min 2 + { id: "now-playing", span: 40 }, // max 12 + { id: "users", span: "wide" }, // non-numeric -> defaultSpan + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "stat-movies", span: 2 }, + { id: "now-playing", span: 12 }, + { id: "users", span: 5 }, + ]); + }); + + it("exposes hidden widgets in registry order", () => { + writeStored([ + { id: "users", span: 5 }, + { id: "stat-storage", span: 3 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.hiddenWidgets.map((w) => w.id)).toEqual( + DASHBOARD_WIDGETS.filter((w) => w.id !== "users" && w.id !== "stat-storage").map((w) => w.id), + ); + }); + + it("addWidget appends with the default span and persists", () => { + writeStored([{ id: "libraries", span: 7 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.addWidget("now-playing"); + }); + + const expected = [ + { id: "libraries", span: 7 }, + { id: "now-playing", span: 12 }, + ]; + expect(result.current.entries).toEqual(expected); + expect(readStored()).toEqual({ version: 1, entries: expected }); + }); + + it("removeWidget hides the widget and persists", () => { + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.removeWidget("trakt-sync"); + }); + + expect(result.current.entries.some((entry) => entry.id === "trakt-sync")).toBe(false); + expect(result.current.hiddenWidgets.map((w) => w.id)).toEqual(["trakt-sync"]); + expect(readStored().entries.some((entry) => entry.id === "trakt-sync")).toBe(false); + }); + + it("moveWidget inserts before the target and persists", () => { + writeStored([ + { id: "libraries", span: 7 }, + { id: "users", span: 5 }, + { id: "recent-activity", span: 12 }, + ]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.moveWidget("recent-activity", "users"); + }); + + expect(result.current.entries.map((entry) => entry.id)).toEqual([ + "libraries", + "recent-activity", + "users", + ]); + expect(readStored().entries.map((entry) => entry.id)).toEqual([ + "libraries", + "recent-activity", + "users", + ]); + }); + + it("moveWidget with a null beforeId moves to the end", () => { + writeStored([ + { id: "libraries", span: 7 }, + { id: "users", span: 5 }, + ]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.moveWidget("libraries", null); + }); + + expect(result.current.entries.map((entry) => entry.id)).toEqual(["users", "libraries"]); + expect(readStored().entries.map((entry) => entry.id)).toEqual(["users", "libraries"]); + }); + + it("resizeWidget clamps the span and persists", () => { + writeStored([{ id: "users", span: 5 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", 6); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 6 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 6 }]); + + act(() => { + result.current.resizeWidget("users", 99); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 8 }]); + + act(() => { + result.current.resizeWidget("users", 1); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 4 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 4 }]); + }); + + it("resetLayout restores the defaults and clears storage", () => { + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.removeWidget("users"); + result.current.resizeWidget("libraries", 12); + }); + expect(result.current.entries).not.toEqual(DEFAULT_LAYOUT); + + act(() => { + result.current.resetLayout(); + }); + + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + expect(window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY)).toBeNull(); + }); + + it("round-trips a customized layout through localStorage", () => { + const first = renderHook(() => useDashboardLayout()); + act(() => { + first.result.current.removeWidget("stat-shows"); + first.result.current.resizeWidget("trakt-sync", 12); + first.result.current.moveWidget("recent-activity", "now-playing"); + }); + const saved = first.result.current.entries; + first.unmount(); + + const second = renderHook(() => useDashboardLayout()); + expect(second.result.current.entries).toEqual(saved); + expect(second.result.current.hiddenWidgets.map((w) => w.id)).toEqual(["stat-shows"]); + }); +}); diff --git a/web/src/components/admin/dashboard/useDashboardLayout.ts b/web/src/components/admin/dashboard/useDashboardLayout.ts new file mode 100644 index 000000000..0ea8cd4bd --- /dev/null +++ b/web/src/components/admin/dashboard/useDashboardLayout.ts @@ -0,0 +1,185 @@ +import { useCallback, useMemo, useState } from "react"; +import { DASHBOARD_WIDGETS, DEFAULT_LAYOUT, findDashboardWidget } from "./registry"; +import type { DashboardLayoutEntry, DashboardWidgetDefinition, WidgetId } from "./types"; + +export const DASHBOARD_LAYOUT_STORAGE_KEY = "silo.admin-dashboard-layout.v1"; + +interface StoredLayout { + version: 1; + entries: DashboardLayoutEntry[]; +} + +function clampSpan(span: unknown, widget: DashboardWidgetDefinition): number { + if (typeof span !== "number" || !Number.isFinite(span)) { + return widget.defaultSpan; + } + return Math.min(widget.maxSpan, Math.max(widget.minSpan, Math.round(span))); +} + +function loadStoredLayout(): DashboardLayoutEntry[] { + let raw: string | null = null; + try { + raw = window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY); + } catch { + return [...DEFAULT_LAYOUT]; + } + if (!raw) { + return [...DEFAULT_LAYOUT]; + } + try { + const parsed = JSON.parse(raw) as Partial | null; + if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.entries)) { + return [...DEFAULT_LAYOUT]; + } + const seen = new Set(); + const entries: DashboardLayoutEntry[] = []; + for (const entry of parsed.entries) { + if (!entry || typeof entry !== "object" || typeof entry.id !== "string") { + continue; + } + const widget = findDashboardWidget(entry.id); + if (!widget || seen.has(widget.id)) { + continue; + } + seen.add(widget.id); + entries.push({ id: widget.id, span: clampSpan(entry.span, widget) }); + } + return entries; + } catch { + return [...DEFAULT_LAYOUT]; + } +} + +function persistLayout(entries: DashboardLayoutEntry[]) { + try { + const stored: StoredLayout = { version: 1, entries }; + window.localStorage.setItem(DASHBOARD_LAYOUT_STORAGE_KEY, JSON.stringify(stored)); + } catch { + // Storage may be unavailable (private mode, quota); the layout still works in-memory. + } +} + +export interface DashboardLayout { + entries: DashboardLayoutEntry[]; + hiddenWidgets: DashboardWidgetDefinition[]; + isCustomizing: boolean; + setCustomizing: (customizing: boolean) => void; + moveWidget: (id: WidgetId, beforeId: WidgetId | null) => void; + resizeWidget: (id: WidgetId, span: number) => void; + removeWidget: (id: WidgetId) => void; + addWidget: (id: WidgetId) => void; + resetLayout: () => void; +} + +export function useDashboardLayout(): DashboardLayout { + const [entries, setEntries] = useState(loadStoredLayout); + const [isCustomizing, setCustomizing] = useState(false); + + const update = useCallback( + (updater: (prev: DashboardLayoutEntry[]) => DashboardLayoutEntry[]) => { + setEntries((prev) => { + const next = updater(prev); + if (next === prev) { + return prev; + } + persistLayout(next); + return next; + }); + }, + [], + ); + + const moveWidget = useCallback( + (id: WidgetId, beforeId: WidgetId | null) => { + update((prev) => { + if (id === beforeId) { + return prev; + } + const moving = prev.find((entry) => entry.id === id); + if (!moving) { + return prev; + } + const without = prev.filter((entry) => entry.id !== id); + if (beforeId === null) { + return [...without, moving]; + } + const index = without.findIndex((entry) => entry.id === beforeId); + if (index === -1) { + return [...without, moving]; + } + return [...without.slice(0, index), moving, ...without.slice(index)]; + }); + }, + [update], + ); + + const resizeWidget = useCallback( + (id: WidgetId, span: number) => { + update((prev) => { + const widget = findDashboardWidget(id); + if (!widget) { + return prev; + } + const nextSpan = clampSpan(span, widget); + let changed = false; + const next = prev.map((entry) => { + if (entry.id !== id || entry.span === nextSpan) { + return entry; + } + changed = true; + return { ...entry, span: nextSpan }; + }); + return changed ? next : prev; + }); + }, + [update], + ); + + const removeWidget = useCallback( + (id: WidgetId) => { + update((prev) => + prev.some((entry) => entry.id === id) ? prev.filter((entry) => entry.id !== id) : prev, + ); + }, + [update], + ); + + const addWidget = useCallback( + (id: WidgetId) => { + update((prev) => { + const widget = findDashboardWidget(id); + if (!widget || prev.some((entry) => entry.id === id)) { + return prev; + } + return [...prev, { id: widget.id, span: widget.defaultSpan }]; + }); + }, + [update], + ); + + const resetLayout = useCallback(() => { + try { + window.localStorage.removeItem(DASHBOARD_LAYOUT_STORAGE_KEY); + } catch { + // Ignore storage failures; state still resets below. + } + setEntries([...DEFAULT_LAYOUT]); + }, []); + + const hiddenWidgets = useMemo(() => { + const visible = new Set(entries.map((entry) => entry.id)); + return DASHBOARD_WIDGETS.filter((widget) => !visible.has(widget.id)); + }, [entries]); + + return { + entries, + hiddenWidgets, + isCustomizing, + setCustomizing, + moveWidget, + resizeWidget, + removeWidget, + addWidget, + resetLayout, + }; +} diff --git a/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx b/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx new file mode 100644 index 000000000..b53e78da0 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx @@ -0,0 +1,157 @@ +import { useMemo } from "react"; +import { Link } from "react-router"; +import { Library, ScanLine, Square } from "lucide-react"; +import { useEventChannel } from "@/components/realtimeEventsContext"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { + useAdminLibraries, + useCancelLibraryScans, + useScanLibrary, +} from "@/hooks/queries/admin/libraries"; +import { useActiveScans } from "@/hooks/queries/admin/scans"; +import { compareActiveScans } from "@/lib/scanRuns"; +import { cn } from "@/lib/utils"; +import type { ScanRun } from "@/api/types"; +import { formatDashboardLibraryScanProgress } from "../format"; +import { LibrarySkeletonRows, SectionError } from "../feedback"; + +export function LibrariesWidget() { + useEventChannel("scans"); + const librariesQuery = useAdminLibraries(); + const libraries = librariesQuery.data ?? []; + const scanLibrary = useScanLibrary(); + const cancelScans = useCancelLibraryScans(); + const { data: activeScans = [] } = useActiveScans(); + + const activeScansByLibraryId = useMemo(() => { + const scansByLibraryID = new Map(); + for (const scan of activeScans) { + if (scan.status !== "accepted" && scan.status !== "running") { + continue; + } + const scans = scansByLibraryID.get(scan.library_id) ?? []; + scans.push(scan); + scansByLibraryID.set(scan.library_id, scans); + } + for (const scans of scansByLibraryID.values()) { + scans.sort(compareActiveScans); + } + return scansByLibraryID; + }, [activeScans]); + + return ( + + + Libraries + + Manage › + + + + {librariesQuery.isLoading ? ( + + ) : librariesQuery.error ? ( + + ) : libraries.length === 0 ? ( +
+ No libraries configured. +
+ ) : ( + libraries.map((lib) => { + const activeLibraryScans = activeScansByLibraryId.get(lib.id) ?? []; + const primaryActiveScan = activeLibraryScans[0]; + const hasActiveScan = activeLibraryScans.length > 0; + const isScanStarting = scanLibrary.isPending && scanLibrary.variables === lib.id; + const isCancellingScan = cancelScans.isPending && cancelScans.variables === lib.id; + const scanProgressLabel = primaryActiveScan + ? formatDashboardLibraryScanProgress(primaryActiveScan, activeLibraryScans.length) + : isScanStarting + ? "Starting scan..." + : ""; + + return ( +
+ {lib.poster_url ? ( + {lib.name} + ) : ( +
+ +
+ )} +
+
{lib.name}
+
+ + {lib.type} · {lib.paths.length} {lib.paths.length === 1 ? "path" : "paths"} + + {scanProgressLabel ? ( + <> + · + + {scanProgressLabel} + + + ) : null} +
+
+
+ +
+
+
+ ); + }) + )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx b/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx new file mode 100644 index 000000000..9c90a5614 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx @@ -0,0 +1,184 @@ +import { Link } from "react-router"; +import { Pause, Play } from "lucide-react"; +import { JellyfinSessionPill } from "@/components/JellyfinSessionPill"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminSessions } from "@/hooks/queries/admin/stats"; +import { + activityMethodMeta, + classifyActivityMethod, + getSessionClientLabel, +} from "@/pages/adminActivityPresentation"; +import type { AdminSession } from "@/api/types"; +import { formatRelativeTime } from "@/lib/date"; +import { SectionError } from "../feedback"; +import { SessionProfilePill } from "./SessionProfilePill"; + +export function NowPlayingWidget() { + const sessionsQuery = useAdminSessions(); + const sessions = sessionsQuery.data ?? []; + + return ( +
+
+
Now Playing
+ {sessions.length > 0 && ( + + View all {sessions.length} streams › + + )} +
+ {sessionsQuery.isLoading ? ( +
+ {Array.from({ length: 2 }).map((_, i) => ( + + ))} +
+ ) : sessionsQuery.error ? ( + + ) : sessions.length === 0 ? ( +
No active streams.
+ ) : ( + <> +
+ {sessions.slice(0, 4).map((session) => ( + + ))} +
+ {sessions.length > 4 && ( + + +{sessions.length - 4} more active streams + + )} + + )} +
+ ); +} + +function StreamCard({ session }: { session: AdminSession }) { + const isEpisode = + session.series_name && session.season_number != null && session.episode_number != null; + const title = isEpisode + ? session.episode_name || `S${session.season_number}E${session.episode_number}` + : session.media_title || `File #${session.media_file_id}`; + const username = session.username || `User #${session.user_id}`; + const elapsed = formatRelativeTime(session.started_at, { + rounding: "floor", + justNowLabel: "Just now", + }); + const clientLabel = getSessionClientLabel(session); + const method = classifyActivityMethod(session); + const methodColor = activityMethodMeta(method).badgeClass; + + return ( +
+ {/* Poster */} +
+ {session.poster_url ? ( + {session.media_title} + ) : ( + + )} + {session.is_paused ? ( +
+
+ + Paused +
+
+ ) : null} +
+ + {/* Info */} +
+ {isEpisode ? ( + <> + {session.content_id ? ( + + {title} + + ) : ( +
{title}
+ )} +
+ S{session.season_number} · E{session.episode_number} + {session.series_name ? ` — ${session.series_name}` : ""} +
+ + ) : ( + <> + {session.content_id ? ( + + {title} + + ) : ( +
{title}
+ )} + {session.media_type && ( +
+ {session.media_type === "movie" ? "Movie" : "Series"} +
+ )} + + )} + + {/* Tags */} +
+ + {method} + + + {clientLabel ? ( + + {clientLabel} + + ) : null} + {session.reporting_node && ( + + {session.node_display_name || session.reporting_node} + + )} + {(session.profile_name || session.profile_id) && ( + + )} +
+ + {/* User */} +
+
+ {username.charAt(0).toUpperCase()} +
+ {username} + {elapsed} +
+
+
+ ); +} diff --git a/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx b/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx new file mode 100644 index 000000000..043132b98 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx @@ -0,0 +1,87 @@ +import { Link } from "react-router"; +import { Play } from "lucide-react"; +import { AdminSessionActions } from "@/components/AdminSessionActions"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAdminSessions } from "@/hooks/queries/admin/stats"; +import { getSessionClientLabel } from "@/pages/adminActivityPresentation"; +import { formatRelativeTime } from "@/lib/date"; +import { ActivitySkeletonRows, SectionError } from "../feedback"; +import { SessionProfilePill } from "./SessionProfilePill"; + +export function RecentActivityWidget() { + const sessionsQuery = useAdminSessions(); + const sessions = sessionsQuery.data ?? []; + + return ( + + + Recent Activity + + View all › + + + + {sessionsQuery.isLoading ? ( + + ) : sessionsQuery.error ? ( + + ) : sessions.length === 0 ? ( +
No recent activity.
+ ) : ( +
+ {sessions.slice(0, 10).map((s) => { + const isEp = s.series_name && s.season_number != null && s.episode_number != null; + const title = isEp + ? s.episode_name || `S${s.season_number}E${s.episode_number}` + : s.media_title || `File #${s.media_file_id}`; + const username = s.username || `User #${s.user_id}`; + const profileDisplay = s.profile_name || s.profile_id || ""; + const clientLabel = getSessionClientLabel(s); + const meta = [ + formatRelativeTime(s.started_at, { rounding: "floor", justNowLabel: "Just now" }), + clientLabel, + ] + .filter(Boolean) + .join(" · "); + return ( +
+
+ +
+
+
+ {username} + {profileDisplay ? ( + <> + {" "} + + + ) : null} + {" started watching "} + + {title} + +
+
{meta}
+
+
+ +
+
+ ); + })} +
+ )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/SessionProfilePill.tsx b/web/src/components/admin/dashboard/widgets/SessionProfilePill.tsx new file mode 100644 index 000000000..1a73a2a13 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/SessionProfilePill.tsx @@ -0,0 +1,7 @@ +export function SessionProfilePill({ label }: { label: string }) { + return ( + + {label} + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/TraktSyncWidget.tsx b/web/src/components/admin/dashboard/widgets/TraktSyncWidget.tsx new file mode 100644 index 000000000..0ff80be45 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/TraktSyncWidget.tsx @@ -0,0 +1,92 @@ +import { Link } from "react-router"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminStats } from "@/hooks/queries/admin/stats"; +import { formatRelativeTime } from "@/lib/date"; +import { SectionError } from "../feedback"; + +export function TraktSyncWidget() { + const statsQuery = useAdminStats(); + const activity = statsQuery.data?.watch_provider_activity; + + if (statsQuery.isLoading) { + return ; + } + + if (statsQuery.error) { + return ( +
+ +
+ ); + } + + const hasActivity = + activity !== undefined && + (activity.trakt_connected_profiles > 0 || + activity.sync_runs_24h > 0 || + activity.pending_exports > 0 || + activity.open_scrobbles > 0); + + if (!activity || !hasActivity) { + return ( +
+ + Trakt not connected +
+ ); + } + + const lastSync = + formatRelativeTime(activity.last_sync_completed_at, { + rounding: "floor", + justNowLabel: "Just now", + }) ?? "never"; + const errors = activity.sync_errors_24h + activity.failed_exports; + const profiles = activity.trakt_connected_profiles; + + return ( +
+ + + Trakt + {" · "} + {profiles.toLocaleString()} {profiles === 1 ? "profile" : "profiles"} + {" · synced "} + {lastSync} + | + {"24h: "} + + {activity.imported_watched_24h.toLocaleString()} + + {" in / "} + + {activity.exported_watched_24h.toLocaleString()} + + {" out · "} + 0 ? "text-destructive font-medium" : "text-foreground font-medium"} + > + {errors.toLocaleString()} + + {errors === 1 ? " error" : " errors"} + + + Manage › + +
+ ); +} + +function TraktMark() { + return ( + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/UsersWidget.tsx b/web/src/components/admin/dashboard/widgets/UsersWidget.tsx new file mode 100644 index 000000000..418621382 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/UsersWidget.tsx @@ -0,0 +1,86 @@ +import { Link, useNavigate } from "react-router"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { useAdminUsers } from "@/hooks/queries/admin/users"; +import { SectionError, UserSkeletonRows } from "../feedback"; + +export function UsersWidget() { + const navigate = useNavigate(); + const usersQuery = useAdminUsers(); + const users = usersQuery.data ?? []; + + return ( + + + Users + + Manage › + + + + {usersQuery.isLoading ? ( + + ) : usersQuery.error ? ( + + ) : users.length === 0 ? ( +
No users.
+ ) : ( + + + + User + Role + Status + + + + {users.slice(0, 8).map((u) => ( + navigate(`/admin/users/${u.id}`)} + > + +
+
+ {u.username.charAt(0).toUpperCase()} +
+
+
{u.username}
+
+ {u.email} +
+
+
+ + + {u.role} + + + + {u.enabled ? "Active" : "Disabled"} + + + + ))} + +
+ )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/statTiles.tsx b/web/src/components/admin/dashboard/widgets/statTiles.tsx new file mode 100644 index 000000000..22465de6e --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/statTiles.tsx @@ -0,0 +1,125 @@ +import type { ReactNode } from "react"; +import { Activity, Film, HardDrive, Tv, Users } from "lucide-react"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminSessions, useAdminStats } from "@/hooks/queries/admin/stats"; +import { formatFileCount } from "../format"; + +function StatTile({ + label, + value, + sub, + icon, + isLoading, + error, +}: { + label: string; + value: string; + sub: string; + icon: ReactNode; + isLoading: boolean; + error: unknown; +}) { + if (isLoading) { + return ; + } + + return ( +
+
+
{label}
+
{icon}
+
+ {error ? ( +
Unavailable
+ ) : ( + <> +
+ {value} +
+
{sub}
+ + )} +
+ ); +} + +export function ActiveStreamsStatWidget() { + const sessionsQuery = useAdminSessions(); + const sessionCount = sessionsQuery.data?.length ?? 0; + return ( + } + isLoading={sessionsQuery.isLoading} + error={sessionsQuery.error} + /> + ); +} + +export function MoviesStatWidget() { + const statsQuery = useAdminStats(); + const stats = statsQuery.data; + return ( + } + isLoading={statsQuery.isLoading || (!stats && !statsQuery.error)} + error={statsQuery.error} + /> + ); +} + +export function ShowsStatWidget() { + const statsQuery = useAdminStats(); + const stats = statsQuery.data; + return ( + } + isLoading={statsQuery.isLoading || (!stats && !statsQuery.error)} + error={statsQuery.error} + /> + ); +} + +export function UsersStatWidget() { + const statsQuery = useAdminStats(); + const stats = statsQuery.data; + return ( + } + isLoading={statsQuery.isLoading || (!stats && !statsQuery.error)} + error={statsQuery.error} + /> + ); +} + +export function StorageStatWidget() { + const statsQuery = useAdminStats(); + const stats = statsQuery.data; + let storageDisplay = "—"; + if (stats) { + const storageGB = stats.total_storage_bytes / (1024 * 1024 * 1024); + const storageTB = storageGB / 1024; + storageDisplay = storageTB >= 1 ? `${storageTB.toFixed(1)} TB` : `${storageGB.toFixed(0)} GB`; + } + return ( + } + isLoading={statsQuery.isLoading || (!stats && !statsQuery.error)} + error={statsQuery.error} + /> + ); +} diff --git a/web/src/pages/AdminDashboard.tsx b/web/src/pages/AdminDashboard.tsx index 148c6d9a0..c30cbe0b6 100644 --- a/web/src/pages/AdminDashboard.tsx +++ b/web/src/pages/AdminDashboard.tsx @@ -1,86 +1,23 @@ -import type { ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { Link, useNavigate } from "react-router"; -import { AdminSessionActions } from "@/components/AdminSessionActions"; import { AdminSectionCommandDialog } from "@/components/AdminSectionCommandDialog"; -import { useEventChannel } from "@/components/realtimeEventsContext"; -import { fetchAdminStats, useAdminStats, useAdminSessions } from "@/hooks/queries/admin/stats"; +import { DashboardGrid } from "@/components/admin/dashboard/DashboardGrid"; +import { useDashboardLayout } from "@/components/admin/dashboard/useDashboardLayout"; +import { fetchAdminStats, useAdminSessions, useAdminStats } from "@/hooks/queries/admin/stats"; import { useAdminPluginInstallations } from "@/hooks/queries/admin/plugins"; import { usePolicyCapability } from "@/hooks/queries/admin/policy"; import { useAdminUsers } from "@/hooks/queries/admin/users"; -import { - useAdminLibraries, - useCancelLibraryScans, - useScanAllLibraries, - useScanLibrary, -} from "@/hooks/queries/admin/libraries"; -import { useActiveScans } from "@/hooks/queries/admin/scans"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; +import { useAdminLibraries, useScanAllLibraries } from "@/hooks/queries/admin/libraries"; import { Button } from "@/components/ui/button"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; -import { - Activity, - Film, - Tv, - Users, - HardDrive, - RefreshCw, - Play, - Pause, - Library, - ScanLine, - Square, -} from "lucide-react"; -import { Skeleton } from "@/components/ui/skeleton"; -import type { - AdminSession, - AdminStats, - Library as LibraryType, - AdminUser, - ScanRun, - WatchProviderActivity, -} from "@/api/types"; +import { LayoutDashboard, Plus, RefreshCw, ScanLine } from "lucide-react"; import { useQueryClient } from "@tanstack/react-query"; import { adminKeys } from "@/hooks/queries/keys"; import { usePageActivity } from "@/hooks/usePageActivity"; -import { cn } from "@/lib/utils"; import { buildAdminCommandNavSections } from "@/lib/adminNavigation"; -import { compareActiveScans, formatActiveScanMode, formatActiveScanProgress } from "@/lib/scanRuns"; -import { JellyfinSessionPill } from "@/components/JellyfinSessionPill"; -import { - activityMethodMeta, - classifyActivityMethod, - getSessionClientLabel, -} from "@/pages/adminActivityPresentation"; const REFRESH_SPINNER_MIN_VISIBLE_MS = 1_000; const DASHBOARD_AUTO_REFRESH_MS = 60_000; const RELATIVE_UPDATED_LABEL_TICK_MS = 30_000; -function formatFileCount(count: number | null | undefined) { - if (count == null) { - return "—"; - } - return count === 1 ? "1 file" : `${count.toLocaleString()} files`; -} - -function formatDashboardLibraryScanProgress(scan: ScanRun, activeScanCount: number) { - const status = scan.status === "running" ? "Scanning" : "Queued"; - const progress = formatActiveScanProgress(scan); - const detail = - progress || (scan.status === "running" ? formatActiveScanMode(scan) : "Waiting for capacity"); - const extraScans = activeScanCount > 1 ? ` + ${activeScanCount - 1} more` : ""; - return `${status}: ${detail}${extraScans}`; -} - export default function AdminDashboard() { const queryClient = useQueryClient(); const statsQuery = useAdminStats(); @@ -91,15 +28,15 @@ export default function AdminDashboard() { const policyCapability = usePolicyCapability(); const scanAll = useScanAllLibraries(); const pageActivity = usePageActivity(); + const layout = useDashboardLayout(); const manualRefreshStartedAtRef = useRef(null); const wasDashboardPollingPausedRef = useRef(!pageActivity.canPollDashboard); const [isManualRefreshPending, setIsManualRefreshPending] = useState(false); const [lastDashboardUpdatedAt, setLastDashboardUpdatedAt] = useState(null); const [relativeUpdatedNow, setRelativeUpdatedNow] = useState(() => Date.now()); + const [isAddPanelOpen, setIsAddPanelOpen] = useState(false); - const sessions = sessionsQuery.data ?? []; const libraries = librariesQuery.data ?? []; - const users = usersQuery.data ?? []; const { refetch: refetchSessions } = sessionsQuery; const { refetch: refetchLibraries } = librariesQuery; const { refetch: refetchUsers } = usersQuery; @@ -228,6 +165,13 @@ export default function AdminDashboard() { refreshDashboard, ]); + const toggleCustomizing = useCallback(() => { + layout.setCustomizing(!layout.isCustomizing); + if (layout.isCustomizing) { + setIsAddPanelOpen(false); + } + }, [layout]); + return (
@@ -262,6 +206,16 @@ export default function AdminDashboard() { /> {isManualRefreshPending ? "Refreshing..." : "Refresh"} +
- - - {statsQuery.data?.watch_provider_activity && ( - - )} - - - -
- - -
- - -
- ); -} - -// --- Sub-components --- - -function StatsRow({ - stats, - sessionCount, - isLoading, - error, -}: { - stats: AdminStats | undefined; - sessionCount: number; - isLoading: boolean; - error: unknown; -}) { - if (isLoading || !stats) { - if (error) { - return ; - } - return ( -
- {Array.from({ length: 5 }).map((_, i) => ( - - ))} -
- ); - } - - const storageGB = stats.total_storage_bytes / (1024 * 1024 * 1024); - const storageTB = storageGB / 1024; - const storageDisplay = - storageTB >= 1 ? `${storageTB.toFixed(1)} TB` : `${storageGB.toFixed(0)} GB`; - - const statCards: { label: string; value: string; sub: string; icon: ReactNode }[] = [ - { - label: "Active Streams", - value: String(sessionCount), - sub: sessionCount === 1 ? "1 session" : `${sessionCount} sessions`, - icon: , - }, - { - label: "Total Movies", - value: stats.total_movies.toLocaleString(), - sub: formatFileCount(stats.total_movie_files), - icon: , - }, - { - label: "Total Shows", - value: stats.total_shows.toLocaleString(), - sub: formatFileCount(stats.total_show_files), - icon: , - }, - { - label: "Users", - value: String(stats.total_users), - sub: `${stats.total_users} registered`, - icon: , - }, - { - label: "Storage", - value: storageDisplay, - sub: formatFileCount(stats.total_files), - icon: , - }, - ]; - - return ( -
- {statCards.map((card) => ( -
-
-
{card.label}
-
{card.icon}
-
-
- {card.value} -
-
{card.sub}
-
- ))} -
- ); -} - -function TraktActivityCard({ activity }: { activity: WatchProviderActivity }) { - const hasActivity = - activity.trakt_connected_profiles > 0 || - activity.sync_runs_24h > 0 || - activity.pending_exports > 0 || - activity.open_scrobbles > 0; - - if (!hasActivity) return null; - - const lastSync = activity.last_sync_completed_at - ? getTimeAgo(activity.last_sync_completed_at) - : "Never"; - - return ( - - - Trakt Activity - - Task details › - - - -
- - - - -
-
-
- Export enabled:{" "} - - {activity.trakt_export_enabled.toLocaleString()} - -
-
- Scrobbling:{" "} - - {activity.trakt_scrobble_enabled.toLocaleString()} - -
-
- Errors:{" "} - 0 - ? "text-destructive font-medium" - : "text-foreground font-medium" - } - > - {(activity.sync_errors_24h + activity.failed_exports).toLocaleString()} - -
-
- - - ); -} - -function TraktMetric({ label, value, detail }: { label: string; value: string; detail: string }) { - return ( -
-
{label}
-
{value}
-
{detail}
-
- ); -} - -function StreamCard({ session }: { session: AdminSession }) { - const isEpisode = - session.series_name && session.season_number != null && session.episode_number != null; - const title = isEpisode - ? session.episode_name || `S${session.season_number}E${session.episode_number}` - : session.media_title || `File #${session.media_file_id}`; - const username = session.username || `User #${session.user_id}`; - const elapsed = getTimeAgo(session.started_at); - const clientLabel = getSessionClientLabel(session); - const method = classifyActivityMethod(session); - const methodColor = activityMethodMeta(method).badgeClass; - - return ( -
- {/* Poster */} -
- {session.poster_url ? ( - {session.media_title} - ) : ( - - )} - {session.is_paused ? ( -
-
- - Paused -
-
- ) : null} -
- - {/* Info */} -
- {isEpisode ? ( - <> - {session.content_id ? ( - - {title} - - ) : ( -
{title}
- )} -
- S{session.season_number} · E{session.episode_number} - {session.series_name ? ` — ${session.series_name}` : ""} -
- - ) : ( - <> - {session.content_id ? ( - - {title} - - ) : ( -
{title}
- )} - {session.media_type && ( -
- {session.media_type === "movie" ? "Movie" : "Series"} -
- )} - - )} - - {/* Tags */} -
- + + + Drag a widget to move it · drag its right edge to resize · × removes it - - {clientLabel ? ( - - {clientLabel} - - ) : null} - {session.reporting_node && ( - - {session.node_display_name || session.reporting_node} - - )} - {(session.profile_name || session.profile_id) && ( - - )} -
- - {/* User */} -
-
layout.resetLayout()} > - {username.charAt(0).toUpperCase()} -
- {username} - {elapsed} -
-
-
- ); -} - -function SessionProfilePill({ label }: { label: string }) { - return ( - - {label} - - ); -} - -function NowPlayingSection({ - sessions, - isLoading, - error, -}: { - sessions: AdminSession[]; - isLoading: boolean; - error: unknown; -}) { - if (error) return null; - - if (isLoading) { - return ( -
-
-
Now Playing
-
-
- {Array.from({ length: 2 }).map((_, i) => ( - - ))} + Reset to default layout +
-
- ); - } - - if (sessions.length === 0) return null; - - return ( -
-
-
Now Playing
- - View all {sessions.length} streams › - -
-
- {sessions.slice(0, 4).map((session) => ( - - ))} -
- {sessions.length > 4 && ( - - +{sessions.length - 4} more active streams - )} -
- ); -} -function LibrariesCard({ - libraries, - isLoading, - error, -}: { - libraries: LibraryType[]; - isLoading: boolean; - error: unknown; -}) { - useEventChannel("scans"); - const scanLibrary = useScanLibrary(); - const cancelScans = useCancelLibraryScans(); - const { data: activeScans = [] } = useActiveScans(); - - const activeScansByLibraryId = useMemo(() => { - const scansByLibraryID = new Map(); - for (const scan of activeScans) { - if (scan.status !== "accepted" && scan.status !== "running") { - continue; - } - const scans = scansByLibraryID.get(scan.library_id) ?? []; - scans.push(scan); - scansByLibraryID.set(scan.library_id, scans); - } - for (const scans of scansByLibraryID.values()) { - scans.sort(compareActiveScans); - } - return scansByLibraryID; - }, [activeScans]); - - return ( - - - Libraries - - Manage › - - - - {isLoading ? ( - - ) : error ? ( - - ) : libraries.length === 0 ? ( -
- No libraries configured. -
- ) : ( - libraries.map((lib) => { - const activeLibraryScans = activeScansByLibraryId.get(lib.id) ?? []; - const primaryActiveScan = activeLibraryScans[0]; - const hasActiveScan = activeLibraryScans.length > 0; - const isScanStarting = scanLibrary.isPending && scanLibrary.variables === lib.id; - const isCancellingScan = cancelScans.isPending && cancelScans.variables === lib.id; - const scanProgressLabel = primaryActiveScan - ? formatDashboardLibraryScanProgress(primaryActiveScan, activeLibraryScans.length) - : isScanStarting - ? "Starting scan..." - : ""; - - return ( -
- {lib.poster_url ? ( - {lib.name} - ) : ( -
- -
- )} -
-
{lib.name}
-
- - {lib.type} · {lib.paths.length} {lib.paths.length === 1 ? "path" : "paths"} - - {scanProgressLabel ? ( - <> - · - - {scanProgressLabel} - - - ) : null} -
-
-
- -
-
-
- ); - }) - )} - - - ); -} - -function UsersCard({ - users, - isLoading, - error, -}: { - users: AdminUser[]; - isLoading: boolean; - error: unknown; -}) { - const navigate = useNavigate(); - - return ( - - - Users - - Manage › - - - - {isLoading ? ( - - ) : error ? ( - - ) : users.length === 0 ? ( -
No users.
- ) : ( - - - - User - Role - Status - - - - {users.slice(0, 8).map((u) => ( - navigate(`/admin/users/${u.id}`)} - > - -
-
- {u.username.charAt(0).toUpperCase()} -
-
-
{u.username}
-
- {u.email} -
-
-
- - - {u.role} - - - - {u.enabled ? "Active" : "Disabled"} - - - - ))} - -
- )} - - - ); -} - -function ActivityCard({ - sessions, - isLoading, - error, -}: { - sessions: AdminSession[]; - isLoading: boolean; - error: unknown; -}) { - if (!isLoading && !error && sessions.length === 0) return null; - - return ( - - - Recent Activity - - View all › - - - - {isLoading ? ( - - ) : error ? ( - - ) : ( -
- {sessions.slice(0, 10).map((s) => { - const isEp = s.series_name && s.season_number != null && s.episode_number != null; - const title = isEp - ? s.episode_name || `S${s.season_number}E${s.episode_number}` - : s.media_title || `File #${s.media_file_id}`; - const username = s.username || `User #${s.user_id}`; - const profileDisplay = s.profile_name || s.profile_id || ""; - const clientLabel = getSessionClientLabel(s); - const meta = [getTimeAgo(s.started_at), clientLabel].filter(Boolean).join(" · "); - return ( -
-
- -
-
-
- {username} - {profileDisplay ? ( - <> - {" "} - - - ) : null} - {" started watching "} - - {title} - -
-
{meta}
-
-
- -
-
- ); - })} -
- )} - - + +
); } // --- Helpers --- -function getTimeAgo(dateStr: string): string { - const now = Date.now(); - const then = new Date(dateStr).getTime(); - const diff = Math.max(0, now - then); - const minutes = Math.floor(diff / 60000); - if (minutes < 1) return "Just now"; - if (minutes < 60) return `${minutes}m ago`; - const hours = Math.floor(minutes / 60); - if (hours < 24) return `${hours}h ago`; - const days = Math.floor(hours / 24); - return `${days}d ago`; -} - function formatRelativeUpdatedLabel(now: number, updatedAt: number) { const elapsedMinutes = Math.floor(Math.max(0, now - updatedAt) / 60_000); if (elapsedMinutes < 1) { @@ -1005,43 +285,3 @@ function delay(ms: number) { window.setTimeout(resolve, ms); }); } - -function SectionError({ message }: { message: string }) { - return
{message}
; -} - -function LibrarySkeletonRows() { - return ( - <> - {Array.from({ length: 3 }).map((_, i) => ( - - ))} - - ); -} - -function UserSkeletonRows() { - return ( -
- {Array.from({ length: 4 }).map((_, i) => ( - - ))} -
- ); -} - -function ActivitySkeletonRows() { - return ( -
- {Array.from({ length: 4 }).map((_, i) => ( -
- -
- - -
-
- ))} -
- ); -} From b6aa86b1fade0cd06d8cf57a0975318de39d6906 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:29:38 -0400 Subject: [PATCH 004/163] feat(nodemetrics): sample per-node system and gpu usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New internal/nodemetrics sampler on nodes and the API host: CPU (cgroup quota-corrected) with load and cores, memory via the meminfo/cgroup readers extracted from postgres_tune, network throughput, and Statfs disk usage on the transcode scratch dir and library roots — every mount probed in its own bounded goroutine so a hung network filesystem can never block sampling, health, or a scrape. GPU usage comes from DRM fdinfo of the node's own ffmpeg children, tracked per DRM client and summed per device, with nvidia-smi enrichment behind a circuit breaker; per-device session counts now include single-device QSV/VAAPI and NVENC workloads and join the sampler through a device alias set. Health responses carry the sample (bounded at 256KiB body / 32KiB stats); the 30s health write persists it to a new stream_nodes.last_stats column; node modes mount /metrics with disk series labeled opaquely so library paths stay off the unauthenticated surface; GET /admin/system/resources serves the API host's own sample. Admin UI gains the Nodes system column, live GPU busy/session numbers, and a dashboard server-resources card. Also: hardware detection now skips configured hw_device entries this process cannot open — classified for reporting (detected_backends gains "skipped") but never smoke-encoded — so proxy nodes reading the cluster-wide hw_device stop probe-failing with driver errors and the Nodes page no longer warns about them. Branch-new lint findings cleaned across the touched packages. Phase 3 of the node GPU observability plan. Related issue: #780 Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 64 ++ docs/admin-api.md | 85 +++ docs/wiki/deployment/docker.md | 53 ++ internal/api/handlers/nodes.go | 6 +- internal/api/handlers/nodes_test.go | 4 +- internal/api/handlers/system.go | 51 ++ .../api/handlers/system_resources_test.go | 119 ++++ internal/api/router.go | 6 + internal/api/testdata/media_routes.txt | 1 + internal/catalog/folder_repo.go | 24 + internal/database/postgres_tune.go | 62 +- internal/nodemetrics/cgroupcpu.go | 187 ++++++ internal/nodemetrics/collector.go | 166 ++++++ internal/nodemetrics/collector_test.go | 181 ++++++ internal/nodemetrics/disk.go | 221 +++++++ internal/nodemetrics/disk_test.go | 353 +++++++++++ internal/nodemetrics/fdinfo.go | 290 ++++++++++ internal/nodemetrics/fdinfo_test.go | 339 +++++++++++ internal/nodemetrics/meminfo.go | 181 ++++++ internal/nodemetrics/nvidia.go | 160 +++++ internal/nodemetrics/nvidia_test.go | 276 +++++++++ internal/nodemetrics/sampler.go | 450 +++++++++++++++ internal/nodemetrics/sampler_test.go | 546 ++++++++++++++++++ internal/nodemetrics/snapshot.go | 139 +++++ internal/nodemetrics/statfs_other.go | 11 + internal/nodemetrics/statfs_unix.go | 30 + internal/nodemetrics/system.go | 236 ++++++++ internal/nodepool/health.go | 107 +++- internal/nodepool/health_stats_test.go | 175 ++++++ internal/nodepool/planner_test.go | 6 +- internal/nodepool/proxy_pool.go | 4 +- internal/nodepool/repository.go | 29 +- .../nodepool/repository_last_stats_test.go | 121 ++++ internal/nodepool/transcode_pool.go | 15 +- internal/playback/capabilityhash.go | 2 + internal/playback/directplay_test.go | 6 - internal/playback/gpudetect.go | 156 ++++- internal/playback/gpudetect_test.go | 76 +++ internal/playback/hwdevice.go | 132 ++++- internal/playback/hwdevice_test.go | 138 ++++- internal/playback/transcode.go | 37 +- internal/proxy/capability_snapshot_test.go | 2 +- internal/proxy/metrics_test.go | 117 ++++ internal/proxy/server.go | 65 ++- internal/proxy/testdata/media_routes.txt | 2 + internal/tonemap/tonemap.go | 8 +- internal/transcodenode/metrics_test.go | 132 +++++ internal/transcodenode/server.go | 66 ++- .../transcodenode/testdata/media_routes.txt | 2 + .../sql/20260827004808_node_last_stats.sql | 23 + web/src/api/types.ts | 86 +++ web/src/hooks/queries/admin/system.ts | 25 + web/src/hooks/queries/keys.ts | 1 + web/src/pages/AdminDashboard.tsx | 99 +++- web/src/pages/AdminNodes.tsx | 65 ++- web/src/pages/adminNodesPresentation.test.ts | 424 +++++++++++++- web/src/pages/adminNodesPresentation.ts | 512 +++++++++++++++- 57 files changed, 6695 insertions(+), 179 deletions(-) create mode 100644 internal/api/handlers/system_resources_test.go create mode 100644 internal/nodemetrics/cgroupcpu.go create mode 100644 internal/nodemetrics/collector.go create mode 100644 internal/nodemetrics/collector_test.go create mode 100644 internal/nodemetrics/disk.go create mode 100644 internal/nodemetrics/disk_test.go create mode 100644 internal/nodemetrics/fdinfo.go create mode 100644 internal/nodemetrics/fdinfo_test.go create mode 100644 internal/nodemetrics/meminfo.go create mode 100644 internal/nodemetrics/nvidia.go create mode 100644 internal/nodemetrics/nvidia_test.go create mode 100644 internal/nodemetrics/sampler.go create mode 100644 internal/nodemetrics/sampler_test.go create mode 100644 internal/nodemetrics/snapshot.go create mode 100644 internal/nodemetrics/statfs_other.go create mode 100644 internal/nodemetrics/statfs_unix.go create mode 100644 internal/nodemetrics/system.go create mode 100644 internal/nodepool/health_stats_test.go create mode 100644 internal/nodepool/repository_last_stats_test.go create mode 100644 internal/proxy/metrics_test.go create mode 100644 internal/transcodenode/metrics_test.go create mode 100644 migrations/sql/20260827004808_node_last_stats.sql diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 79d171e5b..fe01914b7 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -80,6 +80,7 @@ import ( _ "github.com/Silo-Server/silo-server/internal/metadata/nfo" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/nodesessions" @@ -163,6 +164,48 @@ func nodeCapabilityFetcher(jwtSecret string) nodepool.CapabilityFetcher { } } +// libraryPathProvider adapts the folder repository to the sampler's media-root +// provider. +// +// It runs on the sampling goroutine every interval, so it is bounded +// separately from the caller's context: a database that has stopped answering +// must cost one interval's disk sampling, not the sampler. A failed read +// reports no roots for that pass rather than an error, because the previous +// pass's mounts are still tracked and keep reporting. +func libraryPathProvider(repo *catalog.FolderRepository) func(context.Context) []string { + if repo == nil { + return nil + } + return func(ctx context.Context) []string { + queryCtx, cancel := context.WithTimeout(ctx, libraryPathQueryTimeout) + defer cancel() + paths, err := repo.DistinctLibraryPaths(queryCtx) + if err != nil { + slog.DebugContext(ctx, "library paths unavailable for resource sampling", "component", "app", "error", err) + return nil + } + return paths + } +} + +// libraryPathQueryTimeout bounds the per-sample library root lookup. +const libraryPathQueryTimeout = 2 * time.Second + +// hostRenderDeviceIdentities adapts the playback hardware walk to what the +// sampler needs, keeping the sampler free of any playback dependency. +func hostRenderDeviceIdentities() []nodemetrics.DeviceIdentity { + devices := playback.RenderDeviceIdentities() + identities := make([]nodemetrics.DeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, nodemetrics.DeviceIdentity{ + Path: device.Path, + PCIAddress: device.PCIAddress, + Vendor: device.Vendor, + }) + } + return identities +} + // nodeCapabilityRequestTimeout bounds one capability request. A cold node runs // ffmpeg probes to answer and advertises a probe budget of up to ~2 minutes; // the fetch runs detached from the health sweep, so matching that budget is @@ -872,6 +915,9 @@ func main() { // Keep the capability hash /health advertises current, so the API's // sweep refetches this proxy's inventory only when it actually changed. srv.StartCapabilitySnapshots(appCtx) + // Sample host and GPU resources so /health, /status and /metrics can + // report them without doing any work on the request. + srv.StartMetricsSampler(appCtx) handler = srv.Handler() } else { srv := transcodenode.NewServer(watcher, tracker) @@ -885,6 +931,9 @@ func main() { // The first capability snapshot waits on warmup so it measures a // primed encoder rather than racing it into a probe failure. srv.StartCapabilitySnapshots(appCtx, srv.StartHardwareEncoderWarmup(appCtx)) + // Sample host and GPU resources so /health, /status and /metrics can + // report them without doing any work on the request. + srv.StartMetricsSampler(appCtx) // Reclaim orphaned transcode dirs at boot and hourly thereafter, bound // to appCtx so it stops on shutdown. srv.StartOrphanSweeper(appCtx) @@ -1108,6 +1157,21 @@ func main() { healthChecker.Start(appCtx) slog.Info("node pools initialized", "proxy_nodes", len(proxyNodes), "transcode_nodes", len(transcodeNodes)) + // The API host runs the same sampler the nodes do. It is not a + // registered stream node, so without this the machine serving admin + // requests — and, in integrated mode, transcoding — is the one host with + // no resource visibility. Unlike a node it also samples library roots: + // it is the process that knows what the library is, and its own view of + // a media mount is the one that is authoritative. + resourceSampler := nodemetrics.NewSampler(nodemetrics.Options{ + ScratchDir: cfg.Playback.TranscodeDir, + MediaRoots: libraryPathProvider(catalog.NewFolderRepository(pool)), + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: hostRenderDeviceIdentities, + }) + resourceSampler.Start(appCtx) + deps.ResourceSampler = resourceSampler + // Subscribe to node pool change events for multi-instance reload. _ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) { if event.Type == cache.EventNodePoolChanged { diff --git a/docs/admin-api.md b/docs/admin-api.md index b505b6b29..970469ddd 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -30,6 +30,60 @@ Always `200 OK` with a JSON array. | `capabilities_hash` | string | Identity of that report, as computed by the node. Omitted with `capabilities`. | | `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | | `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | +| `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | + +### `last_stats` + +Written by the same 30-second health check that writes `active_jobs`, so it is +exactly as old as `last_health_check` and never fresher. It is the current +sample only: nothing here is a time series, and operators who want history +scrape the node's own `GET /metrics` (unauthenticated, on the node's listener, +same `streamapp_node_*` gauges, with disk series labeled by role rather than by +path). A sample larger than 32 KiB is dropped rather than stored — the health +verdict is what routes streams, and no honest sample comes close to that. + +`last_stats.system`: + +| Field | Type | Meaning | +|---|---|---| +| `cpu_pct` | int | Aggregate busy percentage across all cores over the last sampling interval (5s), 0-100. Idle and iowait both count as not busy. Under a cgroup this is the container's own consumption against its own quota, not the host's. | +| `load1` | float | 1-minute load average. Unlike `cpu_pct` it also counts tasks blocked on storage, so a node stuck on I/O looks idle in one and busy in the other. Always host-wide: the kernel keeps no per-cgroup load average. | +| `cores` | int | CPUs this process may run on — the cgroup's CPU quota rounded up where one is set, otherwise every CPU the kernel reports. This is what `cpu_pct` is normalized against and what `load1` must be read relative to. | +| `mem_used_mb`, `mem_total_mb` | int | Memory. Under a cgroup these are the cgroup's limit and working set (page cache excluded), not the host's. | +| `disks` | object[] | Sampled mounts, transcode scratch first, deduplicated by filesystem and capped at 8 — unmeasurable paths included, so the array never grows with the library count. | +| `net_rx_bps`, `net_tx_bps` | int | Aggregate throughput in **bits** per second, loopback excluded. In a container this is the container's own network namespace. | + +Each entry in `disks`: + +| Field | Type | Meaning | +|---|---|---| +| `path` | string | The sampled path. | +| `used_gb`, `total_gb` | float | Capacity in GiB. Used counts filesystem-reserved blocks, matching `df`. | +| `stale` | bool | The numbers are real but carried over from an earlier pass because the current probe has not returned — the normal reading for a network mount whose server went away. Omitted when false. | +| `unavailable` | bool | The path has never been measured on this node (it does not exist here, or the first probe is still hanging). `used_gb`/`total_gb` are meaningless. Omitted when false. | + +Each entry in `last_stats.gpu`: + +| Field | Type | Meaning | +|---|---|---| +| `device` | string | The render node path (`/dev/dri/renderD128`), or `cuda:N` for an NVIDIA GPU with no readable DRM node. | +| `vendor` | string | `intel`, `nvidia` or `amd`. Omitted when sysfs names a vendor we do not recognize. | +| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. A workload started with no `playback.hw_device` configured under QSV/VAAPI has no device name until ffmpeg picks one, and is the one case not counted here. | +| `video_busy_pct`, `render_busy_pct` | int | Engine busy percentages over the sampling interval. | +| `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. Present only with an enrichment source — absent is not zero, and must not be rendered as an idle GPU. | +| `vram_used_mb`, `vram_total_mb` | int | GPU memory, on the same terms as `total_busy_pct`. | +| `source` | string | What produced the numbers: `fdinfo`, `nvidia-smi`, `fdinfo+nvidia-smi`, or `unavailable`. | + +`source` is what tells an operator how far to trust the busy percentages. +`fdinfo` is the unprivileged DRM baseline and covers **only this node's own +ffmpeg children** — a GPU shared with anything outside Silo reads as less busy +than it is. `nvidia-smi` is whole-GPU. `unavailable` means nothing could measure +the device this interval; its percentages are zeros with no measurement behind +them. + +A node reports these fields in its own `/health` and `/status`; the API stores +them opaquely and never routes on them. Nothing in node selection reads +`last_stats`. Capability reports are refreshed by the background health sweep, not by this read: a node advertises a `capabilities_hash` in its own health response, and @@ -99,6 +153,9 @@ than as an error status. `404 Not Found` for an unknown id. | `active_jobs`, `egress_kbps` | int | What it reported. Zero when unhealthy. | | `capabilities_hash` | string | The hash the node advertised on this check. Omitted when the node reports none. | +The check also persists the node's resource sample, so `last_stats` on the list +response reflects this check immediately. The sample itself is not echoed here. + This is the node's *current* hash, not the stored one. A value here that differs from the `capabilities_hash` in the list response means the background sweep has a refetch pending; this route does not fetch capabilities itself. @@ -157,6 +214,34 @@ explaining why it could not be probed. The full report for one node — includin `detected_backends`, `boot_id` and `capability_hash` — is what `GET /api/v1/admin/nodes` stores per node in `capabilities`. +## `GET /api/v1/admin/system/resources` + +Reports the **API host's own** current resource sample — the counterpart to the +per-node `last_stats` above. + +The API host is not a registered stream node, so without this route the one +machine an operator cannot see is the machine serving the request (and, in +integrated mode, doing the transcoding). Unlike a node, this host also samples +the configured library roots: it is the process that knows what the library is, +and its view of a media mount is the authoritative one. + +Always `200 OK`. It reads a snapshot the sampler already published, so it costs +nothing and cannot hang regardless of what a mount or a GPU query is doing. + +| Field | Type | Meaning | +|---|---|---| +| `available` | bool | This host can be sampled. False on a non-Linux host, before the first sample lands, or when no sampler is running — in which case the fields below are absent. | +| `sampled_at` | RFC3339 string | When the sample was taken. Omitted when there is none. | +| `system` | object | Same shape as `last_stats.system` above. | +| `gpu` | object[] | Same shape as `last_stats.gpu` above. | + +Sampling is Linux-only: `available: false` on macOS or Windows is expected and +is not an error. History and alerting are Prometheus's job — the same numbers +are exposed as `streamapp_node_*` gauges on this process's existing `/metrics` +endpoint, with one deliberate difference: `/metrics` is unauthenticated, so its +disk series are labeled `mount="scratch"` / `mount="library-N"` and the library +paths themselves appear only here, behind admin auth. + ## `GET /api/v1/admin/stream-telemetry/parity` Returns the merged stream-telemetry view beside the two legacy live-session diff --git a/docs/wiki/deployment/docker.md b/docs/wiki/deployment/docker.md index fb5676293..260841ed9 100644 --- a/docs/wiki/deployment/docker.md +++ b/docs/wiki/deployment/docker.md @@ -209,6 +209,59 @@ NVIDIA_GPU_COUNT=1 Windows uses `;` instead of `:` between entries in `COMPOSE_FILE`. +## Node metrics + +Every Silo process — the API host and each proxy or transcode node — samples its +own CPU, memory, disk, network and GPU usage every five seconds. The numbers +appear on the admin Nodes page, in each node's `/health` and `/status` +responses, and as `streamapp_node_*` gauges on that process's `/metrics` +endpoint. `/metrics` is unauthenticated on node listeners, matching the API +listener; it exposes host resource counters, not media. Disk series are labeled +by role — `mount="scratch"`, `mount="library-1"` — rather than by path, so an +anonymous scrape cannot enumerate where your media lives. The paths themselves +are reported by the admin-authenticated `GET /api/v1/admin/system/resources` and +on the Nodes page. + +Sampling is current-sample only. Silo stores no history — point Prometheus at +`/metrics` if you want trends or alerts. + +**Works out of the box.** System metrics and per-device GPU busyness need no +extra packages, no privileges and no configuration. GPU usage comes from DRM +fdinfo, which the kernel exposes for any process holding a `/dev/dri` device, so +the standard VA-API overlay above is enough for Intel (i915, xe) and AMD +(amdgpu). It measures Silo's own ffmpeg processes only: a GPU shared with +something outside Silo will read as less busy than it is. Each device reports a +`source` field saying which measurement it used. + +**NVIDIA needs the container toolkit.** The proprietary driver implements no +fdinfo, so `nvidia-smi` is the only signal — and it is a whole-GPU one, so it +also sees other tenants. It is already required for NVENC, so the NVIDIA overlay +above gives you GPU utilization, encoder/decoder utilization and VRAM with no +extra step. If the binary is missing or fails repeatedly, that device reports +`source: unavailable` and nothing else degrades. + +**Whole-GPU Intel sampling is not implemented yet.** Reading Intel utilization +across all tenants needs `intel_gpu_top`, which requires `CAP_PERFMON` (or root) +plus a permissive `kernel.perf_event_paranoid` — privileges a plain `/dev/dri` +passthrough container does not have and should not be given by default. Until +that lands, Intel GPUs report the fdinfo baseline only, and `total_busy_pct` is +absent rather than zero. + +**What the numbers mean inside a container.** `/proc/stat` and `/proc/meminfo` +describe the *host*, not the container, so Silo corrects both against the +container's cgroup. Memory reports the cgroup limit and working set (page cache +excluded), which is what the kernel will OOM-kill against. CPU reports the +cgroup's own consumption against its own quota, so a container limited to +`cpus: 2` on a 64-core host reads 100% when it is pegged — not the 3% of the +host machine that same work amounts to — and `cores` is the quota, not the +host's core count. `/proc/net/dev` is +already per-namespace, so network throughput is the container's own traffic. +Disk figures come from `statfs` on the paths the container can see — the +transcode scratch directory on every node, plus the configured library roots on +the API host. A mount that stops responding reports its last good numbers marked +`stale` and never delays a health response; a path a node cannot see at all is +reported `unavailable` rather than as an empty disk. + ## Optional Meilisearch PostgreSQL full-text search needs no extra service. To offer Meilisearch as an diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 81d3143ca..cd74105c8 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -27,7 +27,7 @@ type NodeRepository interface { Create(ctx context.Context, input nodepool.CreateNodeInput) (*nodepool.Node, error) Update(ctx context.Context, id int, input nodepool.UpdateNodeInput) (*nodepool.Node, error) Delete(ctx context.Context, id int) error - UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int) error + UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int, lastStats []byte) error } // NodeListEnabled queries enabled nodes by type for pool reload. @@ -251,9 +251,9 @@ func (h *NodeHandler) HandleCheckNode(w http.ResponseWriter, r *http.Request) { return } - healthy, activeJobs, egressKbps, capabilitiesHash := nodepool.CheckNode(r.Context(), node) + healthy, activeJobs, egressKbps, capabilitiesHash, lastStats := nodepool.CheckNode(r.Context(), node) - if err := h.repo.UpdateHealth(r.Context(), id, healthy, activeJobs, egressKbps); err != nil { + if err := h.repo.UpdateHealth(r.Context(), id, healthy, activeJobs, egressKbps, lastStats); err != nil { slog.ErrorContext(r.Context(), "persisting health check result", "component", "api", "node_id", id, "error", err) } diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 73cba44dd..6164d8423 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -81,7 +81,9 @@ func (s *stubNodeRepository) Update(context.Context, int, nodepool.UpdateNodeInp func (s *stubNodeRepository) Delete(context.Context, int) error { return nil } -func (s *stubNodeRepository) UpdateHealth(context.Context, int, bool, int, int) error { return nil } +func (s *stubNodeRepository) UpdateHealth(context.Context, int, bool, int, int, []byte) error { + return nil +} // The node list is the admin's inventory view, so it must carry the stored // capability report, its age, and the derived GPU identities beside the diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index 7d89e010e..184c18c93 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -9,12 +9,18 @@ import ( "github.com/Silo-Server/silo-server/internal/buildinfo" "github.com/Silo-Server/silo-server/internal/logredact" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" ) const remoteNodeInventoryProbeTimeout = 5 * time.Second +// resourceSampler is the read side of the local host's resource sampler. +type resourceSampler interface { + Snapshot() nodemetrics.Snapshot +} + // SystemHandler serves read-only system inspection endpoints. type SystemHandler struct { transcodePool *nodepool.TranscodePool @@ -23,6 +29,7 @@ type SystemHandler struct { hwAccel string hwDevice string buildInfo buildinfo.Info + resources resourceSampler } // NewSystemHandler creates a SystemHandler. hwAccel and hwDevice are the @@ -39,6 +46,50 @@ func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret, ffmpegPa } } +// SetResourceSampler wires the local host's resource sampler. Without one, +// /admin/system/resources reports the host as unsampled rather than failing: +// the endpoint's answer is "what does this host look like right now", and +// "nothing is measuring it" is a valid answer to that. +func (h *SystemHandler) SetResourceSampler(sampler resourceSampler) { + h.resources = sampler +} + +// SystemResources is the local host's current resource sample. +type SystemResources struct { + // Available is false on a host that cannot be sampled (non-Linux, or before + // the first sample lands), in which case the two fields below are absent. + Available bool `json:"available"` + SampledAt string `json:"sampled_at,omitempty"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` +} + +// HandleSystemResources handles GET /admin/system/resources. +// +// This is the API host's own sample — the counterpart to the per-node +// last_stats on /admin/nodes, which the Nodes page reads. The API host is not a +// registered stream node, so without this route the machine actually serving +// the request is the one machine an operator cannot see. +// +// It reads a snapshot the sampler already published, so it costs nothing and +// cannot hang, no matter what a mount or a GPU query is doing. +func (h *SystemHandler) HandleSystemResources(w http.ResponseWriter, _ *http.Request) { + if h.resources == nil { + writeJSON(w, http.StatusOK, SystemResources{}) + return + } + snapshot := h.resources.Snapshot() + response := SystemResources{ + Available: snapshot.Available, + System: snapshot.System, + GPU: snapshot.GPU, + } + if !snapshot.SampledAt.IsZero() { + response.SampledAt = snapshot.SampledAt.UTC().Format(time.RFC3339) + } + writeJSON(w, http.StatusOK, response) +} + // NodeHWAccel reports one transcode node's GPU inventory. type NodeHWAccel struct { NodeURL string `json:"node_url"` diff --git a/internal/api/handlers/system_resources_test.go b/internal/api/handlers/system_resources_test.go new file mode 100644 index 000000000..70c9c76db --- /dev/null +++ b/internal/api/handlers/system_resources_test.go @@ -0,0 +1,119 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +func TestSystemResourcesReportsLocalSample(t *testing.T) { + t.Parallel() + + sampledAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + total := 71 + handler := &SystemHandler{} + handler.SetResourceSampler(nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: sampledAt, + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, + VideoBusyPct: 63, RenderBusyPct: 12, TotalBusyPct: &total, + Source: nodemetrics.SourceFdinfo, + }}, + })) + + rec := httptest.NewRecorder() + handler.HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + Available bool `json:"available"` + SampledAt string `json:"sampled_at"` + System *struct { + CPUPct int `json:"cpu_pct"` + Disks []struct { + Path string `json:"path"` + } `json:"disks"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + TotalBusyPct *int `json:"total_busy_pct"` + } `json:"gpu"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if !body.Available { + t.Fatalf("available = false: %s", rec.Body) + } + if body.SampledAt != "2026-08-26T12:00:00Z" { + t.Fatalf("sampled_at = %q", body.SampledAt) + } + if body.System == nil || body.System.CPUPct != 41 || len(body.System.Disks) != 1 { + t.Fatalf("system = %+v", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("gpu = %+v", body.GPU) + } + if body.GPU[0].TotalBusyPct == nil || *body.GPU[0].TotalBusyPct != 71 { + t.Fatalf("total_busy_pct = %v, want 71", body.GPU[0].TotalBusyPct) + } +} + +// "Nothing is measuring this host" is a valid answer to "what does this host +// look like", so an unsampled host answers 200 with available:false rather than +// failing the admin page that reads it. +func TestSystemResourcesReportsUnavailableWithoutASampler(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + (&SystemHandler{}).HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if available, _ := body["available"].(bool); available { + t.Fatalf("available = true without a sampler: %s", rec.Body) + } + for _, key := range []string{"system", "gpu", "sampled_at"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sample: %s", key, rec.Body) + } + } +} + +// A host that cannot be sampled (non-Linux) is reported the same way as one +// with no sampler at all, so a client has one case to handle. +func TestSystemResourcesReportsUnavailableHost(t *testing.T) { + t.Parallel() + + handler := &SystemHandler{} + handler.SetResourceSampler(nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{})) + + rec := httptest.NewRecorder() + handler.HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if available, _ := body["available"].(bool); available { + t.Fatalf("available = true on an unsampled host: %s", rec.Body) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index bd4fdcced..a59439ca3 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -49,6 +49,7 @@ import ( "github.com/Silo-Server/silo-server/internal/metadata/tmdb" metatrakt "github.com/Silo-Server/silo-server/internal/metadata/trakt" metadatatranslation "github.com/Silo-Server/silo-server/internal/metadata/translation" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/notifications" @@ -124,6 +125,7 @@ type Dependencies struct { TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) NodeHealthChecker *nodepool.HealthChecker // periodic node health/capability sweep (may be nil) + ResourceSampler *nodemetrics.Sampler // this host's own resource sampler (may be nil) SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger EventBus cache.EventBus AdminStatsProvider handlers.AdminStatsSource @@ -3186,9 +3188,13 @@ func NewRouter(deps Dependencies) chi.Router { sysHWDevice = deps.Config.Playback.HWDevice } systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath, sysHWAccel, sysHWDevice) + if deps.ResourceSampler != nil { + systemHandler.SetResourceSampler(deps.ResourceSampler) + } r.Route("/system", func(r chi.Router) { r.Get("/build", systemHandler.HandleBuildInfo) r.Get("/hw-accel", systemHandler.HandleHWAccel) + r.Get("/resources", systemHandler.HandleSystemResources) }) } diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt index 39c0431ac..0ee622e99 100644 --- a/internal/api/testdata/media_routes.txt +++ b/internal/api/testdata/media_routes.txt @@ -159,6 +159,7 @@ PATCH /api/v1/admin/subtitles/{id}/ non-media GET /api/v1/admin/subtitles/{id}/download non-media GET /api/v1/admin/system/build non-media GET /api/v1/admin/system/hw-accel non-media +GET /api/v1/admin/system/resources non-media GET /api/v1/admin/unmatched non-media GET /api/v1/admin/users non-media POST /api/v1/admin/users non-media diff --git a/internal/catalog/folder_repo.go b/internal/catalog/folder_repo.go index 818a1c60b..4a5a8f235 100644 --- a/internal/catalog/folder_repo.go +++ b/internal/catalog/folder_repo.go @@ -1047,6 +1047,30 @@ func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deleti return unreferenced, nil } +// DistinctLibraryPaths returns every configured library folder path, once each +// and in a stable order. +// +// media_folder_paths is the authoritative list of roots the server was told +// about; a folder can have several. Host resource sampling uses this to decide +// which mounts to report free space on, which is why it reads paths only and +// deliberately does not care which library owns them. +func (r *FolderRepository) DistinctLibraryPaths(ctx context.Context) ([]string, error) { + rows, err := r.pool.Query(ctx, `SELECT DISTINCT path FROM media_folder_paths ORDER BY path`) + if err != nil { + return nil, fmt.Errorf("querying library paths: %w", err) + } + defer rows.Close() + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("scanning library path: %w", err) + } + paths = append(paths, path) + } + return paths, rows.Err() +} + // UpdateLastScanned sets the last_scanned_at timestamp for the given folder. // LibraryRootsForContent returns the media folder root paths the given // content belongs to, resolved through media_item_libraries. The metadata diff --git a/internal/database/postgres_tune.go b/internal/database/postgres_tune.go index fa6951654..62dca0efa 100644 --- a/internal/database/postgres_tune.go +++ b/internal/database/postgres_tune.go @@ -1,7 +1,6 @@ package database import ( - "bufio" "context" "fmt" "math" @@ -10,6 +9,7 @@ import ( "strconv" "strings" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/jackc/pgx/v5/pgxpool" ) @@ -468,20 +468,17 @@ func normalizePostgresTuneDBSize(raw string) (string, error) { } func detectTotalMemoryBytes() (int64, string, error) { - for _, path := range []string{ - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - } { - if mem, err := readCgroupMemoryLimit(path); err == nil && mem > 0 { + for _, path := range nodemetrics.CgroupMemoryLimitPaths() { + if mem, err := nodemetrics.ReadCgroupMemoryLimit(path); err == nil && mem > 0 { return mem, path, nil } } - if mem, err := readMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 { + if mem, err := nodemetrics.ReadMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 { return mem, "/host/proc/meminfo", nil } - mem, err := readMeminfoTotalBytes("/proc/meminfo") + mem, err := nodemetrics.ReadMeminfoTotalBytes("/proc/meminfo") if err != nil { return 0, "", err } @@ -491,35 +488,6 @@ func detectTotalMemoryBytes() (int64, string, error) { return mem, "/proc/meminfo", nil } -func readMeminfoTotalBytes(path string) (int64, error) { - file, err := os.Open(path) - if err != nil { - return 0, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "MemTotal:") { - continue - } - fields := strings.Fields(line) - if len(fields) < 2 { - return 0, fmt.Errorf("malformed MemTotal line") - } - kb, err := strconv.ParseInt(fields[1], 10, 64) - if err != nil { - return 0, err - } - return kb * sizeKB, nil - } - if err := scanner.Err(); err != nil { - return 0, err - } - return 0, fmt.Errorf("MemTotal not found") -} - func runningInContainer() bool { if _, err := os.Stat("/.dockerenv"); err == nil { return true @@ -528,26 +496,6 @@ func runningInContainer() bool { return err == nil && (strings.Contains(string(raw), "docker") || strings.Contains(string(raw), "kubepods")) } -func readCgroupMemoryLimit(path string) (int64, error) { - raw, err := os.ReadFile(path) - if err != nil { - return 0, err - } - value := strings.TrimSpace(string(raw)) - if value == "" || value == "max" { - return 0, fmt.Errorf("no cgroup memory limit") - } - mem, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return 0, err - } - // Docker may expose a huge sentinel when no concrete memory limit is set. - if mem <= 0 || mem > 1<<60 { - return 0, fmt.Errorf("no concrete cgroup memory limit") - } - return mem, nil -} - func parsePostgresTuneByteSize(raw string) (int64, error) { normalized := strings.ToUpper(strings.TrimSpace(raw)) normalized = strings.ReplaceAll(normalized, " ", "") diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go new file mode 100644 index 000000000..b48ba935a --- /dev/null +++ b/internal/nodemetrics/cgroupcpu.go @@ -0,0 +1,187 @@ +package nodemetrics + +import ( + "fmt" + "math" + "os" + "strconv" + "strings" + "time" +) + +// cgroup CPU correction. +// +// /proc/stat describes the host even inside a container, exactly as +// /proc/meminfo does. A transcode node limited to two cores on a 64-core host +// would otherwise report the host's busyness against the host's core count — +// pinned at its quota and dropping segments while the dashboard shows a few +// percent idle. So CPU is corrected the same way memory is: the cgroup's own +// cumulative usage is the busy signal, and its quota is what that usage is +// normalized against. +// +// A host with no cgroup limit reads its root cgroup, which accounts for every +// process on the machine, so an unconstrained deployment reports what it always +// did. + +// cgroupCPUPath locates one cgroup version's CPU accounting. +type cgroupCPUPath struct { + // usage is the file holding cumulative CPU time consumed by the cgroup. + usage string + // usageKey names the row to read when usage is a "key value" table; empty + // when the file holds a bare integer. + usageKey string + // usageUnit is how long one unit in that file is. + usageUnit time.Duration + // quota holds the CPU budget: cgroup v2's " " pair, or + // cgroup v1's quota alone. + quota string + // period is v1's separate period file; empty when quota carries both. + period string +} + +// cgroupCPUUsageKey names the cumulative-usage row of cgroup v2's cpu.stat. +const cgroupCPUUsageKey = "usage_usec" + +// cgroupCPUPaths lists where to read CPU accounting, v2 first. cgroup v1 mounts +// cpu and cpuacct together on most distributions and separately on some, so +// both layouts are tried. +var cgroupCPUPaths = []cgroupCPUPath{ + { + usage: "/sys/fs/cgroup/cpu.stat", + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: "/sys/fs/cgroup/cpu.max", + }, + { + usage: "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage", + usageUnit: time.Nanosecond, + quota: "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us", + period: "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us", + }, + { + usage: "/sys/fs/cgroup/cpuacct/cpuacct.usage", + usageUnit: time.Nanosecond, + quota: "/sys/fs/cgroup/cpu/cpu.cfs_quota_us", + period: "/sys/fs/cgroup/cpu/cpu.cfs_period_us", + }, +} + +// cgroupCPUSample is one cumulative CPU-time reading. Only differences between +// two readings mean anything. +type cgroupCPUSample struct { + usageNS int64 + at time.Time + valid bool +} + +// cgroupCPU returns this process's cgroup CPU reading for the given instant, +// and the CPU budget in cores that reading must be normalized against (0 when +// the cgroup imposes no quota). +func (s *Sampler) cgroupCPU(now time.Time) (cgroupCPUSample, float64) { + for _, paths := range s.cgroupCPUPaths { + usage, err := readCgroupCPUUsage(paths) + if err != nil { + continue + } + quota, err := readCgroupCPUQuota(paths) + if err != nil { + quota = 0 + } + return cgroupCPUSample{usageNS: usage, at: now, valid: true}, quota + } + return cgroupCPUSample{}, 0 +} + +// readCgroupCPUUsage returns cumulative cgroup CPU time in nanoseconds. +func readCgroupCPUUsage(paths cgroupCPUPath) (int64, error) { + var value int64 + var err error + if paths.usageKey != "" { + value, err = readCgroupStatKey(paths.usage, paths.usageKey) + } else { + value, err = readCgroupSingleValue(paths.usage) + } + if err != nil { + return 0, err + } + if value < 0 { + return 0, fmt.Errorf("negative cgroup cpu usage") + } + return value * int64(paths.usageUnit), nil +} + +// readCgroupCPUQuota returns the cgroup's CPU budget in cores, or an error when +// it imposes none. +// +// "No quota" is spelled "max" in v2 and "-1" in v1, and both must read as "this +// cgroup may use the whole host", never as a budget of zero cores. +func readCgroupCPUQuota(paths cgroupCPUPath) (float64, error) { + raw, err := os.ReadFile(paths.quota) + if err != nil { + return 0, err + } + fields := strings.Fields(string(raw)) + if len(fields) == 0 { + return 0, fmt.Errorf("empty cgroup cpu quota") + } + if fields[0] == "max" { + return 0, fmt.Errorf("no cgroup cpu quota") + } + quota, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, err + } + if quota <= 0 { + return 0, fmt.Errorf("no cgroup cpu quota") + } + + period := int64(0) + if len(fields) > 1 { + // cgroup v2 prints the period beside the quota. + period, err = strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, err + } + } else if paths.period != "" { + period, err = readCgroupSingleValue(paths.period) + if err != nil { + return 0, err + } + } + if period <= 0 { + return 0, fmt.Errorf("no cgroup cpu period") + } + return float64(quota) / float64(period), nil +} + +// cgroupCPUPercent converts two cgroup CPU readings into a busy percentage of +// the given core budget. +// +// A usage counter that went backwards means the readings do not describe one +// continuous run (the container was restarted or migrated), so the pair is +// unusable rather than negative. +func cgroupCPUPercent(previous, current cgroupCPUSample, cores float64) (int, bool) { + if !previous.valid || !current.valid || cores <= 0 { + return 0, false + } + elapsedNS := current.at.Sub(previous.at).Nanoseconds() + if elapsedNS <= 0 || current.usageNS < previous.usageNS { + return 0, false + } + busy := float64(current.usageNS-previous.usageNS) * 100 / (float64(elapsedNS) * cores) + return clampPercent(int(busy + 0.5)), true +} + +// cgroupQuotaCores rounds a fractional CPU quota up to whole cores, which is +// how many CPUs the workload can actually be running on at one instant. +func cgroupQuotaCores(quota float64, hostCores int) int { + cores := int(math.Ceil(quota)) + if cores < 1 { + cores = 1 + } + if hostCores > 0 && cores > hostCores { + // A quota above the host's core count is not a limit worth reporting. + return hostCores + } + return cores +} diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go new file mode 100644 index 000000000..bf58e91b9 --- /dev/null +++ b/internal/nodemetrics/collector.go @@ -0,0 +1,166 @@ +package nodemetrics + +import ( + "log/slog" + "strconv" + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +// Prometheus exposure. +// +// The collector is built over the published snapshot rather than over +// promauto gauges written during sampling. A scrape therefore reports exactly +// the numbers the health response and the admin API report for the same +// instant, and — more importantly — a scrape can never wait on sampling, so a +// wedged mount or a hung nvidia-smi cannot turn into a Prometheus scrape +// timeout that looks like the node being down. +// +// Names carry the streamapp_ prefix the rest of the server already uses, and +// the node_ infix so a scrape of an integrated deployment separates host +// resources from request and domain metrics. +var ( + descCPUPercent = prometheus.NewDesc( + "streamapp_node_cpu_percent", + "Aggregate CPU busy percentage across all cores over the last sampling interval.", + nil, nil) + descLoad1 = prometheus.NewDesc( + "streamapp_node_load1", + "1-minute load average.", + nil, nil) + descMemoryUsed = prometheus.NewDesc( + "streamapp_node_memory_used_bytes", + "Memory in use, corrected by the cgroup limit and usage when running under one.", + nil, nil) + descMemoryTotal = prometheus.NewDesc( + "streamapp_node_memory_total_bytes", + "Memory available to this process's cgroup, or the host's when unconstrained.", + nil, nil) + descDiskUsed = prometheus.NewDesc( + "streamapp_node_disk_used_bytes", + "Used bytes on a sampled mount, labeled by role rather than by path.", + []string{"mount"}, nil) + descDiskTotal = prometheus.NewDesc( + "streamapp_node_disk_total_bytes", + "Total bytes on a sampled mount, labeled by role rather than by path.", + []string{"mount"}, nil) + descNetworkRx = prometheus.NewDesc( + "streamapp_node_network_rx_bps", + "Aggregate received bits per second, loopback excluded.", + nil, nil) + descNetworkTx = prometheus.NewDesc( + "streamapp_node_network_tx_bps", + "Aggregate transmitted bits per second, loopback excluded.", + nil, nil) + descGPUVideoBusy = prometheus.NewDesc( + "streamapp_node_gpu_video_busy_percent", + "GPU video engine busy percentage. From DRM fdinfo this covers only this node's own transcodes.", + []string{gpuDeviceLabel}, nil) + descGPURenderBusy = prometheus.NewDesc( + "streamapp_node_gpu_render_busy_percent", + "GPU render engine busy percentage. From DRM fdinfo this covers only this node's own transcodes.", + []string{gpuDeviceLabel}, nil) + descGPUSessions = prometheus.NewDesc( + "streamapp_node_gpu_sessions", + "Active GPU workloads this node has pinned to a device.", + []string{gpuDeviceLabel}, nil) + descGPUVRAMUsed = prometheus.NewDesc( + "streamapp_node_gpu_vram_used_bytes", + "GPU memory in use, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) + descGPUVRAMTotal = prometheus.NewDesc( + "streamapp_node_gpu_vram_total_bytes", + "Total GPU memory, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) +) + +// gpuDeviceLabel is the Prometheus label name every per-GPU series is keyed +// by. +const gpuDeviceLabel = "device" + +// collector adapts a Sampler to prometheus.Collector. +type collector struct{ sampler *Sampler } + +// Describe is deliberately unimplemented (an unchecked collector): the disk and +// GPU label sets are discovered at sample time and legitimately change when a +// mount or a card appears, which a checked collector would reject. +func (collector) Describe(chan<- *prometheus.Desc) {} + +// Collect reads the latest snapshot. It performs no I/O and takes no lock the +// sampler holds while doing I/O, so it cannot block a scrape. +func (c collector) Collect(ch chan<- prometheus.Metric) { + snapshot := c.sampler.Snapshot() + if !snapshot.Available { + return + } + gauge := func(desc *prometheus.Desc, value float64, labels ...string) { + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labels...) + } + if system := snapshot.System; system != nil { + gauge(descCPUPercent, float64(system.CPUPct)) + gauge(descLoad1, system.Load1) + gauge(descMemoryUsed, float64(system.MemUsedMB)*float64(bytesPerMB)) + gauge(descMemoryTotal, float64(system.MemTotalMB)*float64(bytesPerMB)) + gauge(descNetworkRx, float64(system.NetRxBps)) + gauge(descNetworkTx, float64(system.NetTxBps)) + const bytesPerGB = float64(1024 * 1024 * 1024) + libraries := 0 + for _, disk := range system.Disks { + if disk.Unavailable { + // A path this node cannot measure has no value to report, and a + // zero would read as an empty disk in every alert rule. + continue + } + label := diskSeriesLabel(c.sampler.scratchPath(), disk, &libraries) + gauge(descDiskUsed, disk.UsedGB*bytesPerGB, label) + gauge(descDiskTotal, disk.TotalGB*bytesPerGB, label) + } + } + const bytesPerMBFloat = float64(1024 * 1024) + for _, gpu := range snapshot.GPU { + gauge(descGPUVideoBusy, float64(gpu.VideoBusyPct), gpu.Device) + gauge(descGPURenderBusy, float64(gpu.RenderBusyPct), gpu.Device) + gauge(descGPUSessions, float64(gpu.Sessions), gpu.Device) + if gpu.VRAMUsedMB != nil { + gauge(descGPUVRAMUsed, float64(*gpu.VRAMUsedMB)*bytesPerMBFloat, gpu.Device) + } + if gpu.VRAMTotalMB != nil { + gauge(descGPUVRAMTotal, float64(*gpu.VRAMTotalMB)*bytesPerMBFloat, gpu.Device) + } + } +} + +// diskSeriesLabel names a mount for Prometheus without disclosing where it is. +// +// /metrics is deliberately unauthenticated on the same listener that serves the +// API and the SPA, so anything labeled here is public. A library root's path is +// deployment layout, not a host resource counter: publishing it would let any +// anonymous client enumerate a deployment's media mounts, which is precisely +// what the admin-authenticated /admin/system/resources exists to gate. Roles +// keep the series useful — scratch is the volume that kills transcodes when it +// fills, and library ordering is stable for a given configuration — while the +// paths themselves stay behind auth. +// +// libraries counts the non-scratch mounts already labeled in this scrape. +func diskSeriesLabel(scratchDir string, disk DiskStats, libraries *int) string { + if scratchDir != "" && disk.Path == scratchDir { + return "scratch" + } + *libraries++ + return "library-" + strconv.Itoa(*libraries) +} + +// collectorRegistration keeps the default registry to one node collector. +// Integrated mode constructs one sampler, but a test — or a future deployment +// that ran two — must not panic the process on a duplicate registration. +var collectorRegistration sync.Once + +// registerCollector publishes a sampler's readings on the default registry. +func registerCollector(sampler *Sampler) { + collectorRegistration.Do(func() { + if err := prometheus.Register(collector{sampler: sampler}); err != nil { + slog.Warn("node metrics collector not registered", "component", "nodemetrics", "error", err) + } + }) +} diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go new file mode 100644 index 000000000..09a061d1c --- /dev/null +++ b/internal/nodemetrics/collector_test.go @@ -0,0 +1,181 @@ +package nodemetrics + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +// gatherNames collects one scrape's metric names and the first sample value per +// name, which is enough to assert the exposition without asserting on the +// registry's formatting. +func gatherNames(t *testing.T, sampler *Sampler) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + values := make(map[string]float64, len(families)) + for _, family := range families { + for _, metric := range family.GetMetric() { + if _, seen := values[family.GetName()]; seen { + continue + } + values[family.GetName()] = metric.GetGauge().GetValue() + } + } + return values +} + +func TestCollectorExposesSnapshot(t *testing.T) { + f := newDiskFixture(t, "/transcode") + f.answer("/transcode", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + s := f.sampler + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + + values := gatherNames(t, s) + for _, name := range []string{ + "streamapp_node_cpu_percent", + "streamapp_node_load1", + "streamapp_node_memory_used_bytes", + "streamapp_node_memory_total_bytes", + "streamapp_node_disk_used_bytes", + "streamapp_node_disk_total_bytes", + "streamapp_node_network_rx_bps", + "streamapp_node_network_tx_bps", + "streamapp_node_gpu_video_busy_percent", + "streamapp_node_gpu_render_busy_percent", + "streamapp_node_gpu_sessions", + "streamapp_node_gpu_vram_used_bytes", + "streamapp_node_gpu_vram_total_bytes", + } { + if _, ok := values[name]; !ok { + t.Fatalf("%s missing from the scrape (got %v)", name, values) + } + } + // Bytes on the wire, gibibytes in the JSON: the conversion has to survive. + if got := values["streamapp_node_disk_used_bytes"]; got != float64(100<<30) { + t.Fatalf("disk used = %v, want %v bytes", got, float64(100<<30)) + } + if got := values["streamapp_node_gpu_sessions"]; got != 2 { + t.Fatalf("gpu sessions = %v, want 2", got) + } +} + +// /metrics is unauthenticated on the same listener that serves the API and the +// SPA. Labeling disk series by path would let anyone who can reach the server +// enumerate its media mounts — the layout the admin-authenticated resources +// endpoint exists to gate. +func TestCollectorDoesNotLabelDisksByPath(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/nas/movies", "/srv/private/kids-shows") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.answer("/mnt/nas/movies", fsStats{UsedBytes: 2 << 30, TotalBytes: 20 << 30, FSID: "b:1"}) + f.answer("/srv/private/kids-shows", fsStats{UsedBytes: 3 << 30, TotalBytes: 30 << 30, FSID: "c:1"}) + + f.sampleAndSettle(t, 3) + f.sampleAndSettle(t, 3) + + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: f.sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + + labels := map[string]bool{} + for _, family := range families { + if family.GetName() != "streamapp_node_disk_used_bytes" { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if strings.Contains(label.GetValue(), "/") { + t.Fatalf("%s carries a filesystem path in label %s=%q", + family.GetName(), label.GetName(), label.GetValue()) + } + labels[label.GetValue()] = true + } + } + } + for _, want := range []string{"scratch", "library-1", "library-2"} { + if !labels[want] { + t.Fatalf("mount label %q missing from %v", want, labels) + } + } +} + +// A host that cannot be sampled must publish nothing rather than a wall of +// zeros that alert rules would read as a healthy idle machine. +func TestCollectorExposesNothingWhenUnavailable(t *testing.T) { + tree := newProcTree(t) + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.goos = "darwin" + s.sample(context.Background()) + + if values := gatherNames(t, s); len(values) != 0 { + t.Fatalf("scrape returned %v on an unsampled host, want nothing", values) + } +} + +// A scrape must never wait on sampling. If Collect took a lock the disk prober +// holds, a wedged mount would turn into a scrape timeout that looks like the +// node being down — which is the exact failure this package exists to avoid. +func TestCollectorDoesNotBlockOnAWedgedMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/nfs") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.wedge(t, "/nfs") + + f.sampler.sample(context.Background()) + f.awaitProbes(t, 1) // only /transcode can finish; /nfs is parked + + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: f.sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + type result struct { + families []*dto.MetricFamily + err error + } + scraped := make(chan result, 1) + go func() { + families, err := registry.Gather() + scraped <- result{families: families, err: err} + }() + select { + case got := <-scraped: + if got.err != nil { + t.Fatalf("gather: %v", got.err) + } + if len(got.families) == 0 { + t.Fatal("scrape returned no metrics while a mount was wedged") + } + case <-time.After(5 * time.Second): + t.Fatal("scrape blocked behind a wedged mount") + } +} + +// Registration is guarded so a second sampler cannot panic the process on a +// duplicate collector. +func TestRegisterCollectorIsIdempotent(t *testing.T) { + tree := newProcTree(t) + first := newTestSampler(t, tree, newFakeClock(), Options{}) + second := newTestSampler(t, tree, newFakeClock(), Options{}) + registerCollector(first) + registerCollector(second) +} diff --git a/internal/nodemetrics/disk.go b/internal/nodemetrics/disk.go new file mode 100644 index 000000000..2a3b0f6dd --- /dev/null +++ b/internal/nodemetrics/disk.go @@ -0,0 +1,221 @@ +package nodemetrics + +import ( + "strconv" + "time" +) + +// maxSampledDisks caps how many mounts a snapshot reports. A deployment can +// have dozens of library roots, and a health response that grows with the +// library count would eventually be the reason health requests are slow. +const maxSampledDisks = 8 + +// diskProbeTimeout is how long a probe may be outstanding before the entry it +// belongs to is reported stale. It is not a cancellation: statfs(2) on a dead +// NFS server is uninterruptible, so the goroutine stays parked until the mount +// recovers or the process exits. What the timeout bounds is how long a reader +// is told numbers are current. +const diskProbeTimeout = 5 * time.Second + +// fsStats is one filesystem's capacity, in the portable shape this package +// needs from statfs(2). +type fsStats struct { + UsedBytes uint64 + TotalBytes uint64 + // FSID identifies the filesystem itself, so two paths on one volume — the + // common case where scratch and media live on the same disk — are reported + // once instead of twice with identical numbers. It is empty when the + // filesystem publishes no usable id (FUSE mounts do not), in which case each + // path is reported separately rather than collapsed onto a shared non-id. + FSID string +} + +// diskEntry is one path's probe state. Probes run detached from the sample +// loop, so this holds the last good answer for readers to fall back to. +type diskEntry struct { + path string + // inFlight is what keeps a permanently stuck mount from accumulating one + // parked goroutine per sample. A path already being probed is skipped + // entirely; there is never more than one goroutine per path. + inFlight bool + startedAt time.Time + haveGood bool + good fsStats + goodAt time.Time + lastErr bool + unreachable bool +} + +// stale reports whether this entry's last good numbers should be flagged as +// carried over. Either the current probe has outlived its budget — the wedged +// network mount case — or no probe has landed for longer than one full sampling +// cycle plus that budget. +func (e *diskEntry) stale(now time.Time, interval time.Duration) bool { + if e.lastErr { + return true + } + if e.inFlight && now.Sub(e.startedAt) > diskProbeTimeout { + return true + } + return now.Sub(e.goodAt) > interval+diskProbeTimeout +} + +// refreshDisks starts a probe for every path that is not already being probed +// and returns immediately. It never waits for a result: the caller is the +// sample loop, and the whole point of this package is that one bad mount cannot +// delay a node's health answer. +func (s *Sampler) refreshDisks(paths []string, now time.Time) { + s.diskMu.Lock() + defer s.diskMu.Unlock() + + wanted := make(map[string]bool, len(paths)) + for _, path := range paths { + if path != "" { + wanted[path] = true + } + } + s.pruneDisksLocked(wanted) + + seen := make(map[string]bool, len(paths)) + for _, path := range paths { + if path == "" || seen[path] { + continue + } + seen[path] = true + entry := s.disks[path] + if entry == nil { + entry = &diskEntry{path: path} + s.disks[path] = entry + s.diskOrder = append(s.diskOrder, path) + } + if entry.inFlight { + continue + } + entry.inFlight = true + entry.startedAt = now + go s.probeDisk(entry) + } +} + +// pruneDisksLocked forgets paths no longer offered, so a server whose libraries +// churn over months does not accumulate an entry per path ever configured. +// An entry with a probe still parked is kept: dropping it would let the next +// sample start a second goroutine against the same wedged mount, which is +// exactly what the in-flight guard exists to prevent. +// Callers must hold diskMu. +func (s *Sampler) pruneDisksLocked(wanted map[string]bool) { + kept := s.diskOrder[:0] + for _, path := range s.diskOrder { + entry := s.disks[path] + if wanted[path] || (entry != nil && entry.inFlight) { + kept = append(kept, path) + continue + } + delete(s.disks, path) + } + s.diskOrder = kept +} + +// probeDisk runs one statfs and records the outcome. It runs on its own +// goroutine and may never return; that is expected and is why nothing waits on +// it. +func (s *Sampler) probeDisk(entry *diskEntry) { + stats, err := s.statfs(entry.path) + + s.diskMu.Lock() + entry.inFlight = false + if err != nil { + entry.lastErr = true + // A path that has never been measured and just failed is not a mount + // this node can see at all — a media root that exists on another node, + // or a scratch dir that has not been created yet. + entry.unreachable = !entry.haveGood + } else { + entry.lastErr = false + entry.unreachable = false + entry.haveGood = true + entry.good = stats + entry.goodAt = s.now() + } + s.diskMu.Unlock() + + if s.diskProbeDone != nil { + s.diskProbeDone <- entry.path + } +} + +// diskStats reports the latest known state of every tracked path, in the order +// the paths were first offered — scratch dir first, since that is the volume a +// full disk breaks first — deduplicated by filesystem and capped. +func (s *Sampler) diskStats(paths []string, now time.Time) []DiskStats { + s.diskMu.Lock() + defer s.diskMu.Unlock() + + wanted := make(map[string]bool, len(paths)) + for _, path := range paths { + wanted[path] = true + } + + out := make([]DiskStats, 0, min(len(s.diskOrder), maxSampledDisks)) + seenFS := make(map[string]bool, len(s.diskOrder)) + for _, path := range s.diskOrder { + if !wanted[path] { + continue + } + entry := s.disks[path] + if entry == nil { + continue + } + if !entry.haveGood { + // Report it rather than hiding it: a media root this node cannot + // see is a deployment fact an operator needs, and silently dropping + // it looks identical to the path not being configured. + out = append(out, DiskStats{Path: path, Unavailable: true}) + } else { + if entry.good.FSID != "" { + if seenFS[entry.good.FSID] { + continue + } + seenFS[entry.good.FSID] = true + } + out = append(out, DiskStats{ + Path: path, + UsedGB: bytesToGB(entry.good.UsedBytes), + TotalGB: bytesToGB(entry.good.TotalBytes), + Stale: entry.stale(now, s.interval), + }) + } + // The cap applies to every entry, measured or not. A host whose library + // roots all live on other nodes produces nothing but unavailable + // entries, and those grow with the library count just as measured ones + // would. + if len(out) >= maxSampledDisks { + break + } + } + return out +} + +// formatFSID renders a statfs f_fsid, or "" when the filesystem published none. +// +// A zero f_fsid is not an identity: the FUSE protocol has no fsid field at all, +// so every rclone, mergerfs and s3fs mount reports zero — and those are exactly +// the mounts a media server uses as library roots. Formatting that as "0:0" +// would make two unrelated mounts look like one filesystem, and the second one +// would be silently dropped from the disk panel, from Prometheus, and from the +// fullest-mount warning. A mount at 98% nobody can see is worse than a +// duplicated row. +func formatFSID(a, b int64) string { + if a == 0 && b == 0 { + return "" + } + return strconv.FormatInt(a, 16) + ":" + strconv.FormatInt(b, 16) +} + +// bytesToGB converts to gibibytes with two decimals kept, which is the +// precision a capacity readout is read at. +func bytesToGB(value uint64) float64 { + const bytesPerGB = float64(1024 * 1024 * 1024) + gb := float64(value) / bytesPerGB + return float64(int64(gb*100+0.5)) / 100 +} diff --git a/internal/nodemetrics/disk_test.go b/internal/nodemetrics/disk_test.go new file mode 100644 index 000000000..92e557d40 --- /dev/null +++ b/internal/nodemetrics/disk_test.go @@ -0,0 +1,353 @@ +package nodemetrics + +import ( + "context" + "errors" + "os" + "sync" + "testing" + "time" +) + +// diskFixture wires a sampler whose statfs is fully controlled by the test and +// whose probe completions are observable, so nothing here waits on a sleep. +type diskFixture struct { + sampler *Sampler + clock *fakeClock + // mu guards answers and block, which probe goroutines read. + mu sync.Mutex + // answers maps a path to the result its probe returns. + answers map[string]fsStats + // block, when set for a path, parks that probe until it is closed — + // standing in for statfs(2) on a dead NFS server. + block map[string]chan struct{} + done chan string +} + +// wedge makes every subsequent probe of path park forever, as a dead NFS server +// does. Call it only when no probe of that path is in flight, or the test is +// asserting on which sample started the parked probe. +func (f *diskFixture) wedge(t *testing.T, path string) { + t.Helper() + gate := make(chan struct{}) + f.mu.Lock() + f.block[path] = gate + f.mu.Unlock() + t.Cleanup(func() { close(gate) }) +} + +func (f *diskFixture) answer(path string, stats fsStats) { + f.mu.Lock() + defer f.mu.Unlock() + f.answers[path] = stats +} + +func newDiskFixture(t *testing.T, paths ...string) *diskFixture { + t.Helper() + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + clock := newFakeClock() + + f := &diskFixture{ + clock: clock, + answers: map[string]fsStats{}, + block: map[string]chan struct{}{}, + done: make(chan string, 64), + } + roots := paths + scratch := "" + if len(roots) > 0 { + scratch, roots = roots[0], roots[1:] + } + f.sampler = newTestSampler(t, tree, clock, Options{ + ScratchDir: scratch, + MediaRoots: func(context.Context) []string { return roots }, + }) + f.sampler.diskProbeDone = f.done + f.sampler.statfs = func(path string) (fsStats, error) { + f.mu.Lock() + gate, blocked := f.block[path] + stats, known := f.answers[path] + f.mu.Unlock() + if blocked { + <-gate + } + if !known { + return fsStats{}, os.ErrNotExist + } + return stats, nil + } + return f +} + +// sampleAndSettle runs one sampling pass and waits for the probes it started, +// so the next pass begins with nothing in flight. Without this, whether a probe +// launched by the previous pass is still running is a race, and every assertion +// about in-flight state becomes timing-dependent. +func (f *diskFixture) sampleAndSettle(t *testing.T, probes int) []DiskStats { + t.Helper() + disks := f.disks(t) + f.awaitProbes(t, probes) + return disks +} + +// awaitProbes blocks until n probe completions have been reported. Waiting on +// the sampler's own signal keeps the test deterministic without a sleep. +func (f *diskFixture) awaitProbes(t *testing.T, n int) { + t.Helper() + for range n { + select { + case <-f.done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a disk probe to complete") + } + } +} + +func (f *diskFixture) disks(t *testing.T) []DiskStats { + t.Helper() + f.sampler.sample(context.Background()) + system := f.sampler.Snapshot().System + if system == nil { + t.Fatal("no system stats in snapshot") + } + return system.Disks +} + +// Scratch and a media root frequently live on one volume. Reporting both would +// double a dashboard's disk row and make one filling disk look like two. +func TestDiskStatsDeduplicatesByFilesystem(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media", "/other") + f.answer("/transcode", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/media", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/other", fsStats{UsedBytes: 7 << 30, TotalBytes: 8 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 3) // the first pass only launches the probes + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 2 { + t.Fatalf("disks = %+v, want the two distinct filesystems", disks) + } + if disks[0].Path != "/transcode" { + t.Fatalf("disks[0].Path = %q, want the scratch dir first", disks[0].Path) + } + if disks[0].UsedGB != 100 || disks[0].TotalGB != 500 { + t.Fatalf("disks[0] = %+v, want 100/500 GB", disks[0]) + } + if disks[1].Path != "/other" { + t.Fatalf("disks[1].Path = %q, want the second filesystem", disks[1].Path) + } + for _, disk := range disks { + if disk.Stale || disk.Unavailable { + t.Fatalf("disk %+v flagged stale/unavailable on a fresh probe", disk) + } + } +} + +// The contract that matters most: a mount whose server died reports its last +// good numbers marked stale, and never delays a sample. +func TestDiskStatsReportsHungMountAsStaleWithoutBlocking(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/nfs") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.answer("/nfs", fsStats{UsedBytes: 900 << 30, TotalBytes: 1000 << 30, FSID: "n:1"}) + + f.sampleAndSettle(t, 2) + if disks := f.sampleAndSettle(t, 2); disks[1].Stale { + t.Fatalf("disks[1] = %+v, want fresh before the mount wedges", disks[1]) + } + + // The NFS server goes away: every subsequent probe parks forever. Nothing is + // in flight here, so the next sample is unambiguously the one that parks. + f.wedge(t, "/nfs") + + // Well past the probe budget and one sampling interval. + f.clock.advance(time.Minute) + wedgedAt := f.clock.at + disks := f.disks(t) + + f.awaitProbes(t, 1) // the scratch probe still completes normally + if len(disks) != 2 { + t.Fatalf("disks = %+v, want both mounts reported", disks) + } + if disks[1].Path != "/nfs" || !disks[1].Stale { + t.Fatalf("disks[1] = %+v, want /nfs marked stale", disks[1]) + } + if disks[1].UsedGB != 900 || disks[1].TotalGB != 1000 { + t.Fatalf("disks[1] = %+v, want the last good numbers preserved", disks[1]) + } + + // A permanently stuck mount must not accumulate a goroutine per sample. + for range 5 { + f.clock.advance(time.Minute) + f.sampleAndSettle(t, 1) // only the scratch probe can complete + } + f.sampler.diskMu.Lock() + entry := f.sampler.disks["/nfs"] + inFlight := entry.inFlight + startedAt := entry.startedAt + f.sampler.diskMu.Unlock() + if !inFlight { + t.Fatal("stuck mount is not marked in flight") + } + // startedAt still points at the sample that launched the parked probe, which + // is only true if no later sample launched another one. + if !startedAt.Equal(wedgedAt) { + t.Fatalf("stuck mount was re-probed: startedAt = %v, want %v", startedAt, wedgedAt) + } +} + +// A media root that exists on other nodes but not this one is reported rather +// than hidden: an operator needs to see the gap, and zeros would read as an +// empty disk. +func TestDiskStatsReportsUnseenPathAsUnavailable(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media-on-another-node") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + + f.sampleAndSettle(t, 2) + disks := f.sampleAndSettle(t, 2) + + if len(disks) != 2 { + t.Fatalf("disks = %+v, want both paths reported", disks) + } + if !disks[1].Unavailable { + t.Fatalf("disks[1] = %+v, want Unavailable", disks[1]) + } + if disks[1].UsedGB != 0 || disks[1].TotalGB != 0 { + t.Fatalf("disks[1] = %+v, want no capacity numbers alongside Unavailable", disks[1]) + } +} + +// A deployment with dozens of library roots must not grow the health response +// without bound. +func TestDiskStatsCapsMountCount(t *testing.T) { + paths := make([]string, 0, 12) + for i := range 12 { + paths = append(paths, "/mount"+itoa(i)) + } + f := newDiskFixture(t, paths...) + for i, path := range paths { + f.answer(path, fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "fs:" + itoa(i)}) + } + + f.sampleAndSettle(t, len(paths)) + disks := f.sampleAndSettle(t, len(paths)) + + if len(disks) != maxSampledDisks { + t.Fatalf("len(disks) = %d, want the cap of %d", len(disks), maxSampledDisks) + } +} + +// The cap exists so a health response does not grow with the library count, and +// a path this host cannot measure costs an entry just like a measured one. An +// API host whose library roots live on the nodes reports nothing but +// unavailable entries. +func TestDiskStatsCapsUnavailableMountsToo(t *testing.T) { + paths := make([]string, 0, 20) + for i := range 20 { + paths = append(paths, "/mount"+itoa(i)) + } + f := newDiskFixture(t, paths...) + // Only the scratch dir can be measured; every library root fails statfs. + f.answer(paths[0], fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + + f.sampleAndSettle(t, len(paths)) + disks := f.sampleAndSettle(t, len(paths)) + + if len(disks) != maxSampledDisks { + t.Fatalf("len(disks) = %d, want the cap of %d even when the mounts are unavailable", len(disks), maxSampledDisks) + } +} + +// Several filesystems leave statfs's f_fsid zero — FUSE has no fsid in its +// protocol at all, which covers rclone, mergerfs and s3fs. Treating that as an +// identity collapses unrelated media roots onto one entry, and the mount that +// disappears is a real volume with real capacity nobody is watching any more. +func TestDiskStatsDoesNotDeduplicateMountsWithoutAnFSID(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/rclone-movies", "/mnt/rclone-tv") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + // What osStatfs reports for a FUSE mount: no usable filesystem id. + f.answer("/mnt/rclone-movies", fsStats{UsedBytes: 1 << 30, TotalBytes: 100 << 30}) + f.answer("/mnt/rclone-tv", fsStats{UsedBytes: 98 << 30, TotalBytes: 100 << 30}) + + f.sampleAndSettle(t, 3) + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 3 { + t.Fatalf("disks = %+v, want all three mounts reported", disks) + } + if disks[2].Path != "/mnt/rclone-tv" || disks[2].UsedGB != 98 { + t.Fatalf("disks[2] = %+v, want the nearly full second FUSE mount kept", disks[2]) + } +} + +// The dedup itself must still work for filesystems that do publish an id. +func TestFormatFSIDDropsAZeroIdentity(t *testing.T) { + if got := formatFSID(0, 0); got != "" { + t.Fatalf("formatFSID(0, 0) = %q, want no identity", got) + } + if got := formatFSID(0x1a, 0x2b); got != "1a:2b" { + t.Fatalf("formatFSID(0x1a, 0x2b) = %q, want 1a:2b", got) + } +} + +// Library roots change over a server's life. Entries for paths no longer +// configured must be forgotten rather than accumulate for months. +func TestDiskStatsForgetsPathsNoLongerConfigured(t *testing.T) { + roots := []string{"/media-a"} + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + clock := newFakeClock() + done := make(chan string, 16) + + s := newTestSampler(t, tree, clock, Options{ + ScratchDir: "/transcode", + MediaRoots: func(context.Context) []string { return roots }, + }) + s.diskProbeDone = done + s.statfs = func(path string) (fsStats, error) { + return fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: path}, nil + } + + s.sample(context.Background()) + for range 2 { + <-done + } + if disks := s.Snapshot().System.Disks; len(disks) != 2 { + t.Fatalf("disks = %+v, want both paths tracked", disks) + } + + // The library moves to a different root. + roots = []string{"/media-b"} + s.sample(context.Background()) + for range 2 { + <-done + } + s.sample(context.Background()) + + s.diskMu.Lock() + _, stillTracked := s.disks["/media-a"] + order := append([]string(nil), s.diskOrder...) + s.diskMu.Unlock() + if stillTracked { + t.Fatalf("the removed root is still tracked: %v", order) + } + if len(order) != 2 { + t.Fatalf("diskOrder = %v, want only the two current paths", order) + } +} + +func TestOSStatfsOnUnsupportedPlatformIsNotFatal(t *testing.T) { + // Guards the non-Linux build path: the fallback must return an error rather + // than panic, since NewSampler installs it unconditionally. + if _, err := osStatfs("/definitely/not/a/path"); err == nil { + t.Fatal("statfs on a missing path returned no error") + } else if errors.Is(err, errors.ErrUnsupported) { + t.Log("platform has no statfs; sampler correctly reports paths unavailable") + } +} diff --git a/internal/nodemetrics/fdinfo.go b/internal/nodemetrics/fdinfo.go new file mode 100644 index 000000000..a49e13a66 --- /dev/null +++ b/internal/nodemetrics/fdinfo.go @@ -0,0 +1,290 @@ +package nodemetrics + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// DRM fdinfo baseline. +// +// Every process holding a DRM device publishes per-engine nanosecond counters +// under /proc//fdinfo/. This needs no capabilities, no extra binaries +// and no driver-specific library, and it measures exactly the thing worth +// measuring on a transcode node: how much GPU engine time *our* ffmpeg children +// are consuming, per device. Its one blind spot is other tenants — a GPU shared +// with something outside this process looks idle here — which is what the +// nvidia-smi enrichment exists to cover where it can. +// +// Field names differ by driver (i915, xe, amdgpu), so engines are classified by +// name shape rather than matched against a fixed per-driver table: a driver we +// have never seen still reports usefully as long as it follows the DRM fdinfo +// convention. + +// engineClass distinguishes the two engine groups worth separating for a media +// workload: fixed-function video (encode/decode) and everything general-purpose. +type engineClass int + +const ( + engineOther engineClass = iota + engineVideo + engineRender +) + +const ( + fdinfoEnginePrefix = "drm-engine-" + fdinfoPdevKey = "drm-pdev" + fdinfoClientIDKey = "drm-client-id" +) + +// classifyEngine maps a drm-engine-* field name to its class. +// +// - i915 reports render / video / video-enhance / copy +// - xe reports rcs / vcs / vecs / bcs / ccs +// - amdgpu reports gfx / compute / enc / dec / jpeg / vcn +func classifyEngine(field string) engineClass { + name, ok := strings.CutPrefix(field, fdinfoEnginePrefix) + if !ok { + return engineOther + } + name = strings.ToLower(name) + switch { + case strings.HasPrefix(name, "video"), strings.HasPrefix(name, "enc"), + strings.HasPrefix(name, "dec"), strings.HasPrefix(name, "vcn"), + strings.HasPrefix(name, "jpeg"), strings.HasPrefix(name, "vcs"), + strings.HasPrefix(name, "vecs"): + return engineVideo + case strings.HasPrefix(name, "render"), strings.HasPrefix(name, "gfx"), + strings.HasPrefix(name, "compute"), strings.HasPrefix(name, "rcs"), + strings.HasPrefix(name, "ccs"): + return engineRender + default: + // Copy/blitter engines and anything unrecognized: real work, but not + // work a video pipeline's headroom is judged by. + return engineOther + } +} + +// engineCounters is cumulative engine time for one device. +type engineCounters struct { + videoNS uint64 + renderNS uint64 +} + +// fdinfoClient is one DRM client (a drm-client-id on a pdev). A process holds +// the same client on several fds — ffmpeg dups its device fd across filter and +// encoder contexts — and each fd reports the client's full counters, so +// counting per fd would multiply one GPU's busyness by its fd count. +type fdinfoClient struct { + pdev string + clientID string +} + +// readFdinfoCounters reads cumulative engine time per DRM client across the +// given processes, deduplicating the fds one client is held on. +// +// Counters stay per client rather than being summed per device here, because +// only a client's own counter is monotone: a device total falls whenever one of +// several clients exits, and a caller diffing that total would read the drop as +// negative work for every transcode still running on the card. Summing is the +// caller's job, after it has taken per-client deltas — see deviceEngineDeltas. +func readFdinfoCounters(procDir string, pids []int) map[fdinfoClient]engineCounters { + clients := make(map[fdinfoClient]engineCounters) + for _, pid := range pids { + dir := filepath.Join(procDir, strconv.Itoa(pid), "fdinfo") + entries, err := os.ReadDir(dir) + if err != nil { + // A transcode that exited between listing and reading is the normal + // case, not an error worth reporting. + continue + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + continue + } + client, counters, ok := parseFdinfo(pid, string(raw)) + if !ok { + continue + } + // Highest wins: two fds on one client report the same counters, and + // a read that raced an update reports slightly less. + existing := clients[client] + if counters.videoNS > existing.videoNS { + existing.videoNS = counters.videoNS + } + if counters.renderNS > existing.renderNS { + existing.renderNS = counters.renderNS + } + clients[client] = existing + } + } + return clients +} + +// deviceEngineDeltas converts two per-client readings into the engine time each +// device accumulated between them. +// +// A client absent from the previous reading contributes its whole counter: the +// only clients read here are ffmpeg children this process spawned, so a client +// that was not there last pass started during this interval and every +// nanosecond on its counter was earned inside it. +// +// A client that vanished contributes nothing, and — the point of doing this per +// client — costs the device nothing either. The device sum drops when a +// transcode exits, so diffing sums would report zero busy for the whole card +// while its surviving transcodes ran flat out. +func deviceEngineDeltas(previous, current map[fdinfoClient]engineCounters) map[string]engineCounters { + deltas := make(map[string]engineCounters, len(current)) + for client, counters := range current { + before := previous[client] + delta := deltas[client.pdev] + // Counters are monotone per client, so a fall is a driver reset or a + // reused client id, not negative work. + if counters.videoNS > before.videoNS { + delta.videoNS += counters.videoNS - before.videoNS + } + if counters.renderNS > before.renderNS { + delta.renderNS += counters.renderNS - before.renderNS + } + deltas[client.pdev] = delta + } + return deltas +} + +// parseFdinfo reads one /proc//fdinfo/ file. Only DRM fds carry +// drm-pdev; everything else (sockets, files, the transcode output itself) is +// rejected by the missing key. +func parseFdinfo(pid int, content string) (fdinfoClient, engineCounters, bool) { + var client fdinfoClient + var counters engineCounters + for line := range strings.Lines(content) { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case fdinfoPdevKey: + client.pdev = NormalizePCIAddress(value) + case fdinfoClientIDKey: + client.clientID = value + default: + class := classifyEngine(key) + if class == engineOther { + continue + } + // "12345678 ns" — the unit column is part of the convention. + fields := strings.Fields(value) + if len(fields) == 0 { + continue + } + ns, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + if class == engineVideo { + counters.videoNS += ns + } else { + counters.renderNS += ns + } + } + } + if client.pdev == "" { + return fdinfoClient{}, engineCounters{}, false + } + if client.clientID == "" { + // A driver that publishes no drm-client-id leaves one bucket per process + // per device, merged by highest counter. Keying by fd instead would + // multiply a GPU's busyness by the fd count (ffmpeg dups its device fd), + // and keying by the counter values would mint a new identity every time + // the client did work — which is precisely the identity the interval + // deltas have to follow. Two genuinely distinct anonymous clients in one + // process collapse to the busier of the two; that undercounts, which is + // the safe direction and only affects drivers old enough to omit the id. + client.clientID = "anon:pid:" + strconv.Itoa(pid) + } + return client, counters, true +} + +// engineBusyPercent converts an engine-time delta into a busy percentage over +// the elapsed wall time. +// +// The delta is already per-device work done in the interval (deviceEngineDeltas +// takes it per client, so client churn never produces a negative one). It can +// exceed the interval on a device with several engines of one class running +// concurrently, which clamps to 100 rather than reporting an impossible figure. +func engineBusyPercent(deltaNS uint64, elapsedNS int64) int { + if elapsedNS <= 0 || deltaNS == 0 { + return 0 + } + return clampPercent(int(deltaNS * 100 / uint64(elapsedNS))) +} + +// defaultFFmpegChildren returns the pids of this process's direct children that +// are ffmpeg. +// +// Direct children only, deliberately: those are the processes this node +// spawned and is accountable for. Walking the whole process table would pick up +// every other tenant's encoder on a shared host and report their GPU time as +// ours. +func defaultFFmpegChildren(procDir string, pid int) []int { + taskDir := filepath.Join(procDir, strconv.Itoa(pid), "task") + tasks, err := os.ReadDir(taskDir) + if err != nil { + return nil + } + seen := make(map[int]bool) + var pids []int + for _, task := range tasks { + raw, err := os.ReadFile(filepath.Join(taskDir, task.Name(), "children")) + if err != nil { + continue + } + for _, field := range strings.Fields(string(raw)) { + child, err := strconv.Atoi(field) + if err != nil || seen[child] { + continue + } + seen[child] = true + if !processIsFFmpeg(procDir, child) { + continue + } + pids = append(pids, child) + } + } + return pids +} + +// processIsFFmpeg reports whether a pid's comm names an ffmpeg binary. comm is +// truncated to 15 bytes by the kernel, so this is a substring test rather than +// an equality one. +func processIsFFmpeg(procDir string, pid int) bool { + raw, err := os.ReadFile(filepath.Join(procDir, strconv.Itoa(pid), "comm")) + if err != nil { + return false + } + return strings.Contains(strings.ToLower(strings.TrimSpace(string(raw))), "ffmpeg") +} + +// NormalizePCIAddress makes PCI addresses from different sources comparable. +// DRM fdinfo prints a 16-bit domain (0000:03:00.0), nvidia-smi a 32-bit one +// (00000000:03:00.0), and neither guarantees a case. +func NormalizePCIAddress(address string) string { + address = strings.ToLower(strings.TrimSpace(address)) + domain, rest, ok := strings.Cut(address, ":") + if !ok { + return address + } + value, err := strconv.ParseUint(domain, 16, 64) + if err != nil { + return address + } + return fmt.Sprintf("%04x:%s", value, rest) +} diff --git a/internal/nodemetrics/fdinfo_test.go b/internal/nodemetrics/fdinfo_test.go new file mode 100644 index 000000000..639241a85 --- /dev/null +++ b/internal/nodemetrics/fdinfo_test.go @@ -0,0 +1,339 @@ +package nodemetrics + +import ( + "context" + "testing" + "time" +) + +// i915 names its engines render/video/video-enhance and reports one fd per +// context; ffmpeg dups the device fd, so the same drm-client-id shows up more +// than once with identical counters. +const i915Fdinfo = `pos: 0 +flags: 02100002 +mnt_id: 26 +drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-render: 1000000000 ns +drm-engine-copy: 500000000 ns +drm-engine-video: 2000000000 ns +drm-engine-video-enhance: 0 ns +` + +// amdgpu names them gfx/enc/dec and prints a 32-bit-looking domain. +const amdgpuFdinfo = `pos: 0 +drm-driver: amdgpu +drm-pdev: 0000:03:00.0 +drm-client-id: 7 +drm-engine-gfx: 400000000 ns +drm-engine-enc0: 800000000 ns +drm-engine-dec0: 100000000 ns +` + +// A non-DRM fd (a segment being written) must be ignored entirely. +const plainFdinfo = `pos: 4096 +flags: 02100002 +mnt_id: 26 +` + +func TestReadFdinfoCountersDeduplicatesClientsAndClassifiesEngines(t *testing.T) { + tree := newProcTree(t) + // Two fds on one i915 client, plus an unrelated file fd. + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4242/fdinfo/7", i915Fdinfo) + tree.write("4242/fdinfo/9", plainFdinfo) + // A second process on a different card. + tree.write("4243/fdinfo/3", amdgpuFdinfo) + + clients := readFdinfoCounters(tree.root, []int{4242, 4243, 9999}) + if len(clients) != 2 { + t.Fatalf("clients = %v, want one client per card", clients) + } + + intel := clients[fdinfoClient{pdev: "0000:00:02.0", clientID: "42"}] + // Counted once despite two fds; the copy engine is excluded. + if intel.videoNS != 2_000_000_000 || intel.renderNS != 1_000_000_000 { + t.Fatalf("i915 counters = %+v, want video 2e9 render 1e9 counted once", intel) + } + + amd := clients[fdinfoClient{pdev: "0000:03:00.0", clientID: "7"}] + if amd.renderNS != 400_000_000 { + t.Fatalf("amdgpu render = %d, want gfx classified as render", amd.renderNS) + } + if amd.videoNS != 900_000_000 { + t.Fatalf("amdgpu video = %d, want enc0+dec0 classified as video", amd.videoNS) + } + + // Summing is the caller's job, and only after per-client deltas: these + // counters are cumulative per client, so a device total is not diffable. + deltas := deviceEngineDeltas(nil, clients) + if got := deltas["0000:00:02.0"].videoNS; got != 2_000_000_000 { + t.Fatalf("first-reading delta = %d, want the client's whole counter", got) + } +} + +// A driver that publishes no drm-client-id must still produce one stable +// identity per process and device, or the fd dups would multiply its busyness +// and the identity would change every time the client did work. +func TestReadFdinfoCountersKeysAnonymousClientsByProcess(t *testing.T) { + anon := `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-engine-video: 2000000000 ns +` + tree := newProcTree(t) + tree.write("4242/fdinfo/3", anon) + tree.write("4242/fdinfo/7", anon) + + clients := readFdinfoCounters(tree.root, []int{4242}) + if len(clients) != 1 { + t.Fatalf("clients = %v, want the two fds collapsed onto one client", clients) + } + for client, counters := range clients { + if client.clientID != "anon:pid:4242" { + t.Fatalf("clientID = %q, want an identity that survives the client doing work", client.clientID) + } + if counters.videoNS != 2_000_000_000 { + t.Fatalf("videoNS = %d, want the dup counted once", counters.videoNS) + } + } +} + +// PCI addresses arrive with different domain widths and cases depending on the +// source; they have to collapse to one key or a device is counted twice. +func TestNormalizePCIAddress(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"0000:00:02.0", "0000:00:02.0"}, + {"00000000:03:00.0", "0000:03:00.0"}, + {"0000:03:00.0", "0000:03:00.0"}, + {" 00000000:0A:00.0 ", "0000:0a:00.0"}, + {"not-an-address", "not-an-address"}, + } { + if got := NormalizePCIAddress(tc.in); got != tc.want { + t.Fatalf("NormalizePCIAddress(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestSampleGPUMapsPdevToDevicePathAndComputesBusy(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", i915Fdinfo) + + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return []int{4242} }, + DeviceSessions: func() map[string]int { return map[string]int{"/dev/dri/renderD128": 2} }, + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{ + // nvidia-smi-style wide domain, to prove normalization is what + // joins the two views rather than string equality. + Path: "/dev/dri/renderD128", + PCIAddress: "00000000:00:02.0", + Vendor: "intel", + }} + }, + }) + + s.sample(context.Background()) + first := s.Snapshot().GPU + if len(first) != 1 { + t.Fatalf("GPU = %+v, want one device", first) + } + if first[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path, not the PCI address", first[0].Device) + } + if first[0].Vendor != "intel" || first[0].Sessions != 2 { + t.Fatalf("GPU[0] = %+v, want vendor intel and 2 sessions", first[0]) + } + if first[0].Source != SourceFdinfo { + t.Fatalf("Source = %q, want %q", first[0].Source, SourceFdinfo) + } + if first[0].VideoBusyPct != 0 { + t.Fatalf("VideoBusyPct = %d on the first sample, want 0 (nothing to diff against)", first[0].VideoBusyPct) + } + + // Over 4s of wall time: +2s of video engine, +1s of render engine. + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-render: 2000000000 ns +drm-engine-video: 4000000000 ns +`) + clock.advance(4 * time.Second) + s.sample(context.Background()) + + second := s.Snapshot().GPU[0] + if second.VideoBusyPct != 50 { + t.Fatalf("VideoBusyPct = %d, want 50", second.VideoBusyPct) + } + if second.RenderBusyPct != 25 { + t.Fatalf("RenderBusyPct = %d, want 25", second.RenderBusyPct) + } + if second.TotalBusyPct != nil { + t.Fatal("TotalBusyPct set without an enrichment source") + } +} + +// One transcode exiting must not erase the work the others did in that +// interval. Their engine time is what the whole GPU panel is read from, and a +// node with normal session churn loses a client inside most intervals. +func TestSampleGPUKeepsSurvivingTranscodeBusyWhenAPeerExits(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // Two transcodes on one card: the survivor at 2s of video engine time, the + // one about to exit at 5s. + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4243/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 43 +drm-engine-video: 5000000000 ns +`) + + pids := []int{4242, 4243} + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return pids }, + }) + s.sample(context.Background()) + + // The peer exits while the survivor saturates the video engine: +5s of + // engine time over a 5s interval. The device total is 7s before and 7s + // after, so a per-device baseline sees no gain and reports an idle GPU. + pids = []int{4242} + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-video: 7000000000 ns +`) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + if got := s.Snapshot().GPU[0].VideoBusyPct; got != 100 { + t.Fatalf("VideoBusyPct = %d while the surviving transcode ran the engine flat out, want 100", got) + } +} + +// A transcode exiting takes its accumulated engine time out of the device +// total, so the per-device sum falls. That is bookkeeping, not negative work. +func TestSampleGPUClampsNegativeDeltaWhenATranscodeExits(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4243/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 43 +drm-engine-video: 5000000000 ns +`) + + pids := []int{4242, 4243} + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return pids }, + }) + s.sample(context.Background()) + + // The second transcode exits; only the first process remains. + pids = []int{4242} + clock.advance(5 * time.Second) + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if gpu[0].VideoBusyPct != 0 || gpu[0].RenderBusyPct != 0 { + t.Fatalf("busy = %d/%d after a transcode exited, want 0/0", gpu[0].VideoBusyPct, gpu[0].RenderBusyPct) + } + + // The reduced total is the new baseline: the surviving transcode's next + // interval is measured against it, not against the pre-exit sum. + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-video: 3000000000 ns +`) + clock.advance(10 * time.Second) + s.sample(context.Background()) + if got := s.Snapshot().GPU[0].VideoBusyPct; got != 10 { + t.Fatalf("VideoBusyPct after re-baselining = %d, want 10", got) + } +} + +// A device with no DRM counters and no enrichment still has to appear, so an +// operator sees the GPU exists — but it must say so rather than report zeros as +// a measurement. +func TestSampleGPUReportsKnownDeviceAsUnavailable(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:00:02.0", Vendor: "intel"}} + }, + }) + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 || gpu[0].Source != SourceUnavailable { + t.Fatalf("GPU = %+v, want one device sourced %q", gpu, SourceUnavailable) + } +} + +func TestDefaultFFmpegChildrenMatchesOnlyFFmpegChildren(t *testing.T) { + tree := newProcTree(t) + tree.write("100/task/100/children", "200 300\n") + tree.write("100/task/105/children", "400\n") + tree.write("200/comm", "ffmpeg\n") + // The kernel truncates comm to 15 bytes, which is why this is a substring + // test and not an equality one. + tree.write("300/comm", "ffmpeg-static-b\n") + tree.write("400/comm", "postgres\n") + + pids := defaultFFmpegChildren(tree.root, 100) + if len(pids) != 2 { + t.Fatalf("pids = %v, want the two ffmpeg children", pids) + } + for _, pid := range pids { + if pid == 400 { + t.Fatalf("pids = %v, want the non-ffmpeg child excluded", pids) + } + } +} + +func TestEngineBusyPercentClamps(t *testing.T) { + elapsed := (5 * time.Second).Nanoseconds() + if got := engineBusyPercent(10*uint64(elapsed), elapsed); got != 100 { + t.Fatalf("busy over 100%% = %d, want clamped to 100", got) + } + if got := engineBusyPercent(0, elapsed); got != 0 { + t.Fatalf("busy with no work = %d, want 0", got) + } + if got := engineBusyPercent(100, 0); got != 0 { + t.Fatalf("busy with no elapsed time = %d, want 0", got) + } +} + +// Counters are only monotone per client. A client whose counter fell (a driver +// reset, or a reused client id) must contribute nothing rather than wrap. +func TestDeviceEngineDeltasIgnoresCounterRegressions(t *testing.T) { + client := fdinfoClient{pdev: "0000:00:02.0", clientID: "42"} + previous := map[fdinfoClient]engineCounters{client: {videoNS: 5_000_000_000}} + current := map[fdinfoClient]engineCounters{client: {videoNS: 1_000_000_000}} + if got := deviceEngineDeltas(previous, current)["0000:00:02.0"].videoNS; got != 0 { + t.Fatalf("delta on a counter regression = %d, want 0", got) + } +} diff --git a/internal/nodemetrics/meminfo.go b/internal/nodemetrics/meminfo.go new file mode 100644 index 000000000..bb8795819 --- /dev/null +++ b/internal/nodemetrics/meminfo.go @@ -0,0 +1,181 @@ +package nodemetrics + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +const bytesPerKB = int64(1024) + +// cgroupInactiveFileKeyV2 is the cgroup v2 memory.stat key for reclaimable +// page cache. cgroup v1's memory.stat spells the same figure +// "total_inactive_file". +const cgroupInactiveFileKeyV2 = "inactive_file" + +// CgroupMemoryLimitPaths returns the memory-limit files to consult, cgroup v2 +// first. A container that has a limit publishes it in exactly one of these; a +// host has neither. +// +// It returns a fresh slice per call so a caller iterating it cannot reorder the +// preference for everyone else. +func CgroupMemoryLimitPaths() []string { + return []string{ + "/sys/fs/cgroup/memory.max", // cgroup v2 + "/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1 + } +} + +// cgroupUsagePath pairs one cgroup version's current-usage file with the stat +// file and key that names its page cache. +type cgroupUsagePath struct { + usage string + stat string + inactiveFile string +} + +// cgroupMemoryUsagePaths lists where to read current memory charge, v2 first. +// +// Usage counts reclaimable page cache, so reporting it raw would show a node +// that merely read a large file as nearly out of memory; subtracting inactive +// file pages yields the working set, which is the number that actually predicts +// an OOM kill (and is what `docker stats` reports). +var cgroupMemoryUsagePaths = []cgroupUsagePath{ + { + usage: "/sys/fs/cgroup/memory.current", + stat: "/sys/fs/cgroup/memory.stat", + inactiveFile: cgroupInactiveFileKeyV2, + }, + { + usage: "/sys/fs/cgroup/memory/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/memory.stat", + inactiveFile: "total_inactive_file", + }, +} + +// ReadMeminfoTotalBytes returns MemTotal from a /proc/meminfo-formatted file. +// +// This lives here rather than beside its first caller because two unrelated +// subsystems need the same answer — Postgres auto-tuning sizes shared_buffers +// from it, and node metrics report it — and a host's memory is one fact, not +// two implementations of one fact. +func ReadMeminfoTotalBytes(path string) (int64, error) { + fields, err := ReadMeminfoBytes(path) + if err != nil { + return 0, err + } + total, ok := fields["MemTotal"] + if !ok { + return 0, errors.New("MemTotal not found") + } + return total, nil +} + +// ReadMeminfoBytes parses a /proc/meminfo-formatted file into bytes per key. +// Keys are returned without the trailing colon. Lines that do not parse are +// skipped rather than failing the read: meminfo grows new keys across kernel +// versions, and one unfamiliar line must not cost the caller every familiar +// one. +func ReadMeminfoBytes(path string) (map[string]int64, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + values := make(map[string]int64) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + key, value, ok := parseMeminfoLine(scanner.Text()) + if !ok { + continue + } + values[key] = value + } + if err := scanner.Err(); err != nil { + return nil, err + } + return values, nil +} + +// parseMeminfoLine reads one "Key: kB" line, converting to bytes. The +// unit column is optional (a handful of meminfo entries are bare counts). +func parseMeminfoLine(line string) (string, int64, bool) { + key, rest, ok := strings.Cut(line, ":") + if !ok { + return "", 0, false + } + fields := strings.Fields(rest) + if len(fields) == 0 { + return "", 0, false + } + value, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return "", 0, false + } + if len(fields) > 1 && strings.EqualFold(fields[1], "kB") { + value *= bytesPerKB + } + return strings.TrimSpace(key), value, true +} + +// ReadCgroupMemoryLimit reads a cgroup memory-limit file and returns the limit +// in bytes, or an error when the cgroup imposes none. +// +// "No limit" is spelled three different ways depending on kernel and runtime — +// the literal "max", an empty file, or a saturated sentinel close to 2^63 — and +// all three must read as "ask the host instead", never as an enormous budget. +func ReadCgroupMemoryLimit(path string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value := strings.TrimSpace(string(raw)) + if value == "" || value == "max" { + return 0, fmt.Errorf("no cgroup memory limit") + } + mem, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, err + } + // Docker may expose a huge sentinel when no concrete memory limit is set. + if mem <= 0 || mem > 1<<60 { + return 0, fmt.Errorf("no concrete cgroup memory limit") + } + return mem, nil +} + +// readCgroupSingleValue reads a file holding one integer. +func readCgroupSingleValue(path string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value, err := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64) + if err != nil { + return 0, err + } + if value < 0 { + return 0, fmt.Errorf("negative cgroup value") + } + return value, nil +} + +// readCgroupStatKey reads one key out of a "key value" cgroup stat file. +func readCgroupStatKey(path, key string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + for line := range strings.Lines(string(raw)) { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != key { + continue + } + return strconv.ParseInt(fields[1], 10, 64) + } + return 0, fmt.Errorf("%s not found in %s", key, path) +} diff --git a/internal/nodemetrics/nvidia.go b/internal/nodemetrics/nvidia.go new file mode 100644 index 000000000..139dff33e --- /dev/null +++ b/internal/nodemetrics/nvidia.go @@ -0,0 +1,160 @@ +package nodemetrics + +import ( + "context" + "errors" + "log/slog" + "os/exec" + "strconv" + "strings" + "sync" + "time" +) + +// nvidiaSMITimeout bounds one query. A wedged driver makes nvidia-smi hang +// indefinitely, and a metrics sample must never inherit that. +const nvidiaSMITimeout = 3 * time.Second + +// sourceFailureLimit is how many consecutive failures retire an enrichment +// source for the life of the process. +// +// A source that has failed five times running is not having a bad moment: the +// binary is missing its driver, the container lacks the device, or the query +// syntax is not supported by the installed version. Retrying it forever would +// spawn a doomed subprocess every 5 seconds on every node, which costs more +// than the signal is worth. Recovery is a node restart, which is also what +// installing or fixing the toolkit requires. +const sourceFailureLimit = 5 + +// nvidiaSMIFields is the query column order. utilization.gpu is whole-GPU +// busyness including other tenants; the encoder/decoder columns are the +// fixed-function video engines, which is what a transcode node is actually +// competing for. +const nvidiaSMIFields = "index,uuid,pci.bus_id,utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total" + +// runNVIDIASMI is the execution seam. Tests replace it instead of installing a +// fake binary on PATH. +var runNVIDIASMI = func(ctx context.Context) ([]byte, error) { + path, err := exec.LookPath("nvidia-smi") + if err != nil { + return nil, err + } + return exec.CommandContext(ctx, path, + "--query-gpu="+nvidiaSMIFields, + "--format=csv,noheader,nounits").Output() +} + +// nvidiaGPU is one parsed nvidia-smi row. +type nvidiaGPU struct { + Index int + UUID string + PCIAddress string + GPUUtil int + EncoderUtil int + DecoderUtil int + MemUsedMB int64 + MemTotalMB int64 +} + +// sourceBreaker retires an enrichment source after repeated failure. +type sourceBreaker struct { + name string + failures int + tripped bool + logOnce sync.Once +} + +// allow reports whether the source may be queried. +func (b *sourceBreaker) allow() bool { return !b.tripped } + +func (b *sourceBreaker) succeeded() { b.failures = 0 } + +// failed records one failure and trips the breaker at the limit, logging the +// retirement exactly once so a node without the toolkit does not narrate it +// every interval. +func (b *sourceBreaker) failed(err error) { + if b.tripped { + return + } + b.failures++ + if b.failures < sourceFailureLimit { + return + } + b.tripped = true + b.logOnce.Do(func() { + slog.Info("node metrics source unavailable; not retrying until restart", + "component", "nodemetrics", "source", b.name, "failures", b.failures, "error", err) + }) +} + +// queryNVIDIA runs one bounded nvidia-smi query, honoring the breaker. +func (s *Sampler) queryNVIDIA(ctx context.Context) []nvidiaGPU { + if !s.nvidiaBreaker.allow() { + return nil + } + queryCtx, cancel := context.WithTimeout(ctx, nvidiaSMITimeout) + defer cancel() + output, err := s.runNVIDIASMI(queryCtx) + if err != nil { + s.nvidiaBreaker.failed(err) + return nil + } + gpus := parseNVIDIASMI(output) + if len(gpus) == 0 { + // A successful command that says nothing is as useless as a failure and + // is how a stale query syntax presents. + s.nvidiaBreaker.failed(errNoNVIDIARows) + return nil + } + s.nvidiaBreaker.succeeded() + return gpus +} + +// errNoNVIDIARows marks a query that succeeded but produced nothing parseable. +var errNoNVIDIARows = errors.New("nvidia-smi returned no parseable rows") + +// parseNVIDIASMI reads "csv,noheader,nounits" rows. Malformed rows are skipped +// individually; a driver that reports "[N/A]" for one column on one GPU must +// not cost the reading for the others. +func parseNVIDIASMI(output []byte) []nvidiaGPU { + var gpus []nvidiaGPU + for line := range strings.Lines(string(output)) { + fields := strings.Split(line, ",") + if len(fields) < 8 { + continue + } + index, err := strconv.Atoi(strings.TrimSpace(fields[0])) + if err != nil { + continue + } + address := NormalizePCIAddress(fields[2]) + uuid := strings.TrimSpace(fields[1]) + if address == "" && uuid == "" { + continue + } + gpus = append(gpus, nvidiaGPU{ + Index: index, + UUID: uuid, + PCIAddress: address, + GPUUtil: parseNVIDIAInt(fields[3]), + EncoderUtil: parseNVIDIAInt(fields[4]), + DecoderUtil: parseNVIDIAInt(fields[5]), + MemUsedMB: int64(parseNVIDIAInt(fields[6])), + MemTotalMB: int64(parseNVIDIAInt(fields[7])), + }) + } + return gpus +} + +// parseNVIDIAInt reads one numeric column, treating the driver's "[N/A]" and +// "[Not Supported]" placeholders as zero. +func parseNVIDIAInt(field string) int { + value, err := strconv.Atoi(strings.TrimSpace(field)) + if err != nil { + return 0 + } + if value < 0 { + return 0 + } + return value +} diff --git a/internal/nodemetrics/nvidia_test.go b/internal/nodemetrics/nvidia_test.go new file mode 100644 index 000000000..5812e3207 --- /dev/null +++ b/internal/nodemetrics/nvidia_test.go @@ -0,0 +1,276 @@ +package nodemetrics + +import ( + "context" + "errors" + "testing" +) + +const nvidiaSMIOutput = "0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n" + + "1, GPU-5678efgh, 00000000:04:00.0, 5, 0, 0, 100, 8192\n" + +func TestParseNVIDIASMI(t *testing.T) { + gpus := parseNVIDIASMI([]byte(nvidiaSMIOutput)) + if len(gpus) != 2 { + t.Fatalf("gpus = %+v, want 2", gpus) + } + first := gpus[0] + if first.Index != 0 || first.UUID != "GPU-1234abcd" { + t.Fatalf("gpus[0] identity = %+v", first) + } + // The wide domain nvidia-smi prints has to normalize to the sysfs form or it + // will never join with a DRM device. + if first.PCIAddress != "0000:03:00.0" { + t.Fatalf("PCIAddress = %q, want the normalized sysfs form", first.PCIAddress) + } + if first.GPUUtil != 71 || first.EncoderUtil != 63 || first.DecoderUtil != 12 { + t.Fatalf("gpus[0] utilization = %+v", first) + } + if first.MemUsedMB != 812 || first.MemTotalMB != 8192 { + t.Fatalf("gpus[0] memory = %+v", first) + } +} + +// Drivers print "[N/A]" for a column a card does not support. One unsupported +// column must not discard the whole row. +func TestParseNVIDIASMIToleratesPlaceholders(t *testing.T) { + gpus := parseNVIDIASMI([]byte("0, GPU-x, 00000000:03:00.0, [N/A], [Not Supported], 4, 100, 8192\n")) + if len(gpus) != 1 { + t.Fatalf("gpus = %+v, want the row kept", gpus) + } + if gpus[0].GPUUtil != 0 || gpus[0].EncoderUtil != 0 || gpus[0].DecoderUtil != 4 { + t.Fatalf("gpus[0] = %+v, want placeholders read as zero and 4 preserved", gpus[0]) + } +} + +func TestSampleGPUEnrichesWithNVIDIASMI(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return map[string]int{"cuda:0": 3} }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return []byte(nvidiaSMIOutput), nil } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 2 { + t.Fatalf("GPU = %+v, want both cards", gpu) + } + first := gpu[0] + // The proprietary driver exposes no DRM node this process can read, so the + // device is named the way playback addresses it. + if first.Device != "cuda:0" { + t.Fatalf("Device = %q, want cuda:0", first.Device) + } + if first.Vendor != "nvidia" || first.Source != SourceNVIDIASMI { + t.Fatalf("GPU[0] = %+v, want nvidia via nvidia-smi", first) + } + if first.Sessions != 3 { + t.Fatalf("Sessions = %d, want the balancer's count for cuda:0", first.Sessions) + } + if first.TotalBusyPct == nil || *first.TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want 71", first.TotalBusyPct) + } + if first.VideoBusyPct != 63 { + t.Fatalf("VideoBusyPct = %d, want the busier of encoder/decoder", first.VideoBusyPct) + } + if first.VRAMUsedMB == nil || *first.VRAMUsedMB != 812 { + t.Fatalf("VRAMUsedMB = %v, want 812", first.VRAMUsedMB) + } +} + +// A GPU that has both DRM counters and an nvidia-smi row must be one entry +// crediting both sources, not two entries. +func TestSampleGPUMergesFdinfoAndNVIDIASMIOnOneDevice(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", `drm-driver: nvidia-drm +drm-pdev: 0000:03:00.0 +drm-client-id: 1 +drm-engine-video: 1000000000 ns +`) + + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return []int{4242} }, + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", Vendor: "nvidia"}} + }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want the two views merged onto one device", gpu) + } + if gpu[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path kept", gpu[0].Device) + } + if gpu[0].Source != SourceFdinfoNVIDIASMI { + t.Fatalf("Source = %q, want %q", gpu[0].Source, SourceFdinfoNVIDIASMI) + } + if gpu[0].TotalBusyPct == nil || *gpu[0].TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want the whole-GPU reading", gpu[0].TotalBusyPct) + } +} + +// NVENC workloads are counted under a CUDA name or a GPU UUID, but a card whose +// DRM node this process can read is displayed by its render path. Looking +// sessions up by display name alone reports an idle GPU on an NVIDIA node that +// is transcoding. +func TestSampleGPUJoinsNVENCSessionsThroughDeviceAliases(t *testing.T) { + for _, tc := range []struct { + name string + sessions map[string]int + want int + }{ + {name: "counted by cuda index", sessions: map[string]int{"cuda:0": 3}, want: 3}, + {name: "counted by gpu uuid", sessions: map[string]int{"GPU-1234abcd": 2}, want: 2}, + {name: "counted by render path", sessions: map[string]int{"/dev/dri/renderD128": 1}, want: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return tc.sessions }, + DeviceIdentities: func() []DeviceIdentity { + // The proprietary driver with modeset does expose a render + // node, so this is the ordinary bare-metal NVIDIA shape. + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", Vendor: "nvidia"}} + }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if gpu[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path", gpu[0].Device) + } + if gpu[0].Sessions != tc.want { + t.Fatalf("Sessions = %d, want %d for %v", gpu[0].Sessions, tc.want, tc.sessions) + } + }) + } +} + +// Two cards must not both claim a count keyed by a name only one of them +// answers to. +func TestSampleGPUDoesNotDoubleCountSessionsAcrossDevices(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return map[string]int{"cuda:1": 2} }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return []byte(nvidiaSMIOutput), nil } + s.sample(context.Background()) + + total := 0 + for _, gpu := range s.Snapshot().GPU { + total += gpu.Sessions + if gpu.Device == "cuda:0" && gpu.Sessions != 0 { + t.Fatalf("cuda:0 claimed %d sessions belonging to cuda:1", gpu.Sessions) + } + } + if total != 2 { + t.Fatalf("sessions across devices = %d, want the 2 counted exactly once", total) + } +} + +// A host without the NVIDIA toolkit fails this query every 5 seconds forever. +// The breaker stops us from spawning a doomed subprocess for the life of the +// process. +func TestNVIDIACircuitBreakerRetiresSourceAfterRepeatedFailure(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + return nil, errors.New("nvidia-smi: command not found") + } + + for range sourceFailureLimit + 5 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d failures", calls, sourceFailureLimit) + } +} + +// A successful command that parses to nothing is as useless as a failure, and +// is how an unsupported query syntax presents. +func TestNVIDIACircuitBreakerCountsEmptyOutputAsFailure(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + return []byte("\n"), nil + } + for range sourceFailureLimit + 3 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d empty answers", calls, sourceFailureLimit) + } +} + +// A transient failure must not retire the source: a driver busy for one sample +// is normal, and losing the only NVIDIA signal over it would be a regression an +// operator cannot recover from without a restart. +func TestNVIDIACircuitBreakerResetsOnSuccess(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + fail := true + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("busy") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit - 1 { + s.queryNVIDIA(context.Background()) + } + fail = false + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("recovered query returned %d gpus, want 2", len(gpus)) + } + fail = true + for range sourceFailureLimit - 1 { + s.queryNVIDIA(context.Background()) + } + if calls != 2*(sourceFailureLimit-1)+1 { + t.Fatalf("nvidia-smi invoked %d times, want the failure count reset by the success", calls) + } +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go new file mode 100644 index 000000000..758be16e4 --- /dev/null +++ b/internal/nodemetrics/sampler.go @@ -0,0 +1,450 @@ +package nodemetrics + +import ( + "context" + "os" + "runtime" + "slices" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" +) + +// DefaultInterval is how often the sampler takes a reading. Rate-based fields +// (CPU, network, GPU engine busy) are averages over exactly this window, so it +// doubles as the resolution of every derived percentage. Five seconds is short +// enough that an operator watching a transcode start sees it, and long enough +// that the sampling itself is not the load. +const DefaultInterval = 5 * time.Second + +// Options configures a Sampler. Every provider is optional: a nil provider +// means that dimension simply is not reported, which is the correct behavior +// for a proxy node with no scratch dir or a host with no GPU. +type Options struct { + // ScratchDir is the transcode working directory. It is sampled first + // because it is the volume whose filling up silently kills transcodes. + ScratchDir string + // MediaRoots returns library folder paths to sample alongside the scratch + // dir. It is called on the sampling goroutine each pass, so it must be cheap + // and must respect ctx; a nil provider samples the scratch dir only. + MediaRoots func(ctx context.Context) []string + // FFmpegChildren returns the pids whose DRM fdinfo counts as this node's GPU + // work. The default is this process's direct ffmpeg children. + FFmpegChildren func() []int + // DeviceSessions returns active GPU workloads per device, keyed the way the + // playback device balancer keys them (a render node path, or "cuda:N"). + DeviceSessions func() map[string]int + // DeviceIdentities returns this host's render devices, used to translate + // the PCI addresses DRM reports into the /dev/dri paths every other surface + // speaks. + DeviceIdentities func() []DeviceIdentity + // Interval overrides DefaultInterval. + Interval time.Duration + // Now overrides the clock. Tests inject one instead of sleeping. + Now func() time.Time +} + +// Sampler periodically reads host and GPU resource usage into a snapshot. +// +// Exactly one goroutine performs sampling, which is why the per-sample delta +// state below needs no locking; everything a caller can reach concurrently is +// either atomic (the published snapshot) or explicitly guarded (the disk +// entries, which detached probe goroutines also write). +type Sampler struct { + interval time.Duration + now func() time.Time + goos string + scratchDir string + mediaRoots func(ctx context.Context) []string + sessions func() map[string]int + identities func() []DeviceIdentity + ffmpegPIDs func() []int + + // Path seams. Production values point at the real filesystem; tests point + // them at a fake /proc tree. + procDir string + cgroupLimitPaths []string + cgroupUsagePaths []cgroupUsagePath + cgroupCPUPaths []cgroupCPUPath + + snapshot atomic.Pointer[Snapshot] + + // Delta state, owned by the sampling goroutine. + prevCPU cpuTimes + prevNet netCounters + // prevGPU is keyed by DRM client, not by device: only a client's own engine + // counter is monotone, so a per-device baseline would read every client exit + // as negative work for the whole card. + prevGPU map[fdinfoClient]engineCounters + prevGPUAt time.Time + prevCgroupCPU cgroupCPUSample + + // Disk probe state, shared with detached probe goroutines. + diskMu sync.Mutex + disks map[string]*diskEntry + diskOrder []string + statfs func(string) (fsStats, error) + // diskProbeDone, when non-nil, receives each completed probe's path. Tests + // wait on it instead of sleeping; production leaves it nil. + diskProbeDone chan string + + runNVIDIASMI func(ctx context.Context) ([]byte, error) + nvidiaBreaker *sourceBreaker +} + +// NewSampler creates a sampler. It performs no I/O; nothing is read until +// Start. +func NewSampler(opts Options) *Sampler { + interval := opts.Interval + if interval <= 0 { + interval = DefaultInterval + } + now := opts.Now + if now == nil { + now = time.Now + } + procDir := "/proc" + ffmpegPIDs := opts.FFmpegChildren + if ffmpegPIDs == nil { + pid := os.Getpid() + ffmpegPIDs = func() []int { return defaultFFmpegChildren(procDir, pid) } + } + s := &Sampler{ + interval: interval, + now: now, + goos: runtime.GOOS, + scratchDir: opts.ScratchDir, + mediaRoots: opts.MediaRoots, + sessions: opts.DeviceSessions, + identities: opts.DeviceIdentities, + ffmpegPIDs: ffmpegPIDs, + procDir: procDir, + cgroupLimitPaths: CgroupMemoryLimitPaths(), + cgroupUsagePaths: slices.Clone(cgroupMemoryUsagePaths), + cgroupCPUPaths: slices.Clone(cgroupCPUPaths), + prevGPU: map[fdinfoClient]engineCounters{}, + disks: map[string]*diskEntry{}, + statfs: osStatfs, + runNVIDIASMI: runNVIDIASMI, + nvidiaBreaker: &sourceBreaker{name: "nvidia-smi"}, + } + s.snapshot.Store(&Snapshot{}) + return s +} + +// NewFixedSamplerForTest returns a sampler that always answers with the given +// snapshot and never samples anything. +// +// It exists for tests in other packages — the node health and status handlers, +// the admin resources endpoint — that need a known reading without a Linux host +// under them. Nothing in production calls it. +func NewFixedSamplerForTest(snapshot Snapshot) *Sampler { + s := NewSampler(Options{}) + s.snapshot.Store(&snapshot) + return s +} + +// Snapshot returns the most recent sample. It never blocks, never fails, and +// never does I/O — it is safe to call from an HTTP handler on the request path, +// which is the whole reason the sampling loop exists. +func (s *Sampler) Snapshot() Snapshot { + if s == nil { + return Snapshot{} + } + if snapshot := s.snapshot.Load(); snapshot != nil { + return *snapshot + } + return Snapshot{} +} + +// Start samples in the background until ctx is canceled. It returns +// immediately. +// +// On a non-Linux host the loop still runs but does nothing, so callers do not +// need their own platform checks and the published snapshot stays a truthful +// Available=false rather than an absent one. +func (s *Sampler) Start(ctx context.Context) { + if s == nil || ctx == nil { + return + } + registerCollector(s) + go func() { + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + s.sample(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.sample(ctx) + } + } + }() +} + +// sample takes one reading and publishes it. +func (s *Sampler) sample(ctx context.Context) { + if s.goos != "linux" { + s.snapshot.Store(&Snapshot{}) + return + } + now := s.now() + snapshot := Snapshot{ + Available: true, + SampledAt: now, + System: s.sampleSystem(ctx, now), + GPU: s.sampleGPU(ctx, now), + } + s.snapshot.Store(&snapshot) +} + +func (s *Sampler) sampleSystem(ctx context.Context, now time.Time) *SystemStats { + cpuPct, cores := s.cpuStats(now) + + net := readNetCounters(s.procDir, now) + rxBps, txBps, _ := netThroughputBps(s.prevNet, net) + if net.valid { + s.prevNet = net + } + + usedBytes, totalBytes := s.memoryStats() + + paths := s.diskPaths(ctx) + s.refreshDisks(paths, now) + + return &SystemStats{ + CPUPct: cpuPct, + Load1: readLoad1(s.procDir), + Cores: cores, + MemUsedMB: bytesToMB(usedBytes), + MemTotalMB: bytesToMB(totalBytes), + Disks: s.diskStats(paths, now), + NetRxBps: rxBps, + NetTxBps: txBps, + } +} + +// cpuStats reports busy percentage and the core count that percentage is +// normalized against. +// +// Under a cgroup both come from the cgroup, not the host: /proc/stat is +// host-wide even in a container, so a node capped at two cores of a busy +// 64-core machine would otherwise report the machine's load instead of its own +// — and a node pinned at its quota, which is the state worth alerting on, would +// look nearly idle. +func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { + host, hostCores := readCPUTimes(s.procDir) + busyPct, _ = cpuBusyPercent(s.prevCPU, host) + if host.valid { + s.prevCPU = host + } + cores = hostCores + if cores == 0 { + cores = runtime.NumCPU() + } + + sample, quota := s.cgroupCPU(now) + if quota > 0 { + cores = cgroupQuotaCores(quota, hostCores) + } + if !sample.valid { + return busyPct, cores + } + budget := quota + if budget <= 0 { + budget = float64(cores) + } + // Once the cgroup can be read it is the only honest source, so its answer + // stands even when this pass cannot derive one (the first sample, or a + // counter reset). Falling back to the host figure would silently mix two + // different machines' busyness across intervals. + cgroupPct, _ := cgroupCPUPercent(s.prevCgroupCPU, sample, budget) + s.prevCgroupCPU = sample + return cgroupPct, cores +} + +// diskPaths lists the mounts to sample, scratch dir first. +func (s *Sampler) diskPaths(ctx context.Context) []string { + var paths []string + if s.scratchDir != "" { + paths = append(paths, s.scratchDir) + } + if s.mediaRoots != nil { + for _, root := range s.mediaRoots(ctx) { + if root != "" && !slices.Contains(paths, root) { + paths = append(paths, root) + } + } + } + return paths +} + +// sampleGPU merges three independent views of the host's GPUs: the hardware +// inventory (which devices exist), DRM fdinfo (what our own transcodes are +// doing on them), and nvidia-smi (what everyone is doing on an NVIDIA GPU). +// +// Devices are keyed by normalized PCI address wherever one is known, because +// that is the only identifier all three views share — /dev/dri paths are an +// enumeration order and CUDA indices are an nvidia-smi ordering. +func (s *Sampler) sampleGPU(ctx context.Context, now time.Time) []GPUStats { + sessions := map[string]int{} + if s.sessions != nil { + sessions = s.sessions() + } + + byKey := map[string]*GPUStats{} + order := []string{} + // aliases holds every name a device answers to, because the workload counter + // is keyed by whatever playback was configured with (a render path, a CUDA + // index, a GPU UUID) while entries here are keyed by PCI address. + aliases := map[string][]string{} + upsert := func(key string) *GPUStats { + if existing, ok := byKey[key]; ok { + return existing + } + entry := &GPUStats{Device: key, Source: SourceUnavailable} + byKey[key] = entry + order = append(order, key) + aliases[key] = append(aliases[key], key) + return entry + } + alias := func(key string, names ...string) { + for _, name := range names { + if name != "" && !slices.Contains(aliases[key], name) { + aliases[key] = append(aliases[key], name) + } + } + } + + // 1. Known hardware, so a device with no activity still appears. + pathByPCI := map[string]string{} + for _, identity := range s.identityList() { + key := NormalizePCIAddress(identity.PCIAddress) + if key == "" { + key = identity.Path + } + if key == "" { + continue + } + if identity.PCIAddress != "" && identity.Path != "" { + pathByPCI[NormalizePCIAddress(identity.PCIAddress)] = identity.Path + } + entry := upsert(key) + if identity.Path != "" { + entry.Device = identity.Path + } + alias(key, identity.Path) + entry.Vendor = identity.Vendor + } + + // 2. DRM fdinfo deltas for our own ffmpeg children. + clients := readFdinfoCounters(s.procDir, s.ffmpegPIDs()) + elapsedNS := int64(0) + if !s.prevGPUAt.IsZero() { + elapsedNS = now.Sub(s.prevGPUAt).Nanoseconds() + } + for pdev, delta := range deviceEngineDeltas(s.prevGPU, clients) { + entry := upsert(pdev) + if path, ok := pathByPCI[pdev]; ok { + entry.Device = path + alias(pdev, path) + } + if elapsedNS > 0 { + entry.VideoBusyPct = engineBusyPercent(delta.videoNS, elapsedNS) + entry.RenderBusyPct = engineBusyPercent(delta.renderNS, elapsedNS) + } + entry.Source = SourceFdinfo + } + // Clients that vanished (their transcode exited) drop out of the baseline + // with their counters, so the next client on that device is not measured + // against a stale origin. + s.prevGPU = clients + s.prevGPUAt = now + + // 3. nvidia-smi enrichment: the only signal for NVIDIA, and whole-GPU + // (other tenants included) where it applies. + for _, gpu := range s.queryNVIDIA(ctx) { + cudaName := "cuda:" + strconv.Itoa(gpu.Index) + key := gpu.PCIAddress + if key == "" { + key = cudaName + } + entry := upsert(key) + if entry.Device == key && entry.Vendor == "" { + // No DRM node for it (the proprietary driver exposes none this + // process can read), so name it the way playback addresses it. + entry.Device = cudaName + } + // NVENC workloads are counted under a CUDA name or a GPU UUID even when + // the device is displayed by its render path, so both have to resolve to + // this entry. + alias(key, cudaName, gpu.UUID) + entry.Vendor = vendorNVIDIA + total := gpu.GPUUtil + entry.TotalBusyPct = &total + used, capacity := gpu.MemUsedMB, gpu.MemTotalMB + entry.VRAMUsedMB = &used + entry.VRAMTotalMB = &capacity + if entry.Source == SourceFdinfo { + entry.Source = SourceFdinfoNVIDIASMI + } else { + entry.Source = SourceNVIDIASMI + // fdinfo is unimplemented by the proprietary driver, so the video + // engines only have an nvidia-smi reading. + entry.VideoBusyPct = max(gpu.EncoderUtil, gpu.DecoderUtil) + } + } + + if len(order) == 0 { + return nil + } + claimed := map[string]bool{} + out := make([]GPUStats, 0, len(order)) + for _, key := range order { + entry := byKey[key] + entry.Sessions = deviceSessions(sessions, aliases[key], claimed) + out = append(out, *entry) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Device < out[j].Device }) + return out +} + +// deviceSessions totals the balancer's workload counts for one device. +// +// The balancer counts a workload under whatever playback was configured with, +// which is not always the name this package displays: an NVENC job on a card +// that also has a readable render node is counted as "cuda:0" while the entry +// is shown as /dev/dri/renderD128. Looking the count up by display name alone +// reports 0 sessions on a node that is transcoding. Every alias is summed once +// — claimed keeps a name shared by two entries from being counted twice. +// The display name is always among the aliases: it is only ever set to one. +func deviceSessions(sessions map[string]int, aliases []string, claimed map[string]bool) int { + total := 0 + for _, name := range aliases { + if name == "" || claimed[name] { + continue + } + claimed[name] = true + total += sessions[name] + } + return total +} + +// scratchPath returns the transcode scratch dir this sampler was told about, +// used to label its series without publishing the path itself. +func (s *Sampler) scratchPath() string { + if s == nil { + return "" + } + return s.scratchDir +} + +func (s *Sampler) identityList() []DeviceIdentity { + if s.identities == nil { + return nil + } + return s.identities() +} diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go new file mode 100644 index 000000000..67fe3844c --- /dev/null +++ b/internal/nodemetrics/sampler_test.go @@ -0,0 +1,546 @@ +package nodemetrics + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +// fakeClock advances only when a test says so, so every rate in this package is +// computed against an interval the test chose rather than against however long +// the test machine happened to take. +type fakeClock struct{ at time.Time } + +func newFakeClock() *fakeClock { + return &fakeClock{at: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} +} + +func (c *fakeClock) now() time.Time { return c.at } + +func (c *fakeClock) advance(d time.Duration) { c.at = c.at.Add(d) } + +// procTree builds a fake /proc under t.TempDir and returns its path. +type procTree struct { + t *testing.T + root string +} + +func newProcTree(t *testing.T) *procTree { + t.Helper() + return &procTree{t: t, root: t.TempDir()} +} + +// write creates a file relative to the tree root, making parents as needed. +func (p *procTree) write(relPath, content string) { + p.t.Helper() + full := filepath.Join(p.root, relPath) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + p.t.Fatalf("mkdir %s: %v", full, err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + p.t.Fatalf("write %s: %v", full, err) + } +} + +// newTestSampler wires a sampler onto a fake /proc with a fake clock and no +// real subprocess, statfs or hardware access. +func newTestSampler(t *testing.T, tree *procTree, clock *fakeClock, opts Options) *Sampler { + t.Helper() + if opts.Now == nil { + opts.Now = clock.now + } + if opts.FFmpegChildren == nil { + opts.FFmpegChildren = func() []int { return nil } + } + s := NewSampler(opts) + s.goos = "linux" + s.procDir = tree.root + s.cgroupLimitPaths = nil + s.cgroupUsagePaths = nil + s.cgroupCPUPaths = nil + s.statfs = func(string) (fsStats, error) { return fsStats{}, os.ErrNotExist } + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return nil, os.ErrNotExist } + return s +} + +func TestCPUBusyPercentFromProcStatDeltas(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + // user nice system idle iowait irq softirq steal + tree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + tree.write("loadavg", "3.20 1.10 0.90 2/512 9\n") + tree.write("meminfo", "MemTotal: 1024 kB\nMemAvailable: 512 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("first sample CPUPct = %d, want 0 (no previous reading to diff against)", got) + } + + // 400 jiffies of work out of 1000 elapsed. + tree.write("stat", "cpu 300 0 300 1400 0 0 0 0\ncpu0 150 0 150 700 0 0 0 0\ncpu1 150 0 150 700 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 40 { + t.Fatalf("CPUPct = %d, want 40", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2", system.Cores) + } + if system.Load1 != 3.20 { + t.Fatalf("Load1 = %v, want 3.2", system.Load1) + } +} + +// A counter that went backwards means the two readings do not describe one +// continuous run. Reporting the difference would produce a nonsense percentage. +func TestCPUBusyPercentClampsCounterReset(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 1000 0 1000 8000 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + tree.write("stat", "cpu 10 0 10 80 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("CPUPct after counter reset = %d, want 0", got) + } + + // The reset reading becomes the new baseline, so the next interval is + // measured normally rather than against the pre-reset counter. + tree.write("stat", "cpu 60 0 60 180 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 50 { + t.Fatalf("CPUPct after re-baselining = %d, want 50", got) + } +} + +func TestNetworkThroughputExcludesLoopbackAndClampsResets(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + netDev := func(loRx, eth0Rx, eth0Tx int) string { + return "Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + + " lo: " + itoa(loRx) + " 1 0 0 0 0 0 0 " + itoa(loRx) + " 1 0 0 0 0 0 0\n" + + " eth0: " + itoa(eth0Rx) + " 1 0 0 0 0 0 0 " + itoa(eth0Tx) + " 1 0 0 0 0 0 0\n" + } + tree.write("net/dev", netDev(1_000_000, 1000, 2000)) + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + // eth0 gains 5000 rx and 10000 tx bytes over 5s; loopback gains a lot and + // must not appear. + tree.write("net/dev", netDev(99_000_000, 6000, 12000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.NetRxBps != 8000 { // 5000 bytes / 5s * 8 + t.Fatalf("NetRxBps = %d, want 8000", system.NetRxBps) + } + if system.NetTxBps != 16000 { // 10000 bytes / 5s * 8 + t.Fatalf("NetTxBps = %d, want 16000", system.NetTxBps) + } + + // An interface restarting resets the aggregate; that must read as zero, not + // as a negative or a wrapped spike. + tree.write("net/dev", netDev(99_000_000, 10, 20)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + system = s.Snapshot().System + if system.NetRxBps != 0 || system.NetTxBps != 0 { + t.Fatalf("throughput after counter reset = %d/%d, want 0/0", system.NetRxBps, system.NetTxBps) + } +} + +func TestMemoryFromMeminfo(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("net/dev", "") + // 8 GiB total, 6 GiB available. + tree.write("meminfo", "MemTotal: 8388608 kB\nMemFree: 512000 kB\nMemAvailable: 6291456 kB\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != 8192 { + t.Fatalf("MemTotalMB = %d, want 8192", system.MemTotalMB) + } + if system.MemUsedMB != 2048 { + t.Fatalf("MemUsedMB = %d, want 2048", system.MemUsedMB) + } +} + +// In a container /proc/meminfo describes the host. The cgroup limit is what the +// kernel will actually OOM-kill against, so it has to win. +func TestMemoryCorrectedByCgroupLimitAndUsage(t *testing.T) { + for _, tc := range []struct { + name string + limitFile string + limitBody string + usageFile string + statFile string + statBody string + inactiveKey string + wantTotalMB int64 + wantUsedMB int64 + usageBody string + }{ + { + name: "cgroup v2", + limitFile: "memory.max", + limitBody: "2147483648\n", // 2 GiB + usageFile: "memory.current", + usageBody: "1073741824\n", // 1 GiB charged + statFile: "memory.stat", + statBody: "anon 536870912\ninactive_file 536870912\n", + inactiveKey: "inactive_file", + wantTotalMB: 2048, + wantUsedMB: 512, // page cache does not count toward the working set + }, + { + name: "cgroup v1", + limitFile: "memory.limit_in_bytes", + limitBody: "4294967296\n", // 4 GiB + usageFile: "memory.usage_in_bytes", + usageBody: "2147483648\n", + statFile: "memory.stat", + statBody: "total_inactive_file 1073741824\n", + inactiveKey: "total_inactive_file", + wantTotalMB: 4096, + wantUsedMB: 1024, + }, + { + name: "no concrete limit falls back to the host", + limitFile: "memory.max", + limitBody: "max\n", + usageFile: "memory.current", + usageBody: "", + statFile: "memory.stat", + statBody: "", + inactiveKey: "inactive_file", + wantTotalMB: 8192, + wantUsedMB: 2048, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("net/dev", "") + tree.write("meminfo", "MemTotal: 8388608 kB\nMemAvailable: 6291456 kB\n") + + cgroupDir := t.TempDir() + if err := os.WriteFile(filepath.Join(cgroupDir, tc.limitFile), []byte(tc.limitBody), 0o644); err != nil { + t.Fatal(err) + } + if tc.usageBody != "" { + if err := os.WriteFile(filepath.Join(cgroupDir, tc.usageFile), []byte(tc.usageBody), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cgroupDir, tc.statFile), []byte(tc.statBody), 0o644); err != nil { + t.Fatal(err) + } + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupLimitPaths = []string{filepath.Join(cgroupDir, tc.limitFile)} + s.cgroupUsagePaths = []cgroupUsagePath{{ + usage: filepath.Join(cgroupDir, tc.usageFile), + stat: filepath.Join(cgroupDir, tc.statFile), + inactiveFile: tc.inactiveKey, + }} + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != tc.wantTotalMB { + t.Fatalf("MemTotalMB = %d, want %d", system.MemTotalMB, tc.wantTotalMB) + } + if system.MemUsedMB != tc.wantUsedMB { + t.Fatalf("MemUsedMB = %d, want %d", system.MemUsedMB, tc.wantUsedMB) + } + }) + } +} + +// In a container /proc/stat describes the host, exactly as /proc/meminfo does. +// A node capped at two cores of a mostly idle 8-core host and pinned at its +// quota is the state worth alerting on, and reading the host would report it as +// nearly idle. +func TestCPUCorrectedByCgroupQuotaAndUsage(t *testing.T) { + for _, tc := range []struct { + name string + paths cgroupCPUPath + quotaFile string + quotaBody string + usageFile string + usageFirst string + usageLater string + periodFile string + periodBody string + }{ + { + name: "cgroup v2", + paths: cgroupCPUPath{usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond}, + usageFile: "cpu.stat", + usageFirst: "usage_usec 1000000\nuser_usec 500000\n", + usageLater: "usage_usec 11000000\nuser_usec 5000000\n", // +10s of CPU + quotaFile: "cpu.max", + quotaBody: "200000 100000\n", // 2 cores + }, + { + name: "cgroup v1", + paths: cgroupCPUPath{usageUnit: time.Nanosecond}, + usageFile: "cpuacct.usage", + usageFirst: "1000000000\n", + usageLater: "11000000000\n", // +10s of CPU + quotaFile: "cpu.cfs_quota_us", + quotaBody: "200000\n", + periodFile: "cpu.cfs_period_us", + periodBody: "100000\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + // The host has 8 cores and is barely busy; the container is not. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 9900)) + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + cgroupDir := t.TempDir() + writeCgroup := func(name, body string) { + if err := os.WriteFile(filepath.Join(cgroupDir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + writeCgroup(tc.usageFile, tc.usageFirst) + writeCgroup(tc.quotaFile, tc.quotaBody) + paths := tc.paths + paths.usage = filepath.Join(cgroupDir, tc.usageFile) + paths.quota = filepath.Join(cgroupDir, tc.quotaFile) + if tc.periodFile != "" { + writeCgroup(tc.periodFile, tc.periodBody) + paths.period = filepath.Join(cgroupDir, tc.periodFile) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{paths} + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("first sample CPUPct = %d, want 0 (nothing to diff against)", got) + } + + // 10 seconds of CPU over 5 seconds of wall time on a 2-core quota: + // the container is pegged, while the host is 1% busy. + writeCgroup(tc.usageFile, tc.usageLater) + tree.write("stat", hostStat(200, 19800)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 100 { + t.Fatalf("CPUPct = %d, want 100 (the container's own usage against its own quota)", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the 2 cores the cgroup allows, not the host's 8", system.Cores) + } + }) + } +} + +// An unconstrained cgroup still measures this process's domain, but there is no +// quota to normalize against, so the host's core count is the right divisor. +func TestCPUWithoutCgroupQuotaUsesHostCores(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + cgroupDir := t.TempDir() + usage := filepath.Join(cgroupDir, "cpu.stat") + if err := os.WriteFile(usage, []byte("usage_usec 0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cgroupDir, "cpu.max"), []byte("max 100000\n"), 0o644); err != nil { + t.Fatal(err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: filepath.Join(cgroupDir, "cpu.max"), + }} + s.sample(context.Background()) + + // 5 seconds of CPU over 5 seconds of wall time across 2 cores: half busy. + if err := os.WriteFile(usage, []byte("usage_usec 5000000\n"), 0o644); err != nil { + t.Fatal(err) + } + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 50 { + t.Fatalf("CPUPct = %d, want 50", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the host's 2 with no quota set", system.Cores) + } +} + +// A host that cannot be sampled must publish an explicitly unavailable +// snapshot, so a reader can distinguish it from a host that is simply idle. +func TestNonLinuxHostReportsUnavailable(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + s.goos = "darwin" + s.sample(context.Background()) + + snapshot := s.Snapshot() + if snapshot.Available { + t.Fatal("Available = true on a non-Linux host") + } + if snapshot.System != nil || snapshot.GPU != nil { + t.Fatalf("snapshot carries data on a non-Linux host: %+v", snapshot) + } +} + +// The exact JSON is a wire contract: node health responses, the admin resources +// endpoint, and the persisted last_stats column all carry this shape. +func TestSnapshotJSONShape(t *testing.T) { + total := 71 + vramUsed := int64(812) + vramTotal := int64(8192) + snapshot := Snapshot{ + Available: true, + System: &SystemStats{ + CPUPct: 41, + Load1: 3.2, + Cores: 16, + MemUsedMB: 9011, + MemTotalMB: 32768, + Disks: []DiskStats{ + {Path: "/transcode", UsedGB: 210, TotalGB: 500}, + {Path: "/media", UsedGB: 7100, TotalGB: 8000, Stale: true}, + {Path: "/gone", Unavailable: true}, + }, + NetRxBps: 1200000, + NetTxBps: 98000000, + }, + GPU: []GPUStats{{ + Device: "/dev/dri/renderD128", + Vendor: "intel", + Sessions: 2, + VideoBusyPct: 63, + RenderBusyPct: 12, + TotalBusyPct: &total, + VRAMUsedMB: &vramUsed, + VRAMTotalMB: &vramTotal, + Source: SourceFdinfoNVIDIASMI, + }}, + } + + encoded, err := json.Marshal(snapshot) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatal(err) + } + + system, ok := decoded["system"].(map[string]any) + if !ok { + t.Fatalf("system missing from %s", encoded) + } + for _, key := range []string{"cpu_pct", "load1", "cores", "mem_used_mb", "mem_total_mb", "disks", "net_rx_bps", "net_tx_bps"} { + if _, ok := system[key]; !ok { + t.Fatalf("system.%s missing from %s", key, encoded) + } + } + disks := system["disks"].([]any) + if _, ok := disks[0].(map[string]any)["stale"]; ok { + t.Fatalf("stale emitted for a fresh disk: %s", encoded) + } + if stale, _ := disks[1].(map[string]any)["stale"].(bool); !stale { + t.Fatalf("stale missing for a stale disk: %s", encoded) + } + if unavailable, _ := disks[2].(map[string]any)["unavailable"].(bool); !unavailable { + t.Fatalf("unavailable missing: %s", encoded) + } + + gpu := decoded["gpu"].([]any)[0].(map[string]any) + for _, key := range []string{"device", "vendor", "sessions", "video_busy_pct", "render_busy_pct", "total_busy_pct", "vram_used_mb", "vram_total_mb", "source"} { + if _, ok := gpu[key]; !ok { + t.Fatalf("gpu.%s missing from %s", key, encoded) + } + } + + // Absent enrichment must be absent, not zero: an operator reading + // total_busy_pct: 0 would conclude the GPU is idle when nothing measured it. + bare, err := json.Marshal(Snapshot{Available: true, GPU: []GPUStats{{Device: "/dev/dri/renderD128", Source: SourceFdinfo}}}) + if err != nil { + t.Fatal(err) + } + var bareDecoded map[string]any + if err := json.Unmarshal(bare, &bareDecoded); err != nil { + t.Fatal(err) + } + if _, ok := bareDecoded["system"]; ok { + t.Fatalf("system emitted when absent: %s", bare) + } + bareGPU := bareDecoded["gpu"].([]any)[0].(map[string]any) + for _, key := range []string{"total_busy_pct", "vram_used_mb", "vram_total_mb", "vendor"} { + if _, ok := bareGPU[key]; ok { + t.Fatalf("gpu.%s emitted when absent: %s", key, bare) + } + } +} + +// itoa avoids importing strconv into every fixture builder above. +func itoa(v int) string { + if v == 0 { + return "0" + } + digits := "" + for v > 0 { + digits = string(rune('0'+v%10)) + digits + v /= 10 + } + return digits +} diff --git a/internal/nodemetrics/snapshot.go b/internal/nodemetrics/snapshot.go new file mode 100644 index 000000000..bb02d6d0b --- /dev/null +++ b/internal/nodemetrics/snapshot.go @@ -0,0 +1,139 @@ +// Package nodemetrics samples the host resources a media node actually runs +// out of — CPU, memory, disk, network, and GPU engine busyness — into a small +// in-memory snapshot that health responses, the admin API, and Prometheus all +// read without doing any work of their own. +// +// The contract every reader depends on is that sampling is fully decoupled from +// reading. A sample runs on the sampler's own goroutine on a fixed interval and +// publishes an immutable snapshot with one atomic store; a reader loads the +// latest pointer and never waits, never touches /proc, and never runs a +// subprocess. That is what makes it safe to put these numbers in a node's +// /health response, which is the signal the cluster uses to route streams away +// from a dying node: a wedged NFS mount or a hung nvidia-smi degrades one field +// of one snapshot instead of stalling health for every node in the pool. +// +// All sampling is Linux-only. Everywhere else the snapshot reports +// Available=false and the loop idles, so the package is safe to construct +// unconditionally on any platform. +package nodemetrics + +import "time" + +// Snapshot is one complete sampling pass. Snapshots are immutable once +// published: a sampler builds a new one and swaps it in rather than mutating +// the one readers hold. +type Snapshot struct { + // Available reports whether this host can be sampled at all. It is false on + // non-Linux hosts and before the first sample completes, which is why System + // and GPU are pointers/slices rather than values — "no numbers yet" must be + // distinguishable from "everything is zero". + Available bool `json:"available"` + // SampledAt is when this pass ran. Readers use it to tell a live sample from + // a sampler that stopped ticking. + SampledAt time.Time `json:"sampled_at,omitzero"` + // System is the host's CPU/memory/disk/network sample, omitted when the host + // could not be sampled. + System *SystemStats `json:"system,omitempty"` + // GPU is one entry per GPU this process can say anything about, omitted when + // there are none. + GPU []GPUStats `json:"gpu,omitempty"` +} + +// SystemStats is the host resource sample. +type SystemStats struct { + // CPUPct is aggregate busy percentage across all cores between the previous + // sample and this one, 0-100. Under a cgroup it is this container's own + // usage against its own quota, not the host's. + CPUPct int `json:"cpu_pct"` + // Load1 is the 1-minute load average, which unlike CPUPct also counts + // uninterruptible-sleep tasks — a node blocked on storage looks idle in + // CPUPct and busy here. + Load1 float64 `json:"load1"` + // Cores is how many CPUs this process may run on — the cgroup's quota + // rounded up where one is set, otherwise every CPU the kernel reports. It is + // what CPUPct is already normalized against, and what Load1 must be read + // relative to. + Cores int `json:"cores"` + MemUsedMB int64 `json:"mem_used_mb"` + MemTotalMB int64 `json:"mem_total_mb"` + // Disks holds the sampled mounts: the transcode scratch dir first, then any + // media roots the process was told about. Deduplicated by filesystem, so two + // paths on one volume are reported once. + Disks []DiskStats `json:"disks"` + // NetRxBps and NetTxBps are aggregate throughput in *bits* per second, + // matching the node egress_kbps unit used elsewhere in the cluster, with + // loopback excluded. + NetRxBps int64 `json:"net_rx_bps"` + NetTxBps int64 `json:"net_tx_bps"` +} + +// DiskStats is one sampled mount. +type DiskStats struct { + Path string `json:"path"` + UsedGB float64 `json:"used_gb"` + TotalGB float64 `json:"total_gb"` + // Stale marks numbers carried over from an earlier pass because the current + // probe has not returned — the normal reading for a network mount whose + // server went away. The values are real, just old. + Stale bool `json:"stale,omitempty"` + // Unavailable marks a path that has never been measured successfully (it + // does not exist on this node, or the very first probe is still hanging). + // UsedGB/TotalGB are meaningless when it is set. + Unavailable bool `json:"unavailable,omitempty"` +} + +// vendorNVIDIA is the GPUStats.Vendor value nvidia-smi enrichment reports. +const vendorNVIDIA = "nvidia" + +// GPU measurement sources, in order of how much they can see. +const ( + // SourceUnavailable means nothing could measure this device: no owned + // process held it and no whole-GPU query answered. + SourceUnavailable = "unavailable" + // SourceFdinfo is the unprivileged DRM baseline. It sees only engine time + // spent by this process's own ffmpeg children, never another tenant's. + SourceFdinfo = "fdinfo" + // SourceNVIDIASMI is whole-GPU utilization from nvidia-smi, which is the + // only signal for NVIDIA (the proprietary driver implements no fdinfo). + SourceNVIDIASMI = "nvidia-smi" + // SourceFdinfoNVIDIASMI is both, for a device that has DRM counters and an + // nvidia-smi row. + SourceFdinfoNVIDIASMI = "fdinfo+nvidia-smi" +) + +// GPUStats is one GPU's sample. +type GPUStats struct { + // Device is the render-node path (/dev/dri/renderD128) where one is known, + // otherwise the PCI address or a "cuda:N" index for an NVIDIA GPU with no + // DRM node this process can see. + Device string `json:"device"` + // Vendor is "intel", "nvidia", "amd" or empty when unknown. + Vendor string `json:"vendor,omitempty"` + // Sessions is how many of this process's GPU workloads are currently pinned + // to this device. It comes from the playback device balancer, not from the + // driver, so it is exact for our own work and blind to everyone else's. + Sessions int `json:"sessions"` + // VideoBusyPct and RenderBusyPct are engine busy percentages over the last + // sample interval. From fdinfo they cover only our own processes. + VideoBusyPct int `json:"video_busy_pct"` + RenderBusyPct int `json:"render_busy_pct"` + // TotalBusyPct is whole-GPU utilization including other tenants. It is a + // pointer because "no enrichment source" and "idle" are different facts and + // an operator must not read the first as the second. + TotalBusyPct *int `json:"total_busy_pct,omitempty"` + VRAMUsedMB *int64 `json:"vram_used_mb,omitempty"` + VRAMTotalMB *int64 `json:"vram_total_mb,omitempty"` + // Source names what produced the numbers above; see the Source* constants. + Source string `json:"source"` +} + +// DeviceIdentity is one render device as the host's hardware detection sees it. +// The sampler needs it only to translate the PCI addresses DRM fdinfo reports +// into the /dev/dri paths every other surface (settings, session accounting, +// the admin UI) speaks, which is why this package takes it as a provider +// instead of doing its own hardware walk. +type DeviceIdentity struct { + Path string + PCIAddress string + Vendor string +} diff --git a/internal/nodemetrics/statfs_other.go b/internal/nodemetrics/statfs_other.go new file mode 100644 index 000000000..3c05a18e7 --- /dev/null +++ b/internal/nodemetrics/statfs_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package nodemetrics + +import "errors" + +// osStatfs has no implementation on platforms without statfs(2). Sampling is +// Linux-only anyway, so this only has to keep the package building. +func osStatfs(string) (fsStats, error) { + return fsStats{}, errors.ErrUnsupported +} diff --git a/internal/nodemetrics/statfs_unix.go b/internal/nodemetrics/statfs_unix.go new file mode 100644 index 000000000..aa4b94651 --- /dev/null +++ b/internal/nodemetrics/statfs_unix.go @@ -0,0 +1,30 @@ +//go:build linux || darwin + +package nodemetrics + +import ( + "golang.org/x/sys/unix" +) + +// osStatfs reports one path's filesystem capacity. +// +// Used space is computed from blocks the filesystem considers free, not from +// the free space available to an unprivileged user (Bavail): the reserved +// margin is genuinely occupied capacity, and reporting it as used is what makes +// the number match what an operator sees in `df`. +// +// This call can block indefinitely on an unresponsive network mount. Callers +// must treat it as such — see probeDisk, which runs it on a goroutine nothing +// waits for. +func osStatfs(path string) (fsStats, error) { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + return fsStats{}, err + } + blockSize := uint64(st.Bsize) + return fsStats{ + UsedBytes: (st.Blocks - st.Bfree) * blockSize, + TotalBytes: st.Blocks * blockSize, + FSID: formatFSID(int64(st.Fsid.Val[0]), int64(st.Fsid.Val[1])), + }, nil +} diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go new file mode 100644 index 000000000..fb38aab7f --- /dev/null +++ b/internal/nodemetrics/system.go @@ -0,0 +1,236 @@ +package nodemetrics + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// cpuTimes is one /proc/stat aggregate reading. Both fields are monotonic jiffy +// counters; only differences between two readings mean anything. +type cpuTimes struct { + busy uint64 + total uint64 + valid bool +} + +// netCounters is one /proc/net/dev aggregate reading, excluding loopback. +type netCounters struct { + rx uint64 + tx uint64 + at time.Time + valid bool +} + +// readCPUTimes parses the aggregate "cpu" line of /proc/stat and counts the +// per-core lines beneath it. +// +// Idle and iowait are both subtracted from busy: a core waiting on storage is +// not doing work, and counting it as busy would make every node with a slow +// disk look CPU-bound. +func readCPUTimes(procDir string) (cpuTimes, int) { + raw, err := os.ReadFile(filepath.Join(procDir, "stat")) + if err != nil { + return cpuTimes{}, 0 + } + var times cpuTimes + cores := 0 + for line := range strings.Lines(string(raw)) { + fields := strings.Fields(line) + if len(fields) == 0 || !strings.HasPrefix(fields[0], "cpu") { + continue + } + if fields[0] != "cpu" { + // "cpu0", "cpu1", … — one line per online core. + cores++ + continue + } + if times.valid { + continue + } + var total, idle uint64 + for i, field := range fields[1:] { + value, err := strconv.ParseUint(field, 10, 64) + if err != nil { + continue + } + // Columns are user, nice, system, idle, iowait, irq, softirq, + // steal, guest, guest_nice. guest time is already included in user, + // so summing past steal would double-count it. + if i >= 8 { + break + } + total += value + if i == 3 || i == 4 { + idle += value + } + } + if total == 0 { + continue + } + times = cpuTimes{busy: total - idle, total: total, valid: true} + } + return times, cores +} + +// cpuBusyPercent converts two readings into a busy percentage. +// +// A counter that went backwards means the readings do not describe one +// continuous run (a container was migrated, /proc was remounted, or a test +// rewound the fixture), so the pair is unusable rather than negative. +func cpuBusyPercent(previous, current cpuTimes) (int, bool) { + if !previous.valid || !current.valid { + return 0, false + } + if current.total <= previous.total || current.busy < previous.busy { + return 0, false + } + totalDelta := current.total - previous.total + busyDelta := current.busy - previous.busy + return clampPercent(int((busyDelta*100 + totalDelta/2) / totalDelta)), true +} + +// readLoad1 parses the 1-minute load average from /proc/loadavg. +func readLoad1(procDir string) float64 { + raw, err := os.ReadFile(filepath.Join(procDir, "loadavg")) + if err != nil { + return 0 + } + fields := strings.Fields(string(raw)) + if len(fields) == 0 { + return 0 + } + value, err := strconv.ParseFloat(fields[0], 64) + if err != nil || value < 0 { + return 0 + } + return value +} + +// readNetCounters sums received and transmitted bytes across every interface +// except loopback, which would otherwise double-count traffic a node sends to +// itself (health checks, the local proxy hop). +func readNetCounters(procDir string, at time.Time) netCounters { + raw, err := os.ReadFile(filepath.Join(procDir, "net", "dev")) + if err != nil { + return netCounters{} + } + counters := netCounters{at: at} + for line := range strings.Lines(string(raw)) { + name, rest, ok := strings.Cut(line, ":") + if !ok { + // The two header lines carry no colon. + continue + } + if strings.TrimSpace(name) == "lo" { + continue + } + fields := strings.Fields(rest) + // Eight receive columns then eight transmit columns; bytes lead each. + if len(fields) < 9 { + continue + } + rx, rxErr := strconv.ParseUint(fields[0], 10, 64) + tx, txErr := strconv.ParseUint(fields[8], 10, 64) + if rxErr != nil || txErr != nil { + continue + } + counters.rx += rx + counters.tx += tx + counters.valid = true + } + return counters +} + +// netThroughputBps converts two readings into bits per second. +// +// Interfaces disappearing (a container restarting its veth) drops the aggregate +// counter, and a 32-bit counter on a busy link wraps; both show up as a +// negative delta, and both are reported as zero rather than as a spike, since +// an invented number here would land straight in an operator's bandwidth graph. +func netThroughputBps(previous, current netCounters) (rxBps, txBps int64, ok bool) { + if !previous.valid || !current.valid { + return 0, 0, false + } + seconds := current.at.Sub(previous.at).Seconds() + if seconds <= 0 { + return 0, 0, false + } + rate := func(prev, cur uint64) int64 { + if cur < prev { + return 0 + } + return int64(float64(cur-prev) * 8 / seconds) + } + return rate(previous.rx, current.rx), rate(previous.tx, current.tx), true +} + +// memoryStats reports used and total bytes for this process's memory domain. +// +// /proc/meminfo describes the host even inside a container, so a cgroup limit — +// which is what the kernel will actually OOM-kill against — always wins over +// it, and cgroup usage (page cache excluded) wins over the host's own +// used figure for the same reason. +func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { + fields, err := ReadMeminfoBytes(filepath.Join(s.procDir, "meminfo")) + if err == nil { + totalBytes = fields["MemTotal"] + if available, ok := fields["MemAvailable"]; ok && totalBytes >= available { + usedBytes = totalBytes - available + } + } + + for _, path := range s.cgroupLimitPaths { + limit, err := ReadCgroupMemoryLimit(path) + if err != nil || limit <= 0 { + continue + } + // A limit above the host's memory is not a limit worth reporting; it + // would only make a node look like it has headroom the kernel cannot + // give it. + if totalBytes == 0 || limit < totalBytes { + totalBytes = limit + } + break + } + + if usage, ok := s.cgroupMemoryUsage(); ok { + usedBytes = usage + } + if totalBytes > 0 && usedBytes > totalBytes { + usedBytes = totalBytes + } + return usedBytes, totalBytes +} + +// cgroupMemoryUsage returns the working set of this process's memory cgroup: +// current charge minus reclaimable file pages. +func (s *Sampler) cgroupMemoryUsage() (int64, bool) { + for _, paths := range s.cgroupUsagePaths { + usage, err := readCgroupSingleValue(paths.usage) + if err != nil { + continue + } + if inactive, err := readCgroupStatKey(paths.stat, paths.inactiveFile); err == nil && inactive > 0 && inactive <= usage { + usage -= inactive + } + return usage, true + } + return 0, false +} + +func clampPercent(value int) int { + if value < 0 { + return 0 + } + if value > 100 { + return 100 + } + return value +} + +const bytesPerMB = int64(1024 * 1024) + +func bytesToMB(value int64) int64 { return value / bytesPerMB } diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 752b717e4..e4d8807e3 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -1,8 +1,10 @@ package nodepool import ( + "bytes" "context" "encoding/json" + "io" "log/slog" "net/http" "slices" @@ -19,36 +21,114 @@ type healthResponse struct { // snapshot. A node that predates capability snapshots reports none, which // is how the sweep tells "nothing changed" from "cannot say". CapabilitiesHash string `json:"capabilities_hash"` + // System and GPU are the node's latest resource sample, carried opaquely. + // This package deliberately does not parse them: node metrics are display + // data, nothing here routes on them, and decoding them would make nodepool + // depend on the sampler's schema — so a node running a newer build can add + // fields without an API-side change. + System json.RawMessage `json:"system"` + GPU json.RawMessage `json:"gpu"` } +// maxHealthResponseBytes bounds a node's whole /health body. +// +// A node is a worker that may run on remote, less trusted hardware, and its +// health answer is the one node-controlled payload this process decodes every +// 30 seconds. An honest sample is under 2 KB; this leaves three orders of +// magnitude of headroom while keeping a buggy or hostile build from making the +// API allocate an arbitrary body on a fixed cadence. +const maxHealthResponseBytes = 256 << 10 + +// maxLastStatsBytes bounds the resource blob that is persisted and served. +// +// Past this, the stats are dropped and the health verdict is kept: whether a +// node is alive routes streams, while its dashboard numbers do not, and an +// oversized blob would otherwise be rewritten into a jsonb column every sweep +// and echoed to every admin listing nodes. +const maxLastStatsBytes = 32 << 10 + // CheckNode pings a node's /health endpoint and returns its health status, -// active job count, reported egress bandwidth, and capability hash. -func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int, capabilitiesHash string) { +// active job count, reported egress bandwidth, capability hash, and the opaque +// resource-stats blob to persist (nil when the node reported none). +func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int, capabilitiesHash string, lastStats []byte) { client := &http.Client{Timeout: 5 * time.Second} ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, n.URL+"/api/v1/health", nil) if err != nil { - return false, 0, 0, "" + return false, 0, 0, "", nil } resp, err := client.Do(req) if err != nil { - return false, 0, 0, "" + return false, 0, 0, "", nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return false, 0, 0, "" + return false, 0, 0, "", nil + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxHealthResponseBytes+1)) + if err != nil { + return false, 0, 0, "", nil + } + if len(body) > maxHealthResponseBytes { + // Nothing in the body can be trusted to be well-formed at that size, so + // the node is treated as not answering rather than partially believed. + slog.WarnContext(ctx, "node health response too large to read", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, "limit_bytes", maxHealthResponseBytes) + return false, 0, 0, "", nil } var hr healthResponse - if err := json.NewDecoder(resp.Body).Decode(&hr); err != nil { - return false, 0, 0, "" + if err := json.Unmarshal(body, &hr); err != nil { + return false, 0, 0, "", nil } - return true, hr.ActiveJobs, hr.EgressKbps, hr.CapabilitiesHash + return true, hr.ActiveJobs, hr.EgressKbps, hr.CapabilitiesHash, marshalLastStats(ctx, n, hr) +} + +// marshalLastStats packs a health response's resource fields into the blob +// stored on the node row, or nil when the node sent neither. +// +// nil is what a node predating resource sampling produces, and it must persist +// as SQL NULL rather than as an empty object: "this node cannot report" and +// "this node reported nothing in use" are different states, and only the second +// would justify drawing a zero on a dashboard. +func marshalLastStats(ctx context.Context, n *Node, hr healthResponse) []byte { + system := trimJSONNull(hr.System) + gpu := trimJSONNull(hr.GPU) + if system == nil && gpu == nil { + return nil + } + payload := struct { + System json.RawMessage `json:"system,omitempty"` + GPU json.RawMessage `json:"gpu,omitempty"` + }{System: system, GPU: gpu} + encoded, err := json.Marshal(payload) + if err != nil { + return nil + } + if len(encoded) > maxLastStatsBytes { + slog.WarnContext(ctx, "node resource sample too large to store", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, + "bytes", len(encoded), "limit_bytes", maxLastStatsBytes) + return nil + } + return encoded +} + +// trimJSONNull normalizes an absent field. encoding/json leaves a RawMessage +// nil when the key is missing but sets it to the literal "null" when the key is +// present and null, and both mean the node has nothing to say. +func trimJSONNull(raw json.RawMessage) json.RawMessage { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || string(trimmed) == "null" { + return nil + } + return trimmed } // CapabilityFetcher retrieves one node's full capability report and the hash @@ -147,18 +227,21 @@ func (hc *HealthChecker) Start(ctx context.Context) { }() } +// applyHealthFunc is a pool's copy-on-write health writer. +type applyHealthFunc func(id int, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) + // applyCapabilitiesFunc is a pool's copy-on-write capability writer. type applyCapabilitiesFunc func(id int, capabilities []byte, hash string, refreshedAt time.Time) func (hc *HealthChecker) checkAll(ctx context.Context) { var wg sync.WaitGroup - check := func(n *Node, applyHealth func(int, bool, int, int, time.Time), applyCapabilities applyCapabilitiesFunc) { + check := func(n *Node, applyHealth applyHealthFunc, applyCapabilities applyCapabilitiesFunc) { wg.Go(func() { - healthy, activeJobs, egressKbps, capabilitiesHash := CheckNode(ctx, n) + healthy, activeJobs, egressKbps, capabilitiesHash, lastStats := CheckNode(ctx, n) // Publish the result through the pool lock so readers never see // a Node struct mutated in place (the pool swaps in a copy). - applyHealth(n.ID, healthy, activeJobs, egressKbps, time.Now()) + applyHealth(n.ID, healthy, activeJobs, egressKbps, lastStats, time.Now()) if n.Healthy && !healthy { slog.WarnContext(ctx, "stream node unhealthy", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) @@ -167,7 +250,7 @@ func (hc *HealthChecker) checkAll(ctx context.Context) { } if hc.repo != nil { - if err := hc.repo.UpdateHealth(ctx, n.ID, healthy, activeJobs, egressKbps); err != nil { + if err := hc.repo.UpdateHealth(ctx, n.ID, healthy, activeJobs, egressKbps, lastStats); err != nil { slog.ErrorContext(ctx, "failed to persist node health", "component", "nodepool", "id", n.ID, "error", err) } } diff --git a/internal/nodepool/health_stats_test.go b/internal/nodepool/health_stats_test.go new file mode 100644 index 000000000..b4857c711 --- /dev/null +++ b/internal/nodepool/health_stats_test.go @@ -0,0 +1,175 @@ +package nodepool + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// newStatsHealthNode starts a stand-in node whose /health serves a verbatim +// body, so a test can express exactly what an old or new node puts on the wire. +func newStatsHealthNode(t *testing.T, body string) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server.URL +} + +// Node resource stats pass through this package untouched. Parsing them here +// would couple the health sweep to the sampler's schema for no benefit — +// nothing in nodepool routes on them. +func TestCheckNodePassesResourceStatsThroughOpaquely(t *testing.T) { + url := newStatsHealthNode(t, `{ + "status":"ok", + "active_jobs":2, + "egress_kbps":17, + "capabilities_hash":"sha256:abc", + "system":{"cpu_pct":41,"load1":3.2,"cores":16,"mem_used_mb":9011,"mem_total_mb":32768, + "disks":[{"path":"/transcode","used_gb":210,"total_gb":500}], + "net_rx_bps":1200000,"net_tx_bps":98000000}, + "gpu":[{"device":"/dev/dri/renderD128","vendor":"intel","sessions":2, + "video_busy_pct":63,"render_busy_pct":12,"source":"fdinfo", + "future_field_we_do_not_know":true}] + }`) + + healthy, activeJobs, egressKbps, hash, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy || activeJobs != 2 || egressKbps != 17 || hash != "sha256:abc" { + t.Fatalf("check = %v/%d/%d/%q, want the existing fields unchanged", healthy, activeJobs, egressKbps, hash) + } + if len(lastStats) == 0 { + t.Fatal("lastStats is empty; the node reported a sample") + } + + var decoded struct { + System struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []map[string]any `json:"gpu"` + } + if err := json.Unmarshal(lastStats, &decoded); err != nil { + t.Fatalf("lastStats is not valid JSON: %v (%s)", err, lastStats) + } + if decoded.System.CPUPct != 41 { + t.Fatalf("system.cpu_pct = %d, want 41", decoded.System.CPUPct) + } + if len(decoded.GPU) != 1 { + t.Fatalf("gpu = %v, want one device", decoded.GPU) + } + // A newer node adding a field must survive the round trip, which is the + // point of keeping the payload opaque. + if _, ok := decoded.GPU[0]["future_field_we_do_not_know"]; !ok { + t.Fatalf("unknown gpu field was dropped: %s", lastStats) + } +} + +// A node predating resource sampling — and a node that reports an explicit null +// — must both produce nil, which persists as SQL NULL. Writing an empty object +// instead would draw zeros on a dashboard for a node that measured nothing. +func TestCheckNodeReportsNoStatsForOlderNodes(t *testing.T) { + for name, body := range map[string]string{ + "fields absent": `{"status":"ok","active_jobs":1,"egress_kbps":0,"capabilities_hash":""}`, + "fields null": `{"status":"ok","active_jobs":1,"system":null,"gpu":null}`, + "gpu empty only": `{"status":"ok","active_jobs":1,"gpu":null}`, + } { + t.Run(name, func(t *testing.T) { + url := newStatsHealthNode(t, body) + healthy, _, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy { + t.Fatal("node reported unhealthy") + } + if lastStats != nil { + t.Fatalf("lastStats = %s, want nil so the column is written NULL", lastStats) + } + }) + } +} + +// A node that reports only one half still persists that half. +func TestCheckNodeKeepsAPartialSample(t *testing.T) { + url := newStatsHealthNode(t, `{"status":"ok","gpu":[{"device":"cuda:0","source":"nvidia-smi"}]}`) + _, _, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + var decoded map[string]json.RawMessage + if err := json.Unmarshal(lastStats, &decoded); err != nil { + t.Fatalf("lastStats invalid: %v (%s)", err, lastStats) + } + if _, ok := decoded["system"]; ok { + t.Fatalf("system emitted when the node sent none: %s", lastStats) + } + if _, ok := decoded["gpu"]; !ok { + t.Fatalf("gpu missing: %s", lastStats) + } +} + +// A node is a worker that may run on hardware this process does not control, +// and last_stats is the one thing it dictates that gets written to the control +// plane database every 30 seconds. An oversized sample is dropped, not stored — +// but whether the node is alive still routes streams, so the verdict survives. +func TestCheckNodeDropsAnOversizedResourceSample(t *testing.T) { + padding := strings.Repeat("x", maxLastStatsBytes) + url := newStatsHealthNode(t, `{"status":"ok","active_jobs":3,"egress_kbps":9, + "system":{"cpu_pct":41,"junk":"`+padding+`"}}`) + + healthy, activeJobs, egressKbps, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy || activeJobs != 3 || egressKbps != 9 { + t.Fatalf("check = %v/%d/%d, want the health verdict kept", healthy, activeJobs, egressKbps) + } + if lastStats != nil { + t.Fatalf("lastStats = %d bytes, want the oversized sample dropped", len(lastStats)) + } +} + +// Past the body cap nothing in the response can be trusted to be well formed, +// so the node reads as not answering rather than as partially believed. +func TestCheckNodeRejectsAnOversizedHealthBody(t *testing.T) { + url := newStatsHealthNode(t, `{"status":"ok","active_jobs":3,"junk":"`+ + strings.Repeat("x", maxHealthResponseBytes)+`"}`) + + healthy, activeJobs, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if healthy || activeJobs != 0 || lastStats != nil { + t.Fatalf("check = %v/%d/%s, want an unreadable body treated as no answer", healthy, activeJobs, lastStats) + } +} + +// An unreachable node reports nothing, and its stats must be cleared rather +// than left behind: a dead node's five-minute-old CPU number reads as live. +func TestApplyHealthClearsStatsWhenACheckCarriesNone(t *testing.T) { + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) + + pool.ApplyHealth(1, true, 1, 0, []byte(`{"system":{"cpu_pct":41}}`), time.Now()) + stored := pool.Nodes()[0] + if len(stored.LastStats) == 0 { + t.Fatal("stats were not published to the pool") + } + + pool.ApplyHealth(1, false, 0, 0, nil, time.Now()) + if got := pool.Nodes()[0].LastStats; got != nil { + t.Fatalf("LastStats = %s after a failed check, want nil", got) + } +} + +// The pool publishes immutable copies. A caller's buffer (a decoded HTTP body) +// must not stay aliased into a node other goroutines are reading. +func TestApplyHealthClonesStats(t *testing.T) { + pool := NewProxyPool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) + + buffer := []byte(`{"system":{"cpu_pct":41}}`) + pool.ApplyHealth(1, true, 0, 0, buffer, time.Now()) + copy(buffer, []byte(`{"system":{"cpu_pct":99}}`)) + + if got := string(pool.Nodes()[0].LastStats); got != `{"system":{"cpu_pct":41}}` { + t.Fatalf("LastStats = %s, want the published copy to be independent of the caller's buffer", got) + } +} diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index fe32baf12..74fd50fa4 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -556,7 +556,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // Unlike job reservations, bandwidth bridges ignore health freshness — // a report right after admission would not reflect the streams yet. newer := f.now.Add(5 * time.Second) - f.proxies.ApplyHealth(1, true, 0, 0, newer) + f.proxies.ApplyHealth(1, true, 0, 0, nil, newer) f.now = f.now.Add(10 * time.Second) if got := f.planner.PlanSession("s4", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("stream should still be rejected during bridge window, got %+v", got) @@ -566,7 +566,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // meter now reports 8 Mbps, so one more 4 Mbps stream still won't fit, // but a 2 Mbps one will. f.now = f.now.Add(bandwidthBridgeAge) - f.proxies.ApplyHealth(1, true, 0, 8_000, f.now) + f.proxies.ApplyHealth(1, true, 0, 8_000, nil, f.now) if got := f.planner.PlanSession("s5", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("4 Mbps stream should not fit at 8/10 Mbps, got %+v", got) } @@ -618,7 +618,7 @@ func TestUnknownBitrateAdmittedBelowCap(t *testing.T) { t.Fatal("unknown-bitrate stream should be admitted below cap") } - f.proxies.ApplyHealth(1, true, 0, 10_000, f.now) + f.proxies.ApplyHealth(1, true, 0, 10_000, nil, f.now) if got := f.planner.PlanSession("s2", "", false, 0).ProxyNode; got != nil { t.Fatalf("unknown-bitrate stream should be rejected at cap, got %+v", got) } diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index 6e4d0e1c2..b7bf6fa5e 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -57,10 +57,10 @@ func (p *ProxyPool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *ProxyPool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +func (p *ProxyPool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) + applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, lastStats, checkedAt) } // ApplyCapabilities records a freshly fetched capability report by swapping the diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index be98cadb7..3a44c4422 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -44,6 +44,12 @@ type Node struct { // CapabilitiesRefreshedAt is when Capabilities was last fetched — the age of // the inventory, not of the last health check. CapabilitiesRefreshedAt *time.Time `json:"capabilities_refreshed_at,omitempty"` + // LastStats is the node's resource sample from the last health check — + // {"system":…,"gpu":…} — kept opaque for the same reason as Capabilities. + // It is written by the same 30s health update that writes ActiveJobs, so it + // is exactly as fresh as LastHealthCheck and never fresher. Absent for a + // node that reports no sample. + LastStats json.RawMessage `json:"last_stats,omitempty"` } // CreateNodeInput holds the fields for creating a new node. @@ -110,13 +116,13 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at` +const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats` func scanNode(row pgx.Row) (*Node, error) { var n Node // jsonb is scanned as raw bytes rather than into json.RawMessage directly so // a NULL column stays nil instead of decoding through the JSON codec. - var capabilities []byte + var capabilities, lastStats []byte err := row.Scan( &n.ID, &n.Name, &n.Type, &n.URL, &n.Enabled, &n.Healthy, &n.ActiveJobs, @@ -124,6 +130,7 @@ func scanNode(row pgx.Row) (*Node, error) { &n.MaxBandwidthKbps, &n.EgressKbps, &n.LastHealthCheck, &n.CreatedAt, &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, + &lastStats, ) if err != nil { return nil, err @@ -131,6 +138,9 @@ func scanNode(row pgx.Row) (*Node, error) { if len(capabilities) > 0 { n.Capabilities = json.RawMessage(capabilities) } + if len(lastStats) > 0 { + n.LastStats = json.RawMessage(lastStats) + } return &n, nil } @@ -249,13 +259,18 @@ func (r *Repository) Delete(ctx context.Context, id int) error { return nil } -// UpdateHealth updates a node's health status, active job count, and -// reported egress bandwidth. -func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int) error { +// UpdateHealth updates a node's health status, active job count, reported +// egress bandwidth, and last resource sample. +// +// A nil lastStats writes NULL, which is what a node that reports no sample — +// an older build, or a non-Linux host — must produce. Passing the previous +// value through instead would leave a dead node's numbers on screen looking +// current. +func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int, lastStats []byte) error { tag, err := r.pool.Exec(ctx, - `UPDATE stream_nodes SET healthy = $2, active_jobs = $3, egress_kbps = $4, last_health_check = NOW() + `UPDATE stream_nodes SET healthy = $2, active_jobs = $3, egress_kbps = $4, last_stats = $5, last_health_check = NOW() WHERE id = $1`, - id, healthy, activeJobs, egressKbps) + id, healthy, activeJobs, egressKbps, lastStats) if err != nil { return fmt.Errorf("update node health: %w", err) } diff --git a/internal/nodepool/repository_last_stats_test.go b/internal/nodepool/repository_last_stats_test.go new file mode 100644 index 000000000..a57d0b244 --- /dev/null +++ b/internal/nodepool/repository_last_stats_test.go @@ -0,0 +1,121 @@ +package nodepool + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" +) + +// newLastStatsTestPool follows the repository-wide convention: skip without a +// database, and skip again on a database that predates the migration under test +// rather than failing on a missing column. +func newLastStatsTestPool(t *testing.T) *Repository { + t.Helper() + pool := newNodeTestPool(t) + var column *string + if err := pool.QueryRow(context.Background(), + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'stream_nodes' AND column_name = 'last_stats'`).Scan(&column); err != nil { + t.Skip("test database has not applied the node last_stats migration") + } + return NewRepository(pool) +} + +func createLastStatsNode(t *testing.T, repo *Repository) *Node { + t.Helper() + ctx := context.Background() + unique := time.Now().UnixNano() + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("last-stats-test-%d", unique), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://last-stats-test-%d", unique), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + return node +} + +// The health sweep's existing 30s write is the only write path for resource +// stats. It has to round-trip through an ordinary List — the same read the +// pools and the admin API use. +func TestRepositoryUpdateHealthPersistsLastStats(t *testing.T) { + repo := newLastStatsTestPool(t) + ctx := context.Background() + node := createLastStatsNode(t, repo) + + if node.LastStats != nil { + t.Fatalf("new node already carries stats: %s", node.LastStats) + } + + stats := []byte(`{"system":{"cpu_pct":41,"mem_used_mb":9011},"gpu":[{"device":"/dev/dri/renderD128","source":"fdinfo"}]}`) + if err := repo.UpdateHealth(ctx, node.ID, true, 3, 17, stats); err != nil { + t.Fatalf("update health: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.ActiveJobs != 3 || reloaded.EgressKbps != 17 || !reloaded.Healthy { + t.Fatalf("existing health fields = %+v, want them unchanged by the new column", reloaded) + } + var decoded struct { + System struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + } `json:"gpu"` + } + if err := json.Unmarshal(reloaded.LastStats, &decoded); err != nil { + t.Fatalf("stored stats are not json: %v (%s)", err, reloaded.LastStats) + } + if decoded.System.CPUPct != 41 { + t.Fatalf("stored system.cpu_pct = %d, want 41", decoded.System.CPUPct) + } + if len(decoded.GPU) != 1 || decoded.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("stored gpu = %+v", decoded.GPU) + } +} + +// A node that reports no sample writes NULL, and a node that stops reporting +// clears what it had. Keeping the previous row would leave a dead node's +// numbers on the Nodes page looking current. +func TestRepositoryUpdateHealthWritesNullForNodesWithoutStats(t *testing.T) { + repo := newLastStatsTestPool(t) + ctx := context.Background() + node := createLastStatsNode(t, repo) + + if err := repo.UpdateHealth(ctx, node.ID, true, 1, 0, []byte(`{"system":{"cpu_pct":41}}`)); err != nil { + t.Fatalf("update health: %v", err) + } + if err := repo.UpdateHealth(ctx, node.ID, false, 0, 0, nil); err != nil { + t.Fatalf("update health without stats: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.LastStats != nil { + t.Fatalf("LastStats = %s, want NULL after a check that carried none", reloaded.LastStats) + } + + // The absent column must also be absent from the admin API's JSON, so a + // client can tell "no sample" from "a sample of zeroes". + encoded, err := json.Marshal(reloaded) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if err := json.Unmarshal(encoded, &body); err != nil { + t.Fatal(err) + } + if _, ok := body["last_stats"]; ok { + t.Fatalf("last_stats emitted for a node with none: %s", encoded) + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 75758f19f..0757fd90e 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -82,10 +82,10 @@ func (p *TranscodePool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *TranscodePool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +func (p *TranscodePool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) + applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, lastStats, checkedAt) } // ApplyCapabilities records a freshly fetched capability report by swapping the @@ -97,7 +97,7 @@ func (p *TranscodePool) ApplyCapabilities(id int, capabilities []byte, hash stri } // applyNodeHealth replaces the slice entry for id with an updated copy. -func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { for i, n := range nodes { if n.ID != id { continue @@ -107,6 +107,15 @@ func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps clone.ActiveJobs = activeJobs clone.EgressKbps = egressKbps clone.LastHealthCheck = &checkedAt + // A check that carried no stats clears them rather than keeping the + // previous sample: an unreachable node's five-minute-old CPU number + // looks live on a dashboard, which is worse than no number at all. The + // payload is cloned because the caller's buffer is a decoded HTTP body. + if len(lastStats) > 0 { + clone.LastStats = append(json.RawMessage(nil), lastStats...) + } else { + clone.LastStats = nil + } nodes[i] = &clone return } diff --git a/internal/playback/capabilityhash.go b/internal/playback/capabilityhash.go index 198dbbbd3..a10313183 100644 --- a/internal/playback/capabilityhash.go +++ b/internal/playback/capabilityhash.go @@ -62,6 +62,7 @@ type canonicalDetectedBackend struct { Devices []string `json:"devices"` Device string `json:"device"` Reason string `json:"reason"` + Skipped bool `json:"skipped"` } type canonicalTransformation struct { @@ -115,6 +116,7 @@ func canonicalDetectedBackends(backends []DetectedBackend) []canonicalDetectedBa Devices: sortedStrings(backend.Devices), Device: backend.Device, Reason: backend.Reason, + Skipped: backend.Skipped, }) } slices.SortFunc(out, func(a, b canonicalDetectedBackend) int { diff --git a/internal/playback/directplay_test.go b/internal/playback/directplay_test.go index 55412f895..2f197aaea 100644 --- a/internal/playback/directplay_test.go +++ b/internal/playback/directplay_test.go @@ -20,12 +20,6 @@ import ( dto "github.com/prometheus/client_model/go" ) -const ( - directPlayDarwinGOOS = "darwin" - directPlayLinuxGOOS = "linux" - directPlayWindowsGOOS = "windows" -) - func TestServeDirectPlayHTTPContract(t *testing.T) { const content = "0123456789abcdefghijklmnopqrstuvwxyz" filePath := filepath.Join(t.TempDir(), "fixture.mp4") diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 460f471f6..96654cbcb 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -18,6 +18,13 @@ import ( "golang.org/x/sync/singleflight" ) +// GOOS names this package and its tests compare runtime.GOOS against. +const ( + directPlayDarwinGOOS = "darwin" + directPlayLinuxGOOS = "linux" + directPlayWindowsGOOS = "windows" +) + var ( defaultDRIDir = "/dev/dri" defaultNVIDIAControlDevice = "/dev/nvidiactl" @@ -64,6 +71,11 @@ type DetectedBackend struct { Device string `json:"device,omitempty"` // Reason explains a failure, attributed per device when several were tried. Reason string `json:"reason,omitempty"` + // Skipped reports that no probe was attempted because none of the + // backend's candidate devices is accessible to this process — a proxy + // node reading a cluster-wide hw_device meant for the transcode nodes, + // not a driver failure. Reason still says which devices were skipped. + Skipped bool `json:"skipped,omitempty"` } // HWAccelInfo describes the detected hardware acceleration capability. @@ -112,7 +124,7 @@ func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Dur // DetectHWAccel probes this host's GPU hardware and returns structured info. func DetectHWAccel() HWAccelInfo { - return DetectHWAccelWithFFmpeg("auto", "", "") + return DetectHWAccelWithFFmpeg(hwAccelAuto, "", "") } // DetectHWAccelWithFFmpeg probes this host's GPU hardware and configured FFmpeg. @@ -129,10 +141,10 @@ func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hw candidates := collectHWCandidates(hwDevice) resolved := HWAccelNone var detected []DetectedBackend - if currentGOOS == "linux" { + if currentGOOS == directPlayLinuxGOOS { resolved, detected = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) } - if configured := strings.TrimSpace(hwAccel); configured != "" && configured != "auto" { + if configured := strings.TrimSpace(hwAccel); configured != "" && configured != hwAccelAuto { resolved = configured } return HWAccelInfo{ @@ -176,7 +188,7 @@ func ResolveHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) string { // ResolveHWAccelWithFFmpegContext resolves auto hardware without allowing any // FFmpeg capability probe to outlive ctx. func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string { - if hwAccel != "auto" { + if hwAccel != hwAccelAuto { return hwAccel } if currentGOOS != "linux" { @@ -186,6 +198,10 @@ func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, h return resolved } +// hwAccelAuto is the configured hw_accel value that asks this package to pick +// a backend by probing the host, rather than the operator naming one outright. +const hwAccelAuto = "auto" + // hwAccelPreferenceOrder is the auto-resolution order; the first backend whose // probe passes wins. var hwAccelPreferenceOrder = []string{transcodeHWNVENC, transcodeHWQSV, transcodeHWVAAPI} @@ -204,6 +220,11 @@ type hwCandidates struct { nvidia []string intel []string vaapi []string + // accessible records, for a configured probe set only, which devices this + // process can actually open. nil means the set came from discovery, which + // already filtered on openability. NVENC's empty device string never + // consults it — CUDA selects its GPU without a render-node path. + accessible map[string]bool nvidiaPresent bool // intelPresent describes the inventory rather than the probe set, so a // pinned non-Intel device does not hide an Intel GPU from operators. @@ -220,6 +241,15 @@ func collectHWCandidates(configuredDevice string) hwCandidates { probeDevices := ParseHWDeviceSet(configuredDevice).List() if len(probeDevices) == 0 { probeDevices = candidates.renderDevices + } else { + // A configured device this process cannot open can never pass a smoke + // encode, so it is classified for reporting but never probed. This is + // the normal state of a proxy node reading the cluster-wide hw_device + // meant for the transcode nodes. + candidates.accessible = make(map[string]bool, len(probeDevices)) + for _, device := range probeDevices { + candidates.accessible[device] = deviceOpenable(device) + } } for _, device := range probeDevices { switch { @@ -299,12 +329,12 @@ func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCa } entry := verifyHWAccelBackend(ctx, backend, ffmpegPath, candidates) if !entry.Verified { - slog.Warn("hw_accel=auto: candidate hardware failed its FFmpeg probe", + slog.WarnContext(ctx, "hw_accel=auto: candidate hardware failed its FFmpeg probe", "backend", backend, "devices", entry.Devices, "ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", entry.Reason) } else if resolved == "" { resolved = backend - slog.Info("hw_accel=auto: verified hardware backend", "backend", backend, "device", entry.Device) + slog.InfoContext(ctx, "hw_accel=auto: verified hardware backend", "backend", backend, "device", entry.Device) } detected = append(detected, entry) if resolved != "" && stopAtFirstVerified { @@ -312,7 +342,7 @@ func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCa } } if resolved == "" { - slog.Info("hw_accel=auto: no verified hardware backend, using software encoding") + slog.InfoContext(ctx, "hw_accel=auto: no verified hardware backend, using software encoding") return HWAccelNone, detected } return resolved, detected @@ -325,10 +355,16 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi devices := candidates.probeDevicesFor(backend) entry := DetectedBackend{Backend: backend, Devices: candidates.devicesFor(backend)} reasons := make([]string, 0, len(devices)) + probed := false for _, device := range devices { if ctx.Err() != nil { break } + if !candidates.deviceProbeable(device) { + reasons = append(reasons, hwProbeFailureReason(len(devices), device, "device not accessible on this node")) + continue + } + probed = true available, reason := ffmpegSupportsBackendContext(ctx, backend, ffmpegPath, device) if available { entry.Verified = true @@ -340,10 +376,34 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi if len(reasons) == 0 { reasons = append(reasons, "hardware detection budget exhausted before probing "+backend) } + entry.Skipped = !probed && len(devices) > 0 && ctx.Err() == nil entry.Reason = strings.Join(reasons, "; ") return entry } +// deviceProbeable reports whether a candidate device may be smoke-encoded on. +// The empty device is NVENC's — CUDA needs no render-node path — and a nil map +// means the candidate set came from discovery, which is openable by +// construction. +func (c hwCandidates) deviceProbeable(device string) bool { + if device == "" || c.accessible == nil { + return true + } + return c.accessible[device] +} + +// deviceOpenable mirrors the accessibility filter listRenderDevices applies to +// discovered devices: a device this process cannot open cannot host a probe or +// a transcode. +func deviceOpenable(device string) bool { + f, err := os.Open(device) + if err != nil { + return false + } + _ = f.Close() + return true +} + // hwProbeFailureReason attributes a failure to its device only when several // candidates were tried; a single candidate reads better bare. func hwProbeFailureReason(candidateCount int, device, reason string) string { @@ -490,8 +550,8 @@ func probeFFmpegNVENCContext(ctx context.Context, ffmpegPath, device string, com if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} - } else if !ffmpegOutputHasToken(output, "h264_nvenc") { - return hwProbeResult{reason: "h264_nvenc encoder unavailable"} + } else if !ffmpegOutputHasToken(output, encoderH264NVENC) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264NVENC)} } else if !ffmpegOutputHasToken(output, "hevc_nvenc") { return hwProbeResult{reason: "hevc_nvenc encoder unavailable"} } @@ -519,8 +579,8 @@ func probeFFmpegQSVContext(ctx context.Context, ffmpegPath, device string, comma if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} - } else if !ffmpegOutputHasToken(output, "h264_qsv") { - return hwProbeResult{reason: "h264_qsv encoder unavailable"} + } else if !ffmpegOutputHasToken(output, encoderH264QSV) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264QSV)} } else if !ffmpegOutputHasToken(output, "hevc_qsv") { return hwProbeResult{reason: "hevc_qsv encoder unavailable"} } @@ -534,8 +594,8 @@ func probeFFmpegQSVContext(ctx context.Context, ffmpegPath, device string, comma func probeFFmpegVAAPIContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} - } else if !ffmpegOutputHasToken(output, "h264_vaapi") { - return hwProbeResult{reason: "h264_vaapi encoder unavailable"} + } else if !ffmpegOutputHasToken(output, encoderH264VAAPI) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264VAAPI)} } return smokeEncodeResult(ctx, ffmpegPath, transcodeHWVAAPI, device, commandTimeout) @@ -551,15 +611,28 @@ func smokeEncodeResult(ctx context.Context, ffmpegPath, backend, device string, return hwProbeResult{available: true} } +// H.264 encoder names FFmpeg reports for each hardware backend. +const ( + encoderH264QSV = "h264_qsv" + encoderH264VAAPI = "h264_vaapi" + encoderH264NVENC = "h264_nvenc" +) + +// encoderUnavailableReason reports that a probe's -encoders listing did not +// include the given encoder. +func encoderUnavailableReason(encoder string) string { + return encoder + " encoder unavailable" +} + // hardwareEncoder returns the H.264 encoder paired with a backend. func hardwareEncoder(backend string) string { switch backend { case transcodeHWQSV: - return "h264_qsv" + return encoderH264QSV case transcodeHWVAAPI: - return "h264_vaapi" + return encoderH264VAAPI default: - return "h264_nvenc" + return encoderH264NVENC } } @@ -711,6 +784,57 @@ func readSysfsID(path string) string { return strings.TrimSpace(string(data)) } +// RenderDeviceIdentity is a render device's hardware identity, without the +// probing that a full capability report performs. +type RenderDeviceIdentity struct { + // Path is the render node, e.g. /dev/dri/renderD128. + Path string + // PCIAddress is the sysfs PCI slot, e.g. 0000:03:00.0. + PCIAddress string + // Vendor is "intel", "nvidia", "amd", or empty when sysfs names one we do + // not recognize. + Vendor string +} + +// RenderDeviceIdentities enumerates this host's render devices with the sysfs +// identity of each. +// +// It exists for callers that need to correlate a device across surfaces — +// notably resource sampling, which learns about GPUs by PCI address from DRM +// fdinfo and has to name them the way the rest of the server does. It runs no +// ffmpeg probe and takes no lock: it is a sysfs read, cheap enough to call on a +// sampling interval, unlike DetectHWAccelWithFFmpeg. +func RenderDeviceIdentities() []RenderDeviceIdentity { + if currentGOOS != "linux" { + return nil + } + devices := listRenderDevices(defaultDRIDir) + identities := make([]RenderDeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, RenderDeviceIdentity{ + Path: device, + PCIAddress: renderDevicePCIAddress(device), + Vendor: renderDeviceVendor(device), + }) + } + return identities +} + +// renderDeviceVendor maps a device's sysfs PCI vendor id to a short label. +func renderDeviceVendor(renderDevPath string) string { + name := filepath.Base(renderDevPath) + switch readSysfsID(filepath.Join(sysClassDRMDir, name, "device", "vendor")) { + case "0x8086": + return "intel" + case "0x10de": + return "nvidia" + case "0x1002": + return "amd" + default: + return "" + } +} + // renderDeviceDetails describes every listed device. func renderDeviceDetails(devices []string) []RenderDeviceInfo { details := make([]RenderDeviceInfo, 0, len(devices)) diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 24ab460d2..58dc7eaab 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -180,6 +180,82 @@ func TestDetectHWAccelReportsHostInventoryBehindAPinnedDevice(t *testing.T) { } } +// A proxy node reads the cluster-wide hw_device meant for the transcode nodes: +// the paths and their sysfs vendor entries are visible, but the devices cannot +// be opened. Detection must skip the probes entirely — no ffmpeg spawn, no +// alarming driver error — and say why. +func TestDetectHWAccelSkipsConfiguredDevicesItCannotOpen(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + configured := filepath.Join(env.driDir, "renderD128") + "," + filepath.Join(env.driDir, "renderD129") + for _, name := range []string{"renderD128", "renderD129"} { + if err := os.Remove(filepath.Join(env.driDir, name)); err != nil { + t.Fatal(err) + } + } + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, configured) + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) == 0 { + t.Fatal("DetectedBackends is empty, want skipped qsv/vaapi entries") + } + for _, backend := range info.DetectedBackends { + if !backend.Skipped { + t.Fatalf("backend %q Skipped = false, want true", backend.Backend) + } + if backend.Verified { + t.Fatalf("backend %q Verified = true, want false", backend.Backend) + } + if !strings.Contains(backend.Reason, "not accessible") { + t.Fatalf("backend %q Reason = %q, want an accessibility reason", backend.Backend, backend.Reason) + } + } + if logData, err := os.ReadFile(ffmpeg.logPath); err == nil && len(strings.TrimSpace(string(logData))) > 0 { + t.Fatalf("ffmpeg was spawned for inaccessible devices; log:\n%s", logData) + } +} + +// One configured device is gone, the other works: the accessible one must +// still be probed and win, and the missing one must not be smoke-encoded. +func TestDetectHWAccelProbesOnlyTheAccessibleConfiguredDevices(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + if err := os.Remove(filepath.Join(env.driDir, "renderD128")); err != nil { + t.Fatal(err) + } + configured := filepath.Join(env.driDir, "renderD128") + "," + filepath.Join(env.driDir, "renderD129") + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, configured) + if info.Resolved != "qsv" { + t.Fatalf("Resolved = %q, want qsv from the accessible device", info.Resolved) + } + var qsv *DetectedBackend + for i := range info.DetectedBackends { + if info.DetectedBackends[i].Backend == "qsv" { + qsv = &info.DetectedBackends[i] + } + } + if qsv == nil || qsv.Skipped || !qsv.Verified { + t.Fatalf("qsv entry = %+v, want verified and not skipped", qsv) + } + if qsv.Device != filepath.Join(env.driDir, "renderD129") { + t.Fatalf("qsv Device = %q, want the accessible renderD129", qsv.Device) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("probe touched the inaccessible device; log:\n%s", logData) + } +} + func TestResolveHWAccelWithFFmpegReturnsNoneWhenVAAPISmokeEncodeFails(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x1002") diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index f51f0f997..138a70b3e 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -3,6 +3,7 @@ package playback import ( "log/slog" "os" + "strconv" "strings" "sync" ) @@ -17,7 +18,8 @@ import ( // index/UUID rather than render-node path, so a multi-entry list falls back // to its first entry. A single configured value keeps the historical // pass-through contract for every accelerator, so existing deployments are -// unaffected. +// unaffected — but it is still counted, because per-device session reporting +// has to work on the one-GPU node that most deployments are. // HWDeviceSet is the parsed form of the playback.hw_device setting: an // ordered list of device entries. Order is priority order — ties in load @@ -71,6 +73,56 @@ var hwDeviceLoad = struct { counts map[string]int }{counts: map[string]int{}} +// DefaultNVENCDevice is the accounting key for an NVENC workload that named no +// device. NVENC addresses GPUs through the CUDA runtime rather than a render +// node, and ffmpeg defaults to CUDA device 0, so that is what an unqualified +// NVENC job is actually occupying. +const DefaultNVENCDevice = "cuda:0" + +// HWDeviceLoadSnapshot returns a copy of the active workload count per device. +// +// This exists so node metrics can report how many of this process's GPU jobs +// are pinned to each device — a number no driver can supply, because the driver +// sees processes and not sessions. It is a copy taken under the same lock the +// allocator uses, so a reader never observes a half-applied reservation and +// never holds up one. +func HWDeviceLoadSnapshot() map[string]int { + hwDeviceLoad.mu.Lock() + defer hwDeviceLoad.mu.Unlock() + counts := make(map[string]int, len(hwDeviceLoad.counts)) + for device, count := range hwDeviceLoad.counts { + if count > 0 { + counts[device] = count + } + } + return counts +} + +// nvencAccountingDevice is the key an NVENC workload is counted under. +// +// A configured multi-entry list resolves to its first entry, matching the +// device NVENC will actually use; an unconfigured value counts against the CUDA +// default. Accounting only — selection is untouched, because counting a +// workload and balancing on the count are different decisions and NVENC is +// deliberately excluded from the second. +// +// A bare CUDA index is rewritten to "cuda:N", because the count is only useful +// if it joins with the name resource sampling publishes for the same GPU, and +// that is "cuda:N" (or the render path, which is matched against the same +// index). Counting "1" would leave every explicitly-selected NVIDIA GPU +// reporting zero sessions while it transcodes. A GPU UUID passes through: the +// sampler knows nvidia-smi's UUID for each card and matches on it. +func nvencAccountingDevice(configured string) string { + first := ParseHWDeviceSet(configured).First() + if first == "" { + return DefaultNVENCDevice + } + if index, err := strconv.Atoi(first); err == nil && index >= 0 { + return "cuda:" + strconv.Itoa(index) + } + return first +} + // hwAccelBalancesRenderDevices reports whether the resolved acceleration mode // selects GPUs by render-device path, which is what the balancer hands out. // NVENC addresses GPUs by CUDA index/UUID and is deliberately excluded. @@ -122,16 +174,27 @@ func newHWDeviceRelease(device string) func() { } } +// countHWDeviceWorkload records one active workload against a device without +// influencing selection. It is the accounting half of a reservation, used where +// the device was decided elsewhere (NVENC, which picks by CUDA index, and +// session restarts that keep their original device). +func countHWDeviceWorkload(device string) func() { + hwDeviceLoad.mu.Lock() + hwDeviceLoad.counts[device]++ + hwDeviceLoad.mu.Unlock() + return newHWDeviceRelease(device) +} + // reserveConcreteHWDevice reserves a device that was selected for an earlier // process in the same transcode session. Restarts keep device affinity rather // than running the least-loaded selection again. func reserveConcreteHWDevice(device string) func() { + release := countHWDeviceWorkload(device) hwDeviceLoad.mu.Lock() - hwDeviceLoad.counts[device]++ count := hwDeviceLoad.counts[device] hwDeviceLoad.mu.Unlock() slog.Info("GPU workload device reserved", "device", device, "active_workloads", count) - return newHWDeviceRelease(device) + return release } var nvencMultiDeviceWarnOnce sync.Once @@ -140,37 +203,62 @@ var nvencMultiDeviceWarnOnce sync.Once // device for one GPU workload. resolvedHWAccel must already be resolved (no // "auto"). The returned release must be called exactly once when the ffmpeg // process for this workload has exited; it is idempotent and a no-op for -// workloads that did not reserve (empty or single-device value, or an -// accelerator the balancer does not manage). +// workloads that were not counted (a software accelerator, or a GPU accelerator +// with no configured device to name). // // - Empty value: returns "" so downstream auto-detection applies. -// - Single value: passes through unchanged for every accelerator. +// - Single value: passes through unchanged for every accelerator, and is +// counted for the GPU accelerators, because a node with one GPU is the +// common deployment and its sessions have to be reportable too. // - Multi-device value with QSV/VAAPI: reserves the present device with the // fewest active workloads (ties keep list order) until release. -// - Multi-device value with any other accelerator (including NVENC, which -// addresses GPUs by CUDA index/UUID, not render-node path): falls back to -// the first entry without reserving. +// - NVENC: selection is unchanged — the first entry, or empty for +// auto-detection — but the workload is counted against the device it will +// occupy so per-device reporting covers NVIDIA nodes too. NVENC addresses +// GPUs by CUDA index/UUID rather than render-node path, so it is still +// never balanced across a list. +// - Any other accelerator: falls back to the first entry without counting. func AcquireHWDevice(configured, resolvedHWAccel string) (string, func()) { - return acquireHWDevice(configured, resolvedHWAccel, "") + device, _, release := acquireHWDevice(configured, resolvedHWAccel, "") + return device, release } // acquireHWDevice applies the normal allocator while optionally excluding one // previously failed render device when another present device is available. // The selected device is still reserved through the same accounting path. -func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (string, func()) { +// +// It also returns the key the workload was counted under ("" when it was not +// counted), so a caller that outlives the ffmpeg process — a transcode session +// that restarts one — can re-count the replacement against the same device +// without restating the rule for which workloads count. +func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, workloadDevice string, release func()) { noop := func() {} set := ParseHWDeviceSet(configured) - if !set.Multi() { - return set.First(), noop - } - if !hwAccelBalancesRenderDevices(resolvedHWAccel) { - if resolvedHWAccel == "nvenc" { + if resolvedHWAccel == transcodeHWNVENC { + if set.Multi() { nvencMultiDeviceWarnOnce.Do(func() { slog.Warn("multi-device hw_device is not supported with NVENC (devices are CUDA index/UUID, not render paths); using the first entry", "hw_device", configured, "using", set.First()) }) } - return set.First(), noop + accounted := nvencAccountingDevice(configured) + return set.First(), accounted, countHWDeviceWorkload(accounted) + } + if !hwAccelBalancesRenderDevices(resolvedHWAccel) { + // A software workload occupies no GPU, so counting it would both + // misreport the device and skew the balancer that reads the counts. + return set.First(), "", noop + } + if !set.Multi() { + // Nothing to balance, but there is still a workload on a known device. + // Selection and accounting are separate decisions: skipping the second + // with the first is what left every single-GPU QSV/VAAPI node reporting + // zero sessions beside a busy engine. An empty value has no device to + // name — ffmpeg picks one downstream — so it stays uncounted. + if first := set.First(); first != "" { + return first, first, countHWDeviceWorkload(first) + } + return "", "", noop } // Select and reserve in one critical section so concurrent workload starts // observe each other's reservations instead of piling onto one device. @@ -187,13 +275,13 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (string, f } } hwDeviceLoad.mu.Lock() - device := leastLoadedHWDeviceLocked(present) - hwDeviceLoad.counts[device]++ - count := hwDeviceLoad.counts[device] + selected := leastLoadedHWDeviceLocked(present) + hwDeviceLoad.counts[selected]++ + count := hwDeviceLoad.counts[selected] hwDeviceLoad.mu.Unlock() - slog.Info("GPU workload device selected", "device", device, "active_workloads", count) + slog.Info("GPU workload device selected", "device", selected, "active_workloads", count) - return device, newHWDeviceRelease(device) + return selected, selected, newHWDeviceRelease(selected) } // hwDeviceActiveCount reports the active workload count for one device; test diff --git a/internal/playback/hwdevice_test.go b/internal/playback/hwdevice_test.go index 92f76dbaf..27394b210 100644 --- a/internal/playback/hwdevice_test.go +++ b/internal/playback/hwdevice_test.go @@ -79,6 +79,41 @@ func TestAcquireHWDeviceSingleValuePassesThrough(t *testing.T) { } } +// The single-GPU node is the common deployment. Having nothing to balance is no +// reason to leave its workloads uncounted: sessions would read 0 on a node +// whose engine busy percentage says it is transcoding. +func TestAcquireHWDeviceSingleRenderDeviceIsCounted(t *testing.T) { + for _, accel := range []string{"qsv", "vaapi"} { + t.Run(accel, func(t *testing.T) { + resetDeviceLoad(t) + fakeDeviceStat(t) // the device need not exist for the count to be right + _, releaseFirst := AcquireHWDevice("/dev/dri/renderD128", accel) + _, releaseSecond := AcquireHWDevice("/dev/dri/renderD128", accel) + + if got := HWDeviceLoadSnapshot()["/dev/dri/renderD128"]; got != 2 { + t.Fatalf("snapshot count = %d, want both workloads counted (%v)", got, HWDeviceLoadSnapshot()) + } + releaseFirst() + releaseSecond() + if got := HWDeviceLoadSnapshot(); len(got) != 0 { + t.Fatalf("snapshot after release = %v, want empty", got) + } + }) + } +} + +// With no device configured there is no name to count against: ffmpeg picks the +// device downstream, and inventing a key would report sessions on a device the +// sampler never names. +func TestAcquireHWDeviceUnconfiguredRenderDeviceIsNotCounted(t *testing.T) { + resetDeviceLoad(t) + _, release := AcquireHWDevice("", "vaapi") + defer release() + if got := HWDeviceLoadSnapshot(); len(got) != 0 { + t.Fatalf("snapshot = %v, want no count for an unresolved device", got) + } +} + func TestAcquireHWDeviceBalancesAcrossList(t *testing.T) { resetDeviceLoad(t) fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") @@ -128,16 +163,98 @@ func TestAcquireHWDeviceAllMissingFallsBackToFirst(t *testing.T) { } } -func TestAcquireHWDeviceNVENCMultiListUsesFirstWithoutReserving(t *testing.T) { +// NVENC is counted but never balanced: a multi-entry list still resolves to its +// first entry, because CUDA indexes are not render-node paths and the balancer +// has no way to compare them. +func TestAcquireHWDeviceNVENCMultiListUsesFirstWithoutBalancing(t *testing.T) { resetDeviceLoad(t) fakeDeviceStat(t) // NVENC entries are CUDA indexes/UUIDs, never present as paths - device, release := AcquireHWDevice("0,1", "nvenc") - defer release() - if device != "0" { - t.Fatalf("device = %q, want first NVENC entry", device) + first, releaseFirst := AcquireHWDevice("0,1", "nvenc") + defer releaseFirst() + second, releaseSecond := AcquireHWDevice("0,1", "nvenc") + defer releaseSecond() + + if first != "0" || second != "0" { + t.Fatalf("devices = %q, %q; want both on the first NVENC entry", first, second) } - if got := hwDeviceActiveCount("0"); got != 0 { - t.Fatalf("active count = %d, want no reservation for NVENC", got) + // ffmpeg is handed the bare CUDA index, but the count is keyed the way + // resource sampling names that GPU — otherwise the join drops it. + if got := hwDeviceActiveCount("cuda:0"); got != 2 { + t.Fatalf("active count = %d, want both NVENC workloads counted under cuda:0", got) + } +} + +// NVENC selects GPUs by CUDA index, and the sampler publishes them as "cuda:N". +// Counting the raw configured value would report zero sessions on every +// explicitly-selected NVIDIA GPU while it transcodes. +func TestNVENCAccountingDeviceUsesSamplerNamespace(t *testing.T) { + for _, tc := range []struct{ configured, want string }{ + {"", DefaultNVENCDevice}, + {"0", "cuda:0"}, + {"1", "cuda:1"}, + {"1,0", "cuda:1"}, + {"cuda:1", "cuda:1"}, + {"GPU-1234abcd", "GPU-1234abcd"}, + {"/dev/dri/renderD128", "/dev/dri/renderD128"}, + } { + if got := nvencAccountingDevice(tc.configured); got != tc.want { + t.Fatalf("nvencAccountingDevice(%q) = %q, want %q", tc.configured, got, tc.want) + } + } +} + +// An unconfigured NVENC workload lands on the CUDA default, which is what +// ffmpeg will actually use, so per-device reporting is not blank on the most +// common NVIDIA deployment. +func TestAcquireHWDeviceNVENCUnconfiguredCountsCUDADefault(t *testing.T) { + resetDeviceLoad(t) + device, release := AcquireHWDevice("", "nvenc") + if device != "" { + t.Fatalf("device = %q, want empty so auto-detection applies", device) + } + if got := hwDeviceActiveCount(DefaultNVENCDevice); got != 1 { + t.Fatalf("active count = %d, want the workload counted against %s", got, DefaultNVENCDevice) + } + release() + if got := hwDeviceActiveCount(DefaultNVENCDevice); got != 0 { + t.Fatalf("active count after release = %d, want 0", got) + } +} + +// The snapshot is what node metrics report per device; it must show live +// workloads and drop devices back out once they are released. +func TestHWDeviceLoadSnapshot(t *testing.T) { + resetDeviceLoad(t) + fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") + _, releaseA := AcquireHWDevice("/dev/dri/renderD128,/dev/dri/renderD129", "qsv") + _, releaseB := AcquireHWDevice("/dev/dri/renderD128,/dev/dri/renderD129", "qsv") + _, releaseNVENC := AcquireHWDevice("", "nvenc") + + snapshot := HWDeviceLoadSnapshot() + want := map[string]int{ + "/dev/dri/renderD128": 1, + "/dev/dri/renderD129": 1, + DefaultNVENCDevice: 1, + } + if len(snapshot) != len(want) { + t.Fatalf("snapshot = %v, want %v", snapshot, want) + } + for device, count := range want { + if snapshot[device] != count { + t.Fatalf("snapshot[%q] = %d, want %d (snapshot %v)", device, snapshot[device], count, snapshot) + } + } + + // The copy must not track later changes, or a reader would see counts move + // underneath a snapshot it already published. + releaseA() + releaseB() + releaseNVENC() + if snapshot["/dev/dri/renderD128"] != 1 { + t.Fatal("snapshot changed after release; it is not a copy") + } + if remaining := HWDeviceLoadSnapshot(); len(remaining) != 0 { + t.Fatalf("snapshot after all releases = %v, want empty", remaining) } } @@ -185,15 +302,18 @@ func TestAcquireHWDeviceAvoidsFailedRenderDeviceAndReservesAlternate(t *testing. resetDeviceLoad(t) fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") configured := "/dev/dri/renderD128,/dev/dri/renderD129" - got, release := acquireHWDevice(configured, "qsv", "/dev/dri/renderD128") + got, workload, release := acquireHWDevice(configured, "qsv", "/dev/dri/renderD128") defer release() if got != "/dev/dri/renderD129" { t.Fatalf("alternate device = %q, want renderD129", got) } + if workload != got { + t.Fatalf("workload device = %q, want the selected device %q", workload, got) + } if active := hwDeviceActiveCount(got); active != 1 { t.Fatalf("alternate device active count = %d, want 1", active) } - if got, releaseNVENC := acquireHWDevice(configured, "nvenc", "/dev/dri/renderD128"); got != "/dev/dri/renderD128" { + if got, _, releaseNVENC := acquireHWDevice(configured, "nvenc", "/dev/dri/renderD128"); got != "/dev/dri/renderD128" { releaseNVENC() t.Fatalf("NVENC retry device = %q, want first configured device", got) } else { diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index 04b33d57d..b460098dc 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -176,10 +176,11 @@ type TranscodeSession struct { // written by a previous generation (or a previous session sharing the // directory) and describes media this process has not produced yet. generationStartedAt time.Time - // reserveHWDeviceOnRestart is true when StartTranscode selected and reserved - // one device from a multi-device QSV/VAAPI setting. Each replacement ffmpeg - // process reacquires that same concrete device. - reserveHWDeviceOnRestart bool + // hwWorkloadDevice is the device this session's GPU workload is counted + // against, or empty when it holds none. Each replacement ffmpeg process + // reacquires this same device rather than re-running selection, so a restart + // keeps its GPU affinity and stays visible in per-device reporting. + hwWorkloadDevice string } // NewTranscodeSessionForTest exposes only the output directory needed by tests @@ -294,11 +295,11 @@ func StartTranscode(ctx context.Context, opts TranscodeOpts) (*TranscodeSession, if err := validateToneMapOpts(opts); err != nil { return nil, err } - configuredHWDevices := ParseHWDeviceSet(opts.HWDevice) - reserveHWDeviceOnRestart := configuredHWDevices.Multi() && hwAccelBalancesRenderDevices(opts.HWAccel) // Resolve a multi-device hw_device list to one concrete GPU. Restarts reuse // the selected device, but each ffmpeg process owns its own reservation. - hwDevice, releaseHWDevice := acquireHWDevice(opts.HWDevice, opts.HWAccel, opts.AvoidHWDevice) + // hwWorkloadDevice is whatever the allocator counted this workload under, so + // the rule for which workloads are counted lives in one place. + hwDevice, hwWorkloadDevice, releaseHWDevice := acquireHWDevice(opts.HWDevice, opts.HWAccel, opts.AvoidHWDevice) opts.HWDevice = hwDevice opts.AvoidHWDevice = "" if err := validateToneMapSource(ctx, opts); err != nil { @@ -321,14 +322,14 @@ func StartTranscode(ctx context.Context, opts TranscodeOpts) (*TranscodeSession, // transcode process outlives a disconnected manifest request. ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) s := &TranscodeSession{ - cancel: cancel, - opts: opts, - outputDir: opts.OutputDir, - running: true, - done: make(chan struct{}), - stderr: newBoundedTailBuffer(stderrTailMaxBytes), - lastRequestedSegment: opts.StartSegmentNumber, - reserveHWDeviceOnRestart: reserveHWDeviceOnRestart, + cancel: cancel, + opts: opts, + outputDir: opts.OutputDir, + running: true, + done: make(chan struct{}), + stderr: newBoundedTailBuffer(stderrTailMaxBytes), + lastRequestedSegment: opts.StartSegmentNumber, + hwWorkloadDevice: hwWorkloadDevice, } args := buildFFmpegArgs(opts) @@ -2613,7 +2614,7 @@ func (s *TranscodeSession) restart( s.stderr.Reset() } s.restartCount++ - reserveHWDevice := s.reserveHWDeviceOnRestart + hwWorkloadDevice := s.hwWorkloadDevice s.mu.Unlock() previousOpts := opts @@ -2670,8 +2671,8 @@ func (s *TranscodeSession) restart( // this session on the same concrete GPU while accounting for the replacement // process as a new active workload. releaseHWDevice := func() {} - if reserveHWDevice { - releaseHWDevice = reserveConcreteHWDevice(opts.HWDevice) + if hwWorkloadDevice != "" { + releaseHWDevice = reserveConcreteHWDevice(hwWorkloadDevice) } // As in StartTranscode, stamp the generation before the process can write. diff --git a/internal/proxy/capability_snapshot_test.go b/internal/proxy/capability_snapshot_test.go index fd9676852..80befa289 100644 --- a/internal/proxy/capability_snapshot_test.go +++ b/internal/proxy/capability_snapshot_test.go @@ -90,7 +90,7 @@ func TestProxyCapabilitiesRejectsIncompleteProbeWithoutPublishing(t *testing.T) server.refreshCapabilitySnapshot(context.Background()) published := decodeProxyHealth(t, server).CapabilitiesHash if published == "" { - t.Fatal("no capability hash was published before the cancelled request") + t.Fatal("no capability hash was published before the canceled request") } canceled, cancel := context.WithCancel(context.Background()) diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go new file mode 100644 index 000000000..a609d68ba --- /dev/null +++ b/internal/proxy/metrics_test.go @@ -0,0 +1,117 @@ +package proxy + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +func newMetricsProxyServer(t *testing.T) *Server { + t.Helper() + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = "proxy-metrics-secret" + cfg.Playback.TranscodeDir = t.TempDir() + w.SetConfigForTest(cfg) + return NewServer(w, nil) +} + +// newFakeSampler answers with a fixed reading so the handlers under test are +// exercised without a Linux host beneath them. +func newFakeSampler() *nodemetrics.Sampler { + return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: time.Now(), + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 1, + VideoBusyPct: 8, Source: nodemetrics.SourceFdinfo, + }}, + }) +} + +// A proxy runs ffmpeg too (remux, Dolby Vision RPU strip), so it reports the +// same resource fields a transcode node does — and, like a transcode node, it +// reads them from a published snapshot rather than measuring on the request. +func TestProxyHealthIncludesResourceSampleWithoutBlocking(t *testing.T) { + server := newMetricsProxyServer(t) + server.metrics = newFakeSampler() + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + answered <- recorder + }() + + var recorder *httptest.ResponseRecorder + select { + case recorder = <-answered: + case <-time.After(5 * time.Second): + t.Fatal("health handler blocked") + } + + var body struct { + Status string `json:"status"` + System *struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + } `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + if body.Status != "ok" { + t.Fatalf("status = %q", body.Status) + } + if body.System == nil || body.System.CPUPct != 41 { + t.Fatalf("system = %+v", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("gpu = %+v", body.GPU) + } +} + +// Without a sampler the response is byte-for-byte what it always was. +func TestProxyHealthOmitsResourceFieldsWithoutASampler(t *testing.T) { + server := newMetricsProxyServer(t) + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + for _, key := range []string{"system", "gpu"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sampler: %s", key, recorder.Body) + } + } +} + +func TestProxyMetricsEndpointIsMountedAndUnauthenticated(t *testing.T) { + server := newMetricsProxyServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("GET /metrics = %d, want 200", recorder.Code) + } + if recorder.Body.Len() == 0 { + t.Fatal("GET /metrics returned an empty body") + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 8f85c5eca..3f08d9f75 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -18,11 +18,13 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/cors" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/downloads" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" @@ -59,6 +61,10 @@ type Server struct { // by /health without probing. Nil until the first snapshot or capability // request completes. capabilityHash atomic.Pointer[string] + + // metrics samples host and GPU resources in the background. Nil until + // StartMetricsSampler runs, which leaves health exactly as it was before. + metrics *nodemetrics.Sampler } type remoteArtifactMissReporter interface { @@ -160,6 +166,10 @@ func (s *Server) Handler() http.Handler { MaxAge: 86400, })) r.Get("/api/v1/health", s.handleHealth) + // Unauthenticated, matching the API listener's own /metrics posture: a + // scrape target that needs a credential is a scrape target that goes + // unmonitored, and the exposure is host resource counters, not media. + r.Method(http.MethodGet, "/metrics", promhttp.Handler()) r.Group(func(r chi.Router) { // Streaming and download bytes count toward the node's measured egress. r.Use(s.meterEgress) @@ -322,6 +332,12 @@ type healthResponse struct { // cheap liveness answer, so it never triggers a probe — and is empty until // the first background snapshot completes. CapabilitiesHash string `json:"capabilities_hash,omitempty"` + // System and GPU are this proxy's last resource sample, read from the + // published snapshot for the same reason as the hash above. A proxy runs + // ffmpeg too (remux, Dolby Vision RPU strip), so it reports GPU usage on the + // same code path a transcode node does. + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { @@ -329,15 +345,57 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { if s.tracker != nil { activeJobs = s.tracker.ActiveCount() } + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(healthResponse{ Status: "ok", ActiveJobs: activeJobs, EgressKbps: s.egress.RateKbps(), CapabilitiesHash: s.storedCapabilityHash(), + System: snapshot.System, + GPU: snapshot.GPU, }) } +// StartMetricsSampler begins background resource sampling until ctx is +// canceled, and publishes the readings on /health, /status and /metrics. +// +// A proxy's only working directory is the subtitle/remux scratch under the +// configured transcode dir, so that is the mount it samples; media roots belong +// to the API host, which is the process that knows what the library is. +func (s *Server) StartMetricsSampler(ctx context.Context) { + if s == nil || ctx == nil { + return + } + scratchDir := "" + if s.watcher != nil { + if cfg := s.watcher.Config(); cfg != nil { + scratchDir = strings.TrimSpace(cfg.Playback.TranscodeDir) + } + } + s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ + ScratchDir: scratchDir, + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: renderDeviceIdentities, + }) + s.metrics.Start(ctx) +} + +// renderDeviceIdentities adapts the playback hardware walk to what the sampler +// needs, so the sampler itself stays free of any playback dependency. +func renderDeviceIdentities() []nodemetrics.DeviceIdentity { + devices := playback.RenderDeviceIdentities() + identities := make([]nodemetrics.DeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, nodemetrics.DeviceIdentity{ + Path: device.Path, + PCIAddress: device.PCIAddress, + Vendor: device.Vendor, + }) + } + return identities +} + // requireBearer checks Authorization: Bearer {secret} for admin endpoints. func (s *Server) requireBearer(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -822,12 +880,17 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { } type statusResponse struct { - ActiveSessions int `json:"active_sessions"` + ActiveSessions int `json:"active_sessions"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(statusResponse{ ActiveSessions: s.tracker.ActiveCount(), + System: snapshot.System, + GPU: snapshot.GPU, }) } diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index e102bf277..e78e9cc08 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -4,6 +4,7 @@ GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media GET /stream/direct/{token} media playback viewer_egress true true HEAD /stream/direct/{token} media playback viewer_egress true true @@ -27,6 +28,7 @@ GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media GET /stream/direct/{token} media playback viewer_egress true true HEAD /stream/direct/{token} media playback viewer_egress true true diff --git a/internal/tonemap/tonemap.go b/internal/tonemap/tonemap.go index b90dc640a..76e11be86 100644 --- a/internal/tonemap/tonemap.go +++ b/internal/tonemap/tonemap.go @@ -665,19 +665,23 @@ func qsvVAAPIInitDevice(device string) string { return "vaapi=va:" + device + ",driver=iHD,kernel_driver=i915,vendor_id=0x8086" } +// initHWDeviceFlag is FFmpeg's hardware-device declaration flag, shared by +// every init chain built here. +const initHWDeviceFlag = "-init_hw_device" + // QSVInitDeviceArgs declares the Intel VAAPI display and derives the QSV // device from it. Every QSV command line in the server — transcode, encoder // warmup, capability probes, tone-map smoke tests, chapter thumbnails — must // initialize hardware through this chain, so a driver constraint is fixed in // one place. func QSVInitDeviceArgs(device string) []string { - return []string{"-init_hw_device", qsvVAAPIInitDevice(device), "-init_hw_device", "qsv=qs@va"} + return []string{initHWDeviceFlag, qsvVAAPIInitDevice(device), initHWDeviceFlag, "qsv=qs@va"} } // VAAPIInitDeviceArgs declares one VAAPI device under the alias the caller's // filter graph and encoder reference. func VAAPIInitDeviceArgs(alias, device string) []string { - return []string{"-init_hw_device", "vaapi=" + alias + ":" + device} + return []string{initHWDeviceFlag, "vaapi=" + alias + ":" + device} } // HDRMetadataRemovalFilter removes side data that would otherwise incorrectly diff --git a/internal/transcodenode/metrics_test.go b/internal/transcodenode/metrics_test.go new file mode 100644 index 000000000..a262eafb7 --- /dev/null +++ b/internal/transcodenode/metrics_test.go @@ -0,0 +1,132 @@ +package transcodenode + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +// newFakeSampler answers with a fixed reading so the handlers under test are +// exercised without a Linux host beneath them. +func newFakeSampler() *nodemetrics.Sampler { + return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: time.Now(), + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, + VideoBusyPct: 63, RenderBusyPct: 12, Source: nodemetrics.SourceFdinfo, + }}, + }) +} + +// Health is what the cluster routes on. Reading the sampler's published +// snapshot — rather than measuring on the request — is what keeps a wedged +// mount or a hung GPU query from turning into a health timeout, so this asserts +// the handler answers promptly and completely. +func TestHealthIncludesResourceSampleWithoutBlocking(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + answered <- recorder + }() + + var recorder *httptest.ResponseRecorder + select { + case recorder = <-answered: + case <-time.After(5 * time.Second): + t.Fatal("health handler blocked") + } + + var body struct { + Status string `json:"status"` + System *struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + Sessions int `json:"sessions"` + } `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + if body.Status != "ok" { + t.Fatalf("status = %q", body.Status) + } + if body.System == nil || body.System.CPUPct != 41 { + t.Fatalf("system = %+v, want the sampled cpu percentage", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" || body.GPU[0].Sessions != 2 { + t.Fatalf("gpu = %+v", body.GPU) + } +} + +// A node with no sampler — a non-Linux host, or one built before sampling — +// must serve exactly the health response it always did. +func TestHealthOmitsResourceFieldsWithoutASampler(t *testing.T) { + server := newTestServer(t) + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + for _, key := range []string{"system", "gpu"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sampler: %s", key, recorder.Body) + } + } + if body["status"] != "ok" { + t.Fatalf("status = %v, want the unchanged response", body["status"]) + } +} + +func TestStatusIncludesResourceSample(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + recorder := httptest.NewRecorder() + server.handleStatus(recorder, httptest.NewRequest(http.MethodGet, "/status", nil)) + + var body struct { + System *json.RawMessage `json:"system"` + GPU *json.RawMessage `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("status body: %v (%s)", err, recorder.Body) + } + if body.System == nil || body.GPU == nil { + t.Fatalf("status omitted the resource sample: %s", recorder.Body) + } +} + +// Operators who scrape must get the same numbers the UI shows, without a +// credential — the same posture the API listener's own /metrics has. +func TestMetricsEndpointIsMountedAndUnauthenticated(t *testing.T) { + server := newTestServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("GET /metrics = %d, want 200", recorder.Code) + } + if recorder.Body.Len() == 0 { + t.Fatal("GET /metrics returned an empty body") + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index cc2159b73..86a572ab2 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -17,12 +17,14 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/sync/singleflight" "github.com/Silo-Server/silo-server/internal/chapterthumbs" "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" @@ -121,6 +123,13 @@ type HealthResponse struct { // cheap liveness answer, so it never triggers a probe — and is empty until // the first background snapshot completes. CapabilitiesHash string `json:"capabilities_hash,omitempty"` + // System and GPU are this node's last resource sample. Like the hash above + // they are read from a snapshot the sampler already published, never + // measured on the request: health is what the cluster routes on, so it must + // answer at the same speed whether or not a mount is hung or a GPU query is + // wedged. Both are omitted on a host that cannot be sampled. + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } // sessionIdleTTL is how long a job may go without a manifest or segment @@ -203,6 +212,10 @@ type Server struct { // by /health without probing. Nil until the first snapshot or capability // request completes. capabilityHash atomic.Pointer[string] + + // metrics samples host and GPU resources in the background. Nil until + // StartMetricsSampler runs, which leaves health exactly as it was before. + metrics *nodemetrics.Sampler } // storedCapabilityHash returns the last published capability hash, or empty @@ -574,6 +587,10 @@ func (s *Server) Handler() http.Handler { s.startIdleReaper() r := chi.NewRouter() r.Get("/api/v1/health", s.handleHealth) + // Unauthenticated, matching the API listener's own /metrics posture: a + // scrape target that needs a credential is a scrape target that goes + // unmonitored, and the exposure is host resource counters, not media. + r.Method(http.MethodGet, "/metrics", promhttp.Handler()) r.Group(func(r chi.Router) { r.Use(s.requireBearer) @@ -908,12 +925,50 @@ func (s *Server) trackDownloadPrepare(ctx context.Context, info nodesessions.Ses } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(HealthResponse{ Status: "ok", ActiveJobs: s.activeJobs.Load(), CapabilitiesHash: s.storedCapabilityHash(), + System: snapshot.System, + GPU: snapshot.GPU, + }) +} + +// StartMetricsSampler begins background resource sampling until ctx is +// canceled, and publishes the readings on /health, /status and /metrics. +// +// The scratch dir is the only mount a transcode node samples: it is the volume +// that silently kills transcodes when it fills, and it is the one path the node +// is guaranteed to be able to see. Media roots are the API host's to report, +// because path visibility varies per node and a node cannot tell "root I cannot +// see" from "root that does not exist". +func (s *Server) StartMetricsSampler(ctx context.Context) { + if s == nil || ctx == nil { + return + } + s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ + ScratchDir: s.transcodeDir, + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: renderDeviceIdentities, }) + s.metrics.Start(ctx) +} + +// renderDeviceIdentities adapts the playback hardware walk to what the sampler +// needs, so the sampler itself stays free of any playback dependency. +func renderDeviceIdentities() []nodemetrics.DeviceIdentity { + devices := playback.RenderDeviceIdentities() + identities := make([]nodemetrics.DeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, nodemetrics.DeviceIdentity{ + Path: device.Path, + PCIAddress: device.PCIAddress, + Vendor: device.Vendor, + }) + } + return identities } // buildCapabilitySnapshot runs the node's full capability detection: hardware @@ -1840,15 +1895,20 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { } s.mu.RUnlock() + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") type statusResponse struct { - Status string `json:"status"` - ActiveJobs int32 `json:"active_jobs"` - Sessions []string `json:"sessions"` + Status string `json:"status"` + ActiveJobs int32 `json:"active_jobs"` + Sessions []string `json:"sessions"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } json.NewEncoder(w).Encode(statusResponse{ Status: "ok", ActiveJobs: s.activeJobs.Load(), Sessions: sessionIDs, + System: snapshot.System, + GPU: snapshot.GPU, }) } diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt index 125f4d408..d8f6a798c 100644 --- a/internal/transcodenode/testdata/media_routes.txt +++ b/internal/transcodenode/testdata/media_routes.txt @@ -7,6 +7,7 @@ GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media @@ -21,6 +22,7 @@ GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media diff --git a/migrations/sql/20260827004808_node_last_stats.sql b/migrations/sql/20260827004808_node_last_stats.sql new file mode 100644 index 000000000..c46251ac4 --- /dev/null +++ b/migrations/sql/20260827004808_node_last_stats.sql @@ -0,0 +1,23 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN last_stats jsonb; + +COMMENT ON COLUMN stream_nodes.last_stats IS + 'Most recent host resource sample reported by this node in its health ' + 'response: {"system": {cpu_pct, load1, cores, mem_used_mb, mem_total_mb, ' + 'disks, net_rx_bps, net_tx_bps}, "gpu": [{device, vendor, sessions, ' + 'video_busy_pct, render_busy_pct, total_busy_pct, vram_used_mb, ' + 'vram_total_mb, source}]}. Written by the same 30s health update that ' + 'writes active_jobs, so it is exactly as old as last_health_check. ' + 'Current sample only — this is not a history table; operators who want ' + 'trends scrape the node /metrics endpoint. NULL when the node reported no ' + 'sample: a build predating resource sampling, a non-Linux host, or a node ' + 'that failed its health check.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS last_stats; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index a697cfdb2..9a21ef85b 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3805,6 +3805,12 @@ export interface NodeDetectedBackend { device?: string; /** Why verification failed, attributed per device when several were tried. */ reason?: string; + /** + * No probe was attempted: none of the backend's candidate devices is + * accessible to this node (e.g. a proxy reading the cluster-wide hw_device). + * Not a driver failure; reason lists the skipped devices. + */ + skipped?: boolean; } /** @@ -3827,6 +3833,84 @@ export interface NodeCapabilities { node_url?: string; } +/** One sampled mount inside a resource sample. */ +export interface HostDiskStats { + path?: string; + /** Capacity in GiB. Used counts filesystem-reserved blocks, matching `df`. */ + used_gb?: number; + total_gb?: number; + /** Real numbers carried over from an earlier pass because the probe has not returned. */ + stale?: boolean; + /** Never measured on this host: `used_gb`/`total_gb` are meaningless. */ + unavailable?: boolean; +} + +/** + * A host's CPU/memory/disk/network sample. Every field is optional: sampling is + * Linux-only, individual probes degrade independently, and a server predating + * resource sampling sends none of this. + */ +export interface HostSystemStats { + /** + * Aggregate busy percentage across all cores over the sampling interval, 0-100. + * Under a cgroup it is that container's own usage against its own quota. + */ + cpu_pct?: number; + /** 1-minute load average; unlike cpu_pct it also counts tasks blocked on storage. */ + load1?: number; + /** CPUs this host may use: the cgroup quota where one is set, otherwise the kernel's count. */ + cores?: number; + mem_used_mb?: number; + mem_total_mb?: number; + /** Scratch dir first, then media roots; deduplicated by filesystem. */ + disks?: HostDiskStats[] | null; + /** Aggregate throughput in *bits* per second, loopback excluded. */ + net_rx_bps?: number; + net_tx_bps?: number; +} + +/** One GPU's sample. */ +export interface HostGPUStats { + /** Render node path (/dev/dri/renderD128), a PCI address, or "cuda:N". */ + device?: string; + vendor?: string; + /** Workloads this host has pinned to the device, from the playback balancer. */ + sessions?: number; + /** Engine busy percentages over the sampling interval. */ + video_busy_pct?: number; + render_busy_pct?: number; + /** Whole-GPU utilization including other tenants. Absent is not zero. */ + total_busy_pct?: number | null; + vram_used_mb?: number | null; + vram_total_mb?: number | null; + /** "fdinfo", "nvidia-smi", "fdinfo+nvidia-smi", or "unavailable". */ + source?: string; +} + +/** + * A node's most recent resource sample, written by the same health check that + * writes `active_jobs` — so it is exactly as old as `last_health_check` and + * never fresher. Absent on a node that reports none, and on every server + * predating resource sampling. + */ +export interface NodeLastStats { + system?: HostSystemStats | null; + gpu?: HostGPUStats[] | null; +} + +/** + * The API host's own sample (GET /admin/system/resources) — the counterpart to + * a node's `last_stats`. `available` is false on a host that cannot be sampled + * (non-Linux, no sampler, or before the first sample lands), in which case the + * rest is absent. + */ +export interface SystemResources { + available?: boolean; + sampled_at?: string; + system?: HostSystemStats | null; + gpu?: HostGPUStats[] | null; +} + export interface StreamNode { id: number; name: string; @@ -3849,6 +3933,8 @@ export interface StreamNode { capabilities_refreshed_at?: string; /** Stable per-GPU identities; two nodes sharing one share hardware. */ physical_gpu_keys?: string[]; + /** The node's resource sample from the last health check. */ + last_stats?: NodeLastStats | null; } export interface CreateNodeRequest { diff --git a/web/src/hooks/queries/admin/system.ts b/web/src/hooks/queries/admin/system.ts index 62052ed8e..da23ef32f 100644 --- a/web/src/hooks/queries/admin/system.ts +++ b/web/src/hooks/queries/admin/system.ts @@ -1,7 +1,16 @@ import { useQuery } from "@tanstack/react-query"; import { api } from "@/api/client"; +import type { SystemResources } from "@/api/types"; import { adminKeys } from "../keys"; +/** + * How often the API host's own sample is re-read. The sampler publishes every + * few seconds and the read costs nothing (it returns an already-published + * snapshot), so this is set by how live an operator expects a resource panel to + * feel, not by what the server can afford. + */ +const SYSTEM_RESOURCES_REFRESH_MS = 15_000; + export interface BuildInfo { display: string; revision: string; @@ -46,6 +55,22 @@ export function useBuildInfo() { }); } +/** + * The API host's own resource sample. `retry: false` because a server predating + * the endpoint 404s and there is nothing to retry into — the caller renders the + * same "not being sampled" state it uses for a non-Linux host. + */ +export function useSystemResources(enabled = true) { + return useQuery({ + queryKey: adminKeys.systemResources(), + queryFn: () => api("/admin/system/resources"), + refetchInterval: SYSTEM_RESOURCES_REFRESH_MS, + staleTime: SYSTEM_RESOURCES_REFRESH_MS, + retry: false, + enabled, + }); +} + export function useHWAccelDetection(enabled = true) { return useQuery({ queryKey: adminKeys.hwAccel(), diff --git a/web/src/hooks/queries/keys.ts b/web/src/hooks/queries/keys.ts index 8dd6b00b3..6ad8d2427 100644 --- a/web/src/hooks/queries/keys.ts +++ b/web/src/hooks/queries/keys.ts @@ -457,6 +457,7 @@ export const adminKeys = { itemImages: (id: string) => ["admin", "items", id, "images"] as const, buildInfo: () => ["admin", "system", "buildInfo"] as const, hwAccel: () => ["admin", "system", "hwAccel"] as const, + systemResources: () => ["admin", "system", "resources"] as const, autoscanSettings: () => ["admin", "autoscan", "settings"] as const, autoscanConnections: () => ["admin", "autoscan", "connections"] as const, autoscanSources: () => ["admin", "autoscan", "sources"] as const, diff --git a/web/src/pages/AdminDashboard.tsx b/web/src/pages/AdminDashboard.tsx index 148c6d9a0..2d26734d3 100644 --- a/web/src/pages/AdminDashboard.tsx +++ b/web/src/pages/AdminDashboard.tsx @@ -6,6 +6,7 @@ import { AdminSectionCommandDialog } from "@/components/AdminSectionCommandDialo import { useEventChannel } from "@/components/realtimeEventsContext"; import { fetchAdminStats, useAdminStats, useAdminSessions } from "@/hooks/queries/admin/stats"; import { useAdminPluginInstallations } from "@/hooks/queries/admin/plugins"; +import { useSystemResources } from "@/hooks/queries/admin/system"; import { usePolicyCapability } from "@/hooks/queries/admin/policy"; import { useAdminUsers } from "@/hooks/queries/admin/users"; import { @@ -60,6 +61,8 @@ import { classifyActivityMethod, getSessionClientLabel, } from "@/pages/adminActivityPresentation"; +import type { ResourceMetric } from "@/pages/adminNodesPresentation"; +import { describeResourceSample } from "@/pages/adminNodesPresentation"; const REFRESH_SPINNER_MIN_VISIBLE_MS = 1_000; const DASHBOARD_AUTO_REFRESH_MS = 60_000; @@ -287,6 +290,8 @@ export default function AdminDashboard() { error={statsQuery.error} /> + + {statsQuery.data?.watch_provider_activity && ( )} @@ -400,6 +405,73 @@ function StatsRow({ ); } +/** + * The API host's own CPU/RAM/disk/GPU, which no other dashboard panel covers: + * every stat card above describes the library, and the Nodes page describes the + * workers. In integrated mode this host is also the transcoder. + */ +function ServerResourcesCard({ canPoll }: { canPoll: boolean }) { + const resourcesQuery = useSystemResources(canPoll); + + // A server predating the endpoint 404s. There is nothing to say about it, and + // an error box on the dashboard would be about our request, not the server. + if (resourcesQuery.isError) { + return null; + } + + const sample = describeResourceSample(resourcesQuery.data); + const sampledLabel = + sample.kind === "sampled" && sample.sampledAt ? getTimeAgo(sample.sampledAt) : null; + + return ( + + + Server resources + {sampledLabel && ( + Sampled {sampledLabel} + )} + + + {/* No body yet — still fetching, or paused with the tab in the + background. Either way the host has not said it cannot be sampled, + so the skeleton is the honest state rather than the empty copy. */} + {resourcesQuery.data === undefined ? ( +
+ {Array.from({ length: 4 }).map((_, i) => ( + + ))} +
+ ) : sample.kind === "unavailable" ? ( +
{sample.title}
+ ) : ( +
+ + + + {sample.gpu ? ( + + ) : ( + + )} +
+ )} + + + ); +} + +function ResourceMetricBox({ metric }: { metric: ResourceMetric }) { + return ( + + ); +} + function TraktActivityCard({ activity }: { activity: WatchProviderActivity }) { const hasActivity = activity.trakt_connected_profiles > 0 || @@ -426,22 +498,22 @@ function TraktActivityCard({ activity }: { activity: WatchProviderActivity }) {
- - - - +
{label}
-
{value}
+
{value}
{detail}
); diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 457b378f6..1706a27cf 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -26,7 +26,9 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import { Plus, Pencil, Trash2, RefreshCw, Info, AlertTriangle } from "lucide-react"; import { ConfirmDialog } from "@/components/ConfirmDialog"; import { formatDateTime } from "@/lib/datetime"; -import { describeNodeGPU } from "./adminNodesPresentation"; +import { cn } from "@/lib/utils"; +import type { ResourceMetric } from "./adminNodesPresentation"; +import { describeNodeGPU, describeNodeSystem } from "./adminNodesPresentation"; type NodeType = "proxy" | "transcode"; @@ -34,6 +36,48 @@ function formatMbps(kbps: number): string { return (Math.round(kbps / 100) / 10).toString(); } +/** One derived reading in the System column: "CPU 42%", muted or tinted. */ +function NodeSystemMetric({ metric }: { metric: ResourceMetric }) { + return ( + + {metric.label} + + {metric.value} + + + ); +} + +function NodeSystemCell({ node }: { node: StreamNode }) { + const system = describeNodeSystem(node); + if (system.kind === "unreported") { + return ( + + {system.label} + + ); + } + + return ( +
+
+ + +
+
+ + +
+
+ ); +} + function NodeGPUCell({ node }: { node: StreamNode }) { const gpu = describeNodeGPU(node); if (gpu.kind === "awaiting") { @@ -78,6 +122,19 @@ function NodeGPUCell({ node }: { node: StreamNode }) { {gpu.deviceSummary}
)} + {gpu.live.map((device) => ( +
+ {device.label} + + {device.busy} + + · {device.sessions} +
+ ))}
); } @@ -108,7 +165,7 @@ function NodeSection({ checkingHealthId, }: NodeSectionProps) { const label = type === "proxy" ? "Proxy" : "Transcode"; - const colCount = (showJobs ? 9 : 8) + (type === "proxy" ? 1 : 0); + const colCount = (showJobs ? 10 : 9) + (type === "proxy" ? 1 : 0); return (
@@ -134,6 +191,7 @@ function NodeSection({ Status Health GPU + System {showJobs && {type === "proxy" ? "Streams" : "Jobs"}} {type === "proxy" && Egress} Last Check @@ -192,6 +250,9 @@ function NodeSection({ + + + {showJobs && ( {node.active_jobs} diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index 7ddac0f63..96641d0a7 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -1,6 +1,14 @@ import { describe, expect, it } from "vitest"; -import type { StreamNode } from "@/api/types"; -import { CAPABILITY_STALE_AFTER_MS, describeNodeGPU } from "./adminNodesPresentation"; +import type { HostSystemStats, StreamNode } from "@/api/types"; +import { + CAPABILITY_STALE_AFTER_MS, + DISK_FILL_WARNING_PCT, + describeGPUBusy, + describeNodeGPU, + describeNodeSystem, + describeResourceSample, + formatBitsPerSecond, +} from "./adminNodesPresentation"; const NOW = Date.parse("2026-08-26T12:00:00Z"); @@ -148,6 +156,37 @@ describe("describeNodeGPU", () => { ]); }); + it("does not warn about skipped backends whose devices are inaccessible", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "none", + detected_backends: [ + { + backend: "qsv", + verified: false, + skipped: true, + reason: "/dev/dri/renderD128: device not accessible on this node", + }, + { + backend: "vaapi", + verified: false, + skipped: true, + reason: "/dev/dri/renderD128: device not accessible on this node", + }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.failures).toEqual([]); + expect(presentation.kind === "reported" && presentation.backend.label).toBe("SW"); + expect(presentation.kind === "reported" && presentation.backend.title).toContain( + "not accessible on this node", + ); + }); + it("falls back to software with no hardware backend resolved", () => { const presentation = describeNodeGPU( makeNode({ @@ -282,6 +321,152 @@ describe("describeNodeGPU", () => { expect(presentation).toMatchObject({ stale: false }); }); + it("reports no live devices for a node whose server sends no last_stats", () => { + const presentation = describeNodeGPU(makeNode({ capabilities: { resolved: "qsv" } }), NOW); + + expect(presentation.kind === "reported" && presentation.live).toEqual([]); + }); + + it("matches a live reading to the inventory device it names", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "qsv", + render_device_details: [{ path: "/dev/dri/renderD128", description: "Intel GPU" }], + }, + last_stats: { + gpu: [ + { + device: "/dev/dri/renderD128", + vendor: "intel", + sessions: 2, + video_busy_pct: 42, + render_busy_pct: 12, + source: "fdinfo", + }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.live).toEqual([ + { + key: "/dev/dri/renderD128", + label: "renderD128", + busy: "42%", + busyMuted: false, + sessions: "2 sessions", + title: ["/dev/dri/renderD128 — Intel GPU", "video 42% · render 12%", "source: fdinfo"].join( + "\n", + ), + }, + ]); + }); + + it("matches a live reading by PCI address when the inventory has no matching path", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { + resolved: "vaapi", + render_device_details: [ + { + path: "/dev/dri/renderD129", + pci_address: "0000:03:00.0", + description: "AMD GPU", + }, + ], + }, + last_stats: { + gpu: [{ device: "0000:03:00.0", sessions: 1, video_busy_pct: 7, source: "fdinfo" }], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.live[0]).toMatchObject({ + label: "0000:03:00.0", + sessions: "1 session", + title: expect.stringContaining("0000:03:00.0 — AMD GPU"), + }); + }); + + it("keeps an unmatched device rather than dropping the reading", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { resolved: "nvenc", render_device_details: [] }, + last_stats: { + gpu: [ + { + device: "cuda:0", + vendor: "nvidia", + sessions: 0, + video_busy_pct: 61, + total_busy_pct: 74, + vram_used_mb: 1024, + vram_total_mb: 8192, + source: "nvidia-smi", + }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.live[0]).toEqual({ + key: "cuda:0", + label: "cuda:0", + busy: "61%", + busyMuted: false, + sessions: "idle", + title: [ + "cuda:0", + "video 61%", + "whole GPU 74% (all tenants)", + "VRAM 1.0 GiB of 8.0 GiB", + "source: nvidia-smi", + ].join("\n"), + }); + }); + + // The zeros an unavailable source reports are placeholders, and an operator + // who reads them as an idle GPU draws the wrong conclusion. + it("mutes the busy percentage when nothing measured the device", () => { + const presentation = describeNodeGPU( + makeNode({ + capabilities: { resolved: "qsv" }, + last_stats: { + gpu: [{ device: "/dev/dri/renderD128", sessions: 1, source: "unavailable" }], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.live[0]).toMatchObject({ + busy: "—", + busyMuted: true, + sessions: "1 session", + title: expect.stringContaining("No source could measure this device"), + }); + }); + + it("drops live readings for an unhealthy node whose sample stopped moving", () => { + const presentation = describeNodeGPU( + makeNode({ + healthy: false, + capabilities: { resolved: "qsv" }, + last_stats: { + gpu: [ + { device: "/dev/dri/renderD128", sessions: 3, video_busy_pct: 90, source: "fdinfo" }, + ], + }, + }), + NOW, + ); + + expect(presentation.kind === "reported" && presentation.live).toEqual([]); + }); + it("tolerates physical_gpu_keys without letting it change the presentation", () => { const capabilities = { resolved: "nvenc", @@ -293,3 +478,238 @@ describe("describeNodeGPU", () => { ).toEqual(describeNodeGPU(makeNode({ capabilities }), NOW)); }); }); + +const FULL_SAMPLE: HostSystemStats = { + cpu_pct: 42, + load1: 1.35, + cores: 8, + mem_used_mb: 12800, + mem_total_mb: 32000, + disks: [{ path: "/tmp/silo-transcode", used_gb: 435, total_gb: 500 }], + net_rx_bps: 12_400_000, + net_tx_bps: 3_100_000, +}; + +describe("describeNodeSystem", () => { + it("explains a healthy node that reports no sample at all", () => { + expect(describeNodeSystem(makeNode())).toEqual({ + kind: "unreported", + label: "—", + title: + "This node reported no resource sample. Sampling is Linux-only, and a node running a build from before resource sampling reports none.", + }); + }); + + it("blames the outage, not the sampler, when an unreachable node has no sample", () => { + expect(describeNodeSystem(makeNode({ healthy: false }))).toMatchObject({ + kind: "unreported", + title: "This node is not answering health checks, so it has no current resource sample.", + }); + }); + + // A frozen CPU percentage is indistinguishable from a live one on screen. + it("shows dashes for an unhealthy node still carrying an older sample", () => { + expect( + describeNodeSystem(makeNode({ healthy: false, last_stats: { system: FULL_SAMPLE } })), + ).toMatchObject({ + kind: "unreported", + label: "—", + title: expect.stringContaining("no longer current"), + }); + }); + + it("derives every reading from a complete sample", () => { + const system = describeNodeSystem(makeNode({ last_stats: { system: FULL_SAMPLE } })); + + expect(system).toMatchObject({ + kind: "reported", + cpu: { label: "CPU", value: "42%", detail: "8 cores · load 1.35", muted: false }, + memory: { + label: "RAM", + value: "12.5 GiB of 31.3 GiB", + detail: "40% used", + muted: false, + }, + disk: { + label: "Disk", + value: "87%", + detail: "/tmp/silo-transcode", + title: "/tmp/silo-transcode — 87% full (435.0 GiB of 500.0 GiB)", + muted: false, + warning: true, + }, + network: { label: "Net", value: "↓ 12.4 Mbps · ↑ 3.1 Mbps", muted: false }, + }); + }); + + it("mutes only the readings a partial sample is missing", () => { + const system = describeNodeSystem( + makeNode({ last_stats: { system: { cpu_pct: 12, mem_total_mb: 0, disks: [] } } }), + ); + + expect(system).toMatchObject({ + kind: "reported", + cpu: { value: "12%", detail: "", muted: false }, + memory: { value: "—", muted: true, title: "This sample carries no memory reading." }, + disk: { value: "—", muted: true, title: "This sample carries no disk reading." }, + network: { value: "—", muted: true, title: "This sample carries no network reading." }, + }); + }); + + it("warns exactly at the disk fill threshold and not one point below", () => { + const atThreshold = describeNodeSystem( + makeNode({ + last_stats: { + system: { disks: [{ path: "/scratch", used_gb: DISK_FILL_WARNING_PCT, total_gb: 100 }] }, + }, + }), + ); + const below = describeNodeSystem( + makeNode({ + last_stats: { + system: { + disks: [{ path: "/scratch", used_gb: DISK_FILL_WARNING_PCT - 1, total_gb: 100 }], + }, + }, + }), + ); + + expect(atThreshold).toMatchObject({ disk: { value: "85%", warning: true } }); + expect(below).toMatchObject({ disk: { value: "84%", warning: false } }); + }); + + it("reports the fullest mount and keeps every mount in the tooltip", () => { + const system = describeNodeSystem( + makeNode({ + last_stats: { + system: { + disks: [ + { path: "/tmp/silo-transcode", used_gb: 10, total_gb: 100 }, + { path: "/media/movies", used_gb: 95, total_gb: 100, stale: true }, + { path: "/media/gone", unavailable: true }, + ], + }, + }, + }), + ); + + expect(system).toMatchObject({ + disk: { + value: "95%", + detail: "/media/movies", + warning: true, + title: [ + "/tmp/silo-transcode — 10% full (10.0 GiB of 100.0 GiB)", + "/media/movies — 95% full (95.0 GiB of 100.0 GiB), carried over from an earlier pass", + "/media/gone — unavailable on this host", + ].join("\n"), + }, + }); + }); + + it("names the mount that went away instead of showing a bare dash", () => { + const system = describeNodeSystem( + makeNode({ last_stats: { system: { disks: [{ path: "/media", unavailable: true }] } } }), + ); + + expect(system).toMatchObject({ + disk: { value: "—", muted: true, title: "/media — unavailable on this host" }, + }); + }); +}); + +describe("formatBitsPerSecond", () => { + it("scales a bits-per-second rate to the unit an operator reads", () => { + expect(formatBitsPerSecond(0)).toBe("0 bps"); + expect(formatBitsPerSecond(940)).toBe("940 bps"); + expect(formatBitsPerSecond(12_500)).toBe("13 kbps"); + expect(formatBitsPerSecond(12_400_000)).toBe("12.4 Mbps"); + expect(formatBitsPerSecond(2_500_000_000)).toBe("2.5 Gbps"); + }); + + it("has nothing to say about an absent or impossible rate", () => { + expect(formatBitsPerSecond(undefined)).toBeNull(); + expect(formatBitsPerSecond(null)).toBeNull(); + expect(formatBitsPerSecond(-1)).toBeNull(); + expect(formatBitsPerSecond(Number.NaN)).toBeNull(); + }); +}); + +describe("describeGPUBusy", () => { + it("reports nothing for a host with no GPU rather than an idle one", () => { + expect(describeGPUBusy([])).toBeNull(); + }); + + it("reports the busiest video engine and the total pinned sessions", () => { + expect( + describeGPUBusy([ + { device: "/dev/dri/renderD128", video_busy_pct: 42, sessions: 2, source: "fdinfo" }, + { device: "cuda:0", video_busy_pct: 71, sessions: 1, source: "nvidia-smi" }, + ]), + ).toMatchObject({ + label: "GPU", + value: "71%", + detail: "busiest of 2 GPUs · 3 sessions", + muted: false, + title: [ + "/dev/dri/renderD128 — video 42% · 2 sessions", + "cuda:0 — video 71% · 1 session", + ].join("\n"), + }); + }); + + it("mutes the tile when no device could be measured", () => { + expect( + describeGPUBusy([{ device: "/dev/dri/renderD128", sessions: 0, source: "unavailable" }]), + ).toMatchObject({ + value: "—", + muted: true, + title: "/dev/dri/renderD128 — not measured · 0 sessions", + }); + }); +}); + +describe("describeResourceSample", () => { + it("treats a server with no such endpoint as an unsampled host", () => { + expect(describeResourceSample(undefined)).toMatchObject({ kind: "unavailable" }); + }); + + it("treats an explicit available:false the same way", () => { + expect(describeResourceSample({ available: false })).toMatchObject({ kind: "unavailable" }); + }); + + it("does not claim a sample when available is true but the body carries none", () => { + expect(describeResourceSample({ available: true })).toMatchObject({ kind: "unavailable" }); + }); + + it("derives the host readings and omits the GPU tile when there is no GPU", () => { + const sample = describeResourceSample({ + available: true, + sampled_at: "2026-08-26T12:00:00Z", + system: FULL_SAMPLE, + }); + + expect(sample).toMatchObject({ + kind: "sampled", + cpu: { value: "42%" }, + memory: { value: "12.5 GiB of 31.3 GiB" }, + disk: { value: "87%", warning: true }, + gpu: null, + sampledAt: "2026-08-26T12:00:00Z", + }); + }); + + it("carries the GPU reading through when the host reports one", () => { + const sample = describeResourceSample({ + available: true, + system: FULL_SAMPLE, + gpu: [{ device: "/dev/dri/renderD128", video_busy_pct: 30, sessions: 1, source: "fdinfo" }], + }); + + expect(sample).toMatchObject({ + kind: "sampled", + gpu: { value: "30%", detail: "video engine · 1 session" }, + sampledAt: null, + }); + }); +}); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index ddd715feb..cd10f8f71 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -1,4 +1,13 @@ -import type { NodeCapabilities, NodeRenderDevice, StreamNode } from "@/api/types"; +import type { + HostDiskStats, + HostGPUStats, + HostSystemStats, + NodeCapabilities, + NodeRenderDevice, + StreamNode, + SystemResources, +} from "@/api/types"; +import { formatFileSize } from "@/lib/mediaFormat"; /** * How long a stored capability report may go unconfirmed before it is called @@ -47,8 +56,31 @@ export type NodeGPUPresentation = deviceTitle: string | null; /** No health check has re-confirmed this report recently. */ stale: boolean; + /** + * Live per-device readings from the node's last health check, matched + * against the capability inventory. Empty when the node reports no + * sample — a server or node predating resource sampling renders exactly + * as it did before, rather than showing zeros it never measured. + */ + live: NodeGPULiveDevice[]; }; +/** One GPU's live reading, as rendered next to the capability inventory. */ +export interface NodeGPULiveDevice { + /** Stable key for list rendering; the device id as the node reported it. */ + key: string; + /** Short device name: "renderD128", or the raw id like "cuda:0". */ + label: string; + /** Video-engine busy percentage, or a dash when nothing measured it. */ + busy: string; + /** No measurement behind `busy`: render it muted, since it is not a zero. */ + busyMuted: boolean; + /** "2 sessions", or "idle" for none. */ + sessions: string; + /** Hover text: device identity, both engines, whole-GPU/VRAM, and source. */ + title: string; +} + const BACKEND_BADGE_CLASS: Record = { verified: "bg-success/10 text-success border-success/15", failed: "bg-warning/10 text-warning border-warning/15", @@ -79,11 +111,120 @@ export function describeNodeGPU(node: StreamNode, now: number = Date.now()): Nod deviceSummary: devices.summary, deviceTitle: devices.title, stale: isCapabilityReportStale(node, now), + live: describeLiveGPUs(node), }; } +/** + * Live GPU readings for a node, matched to its capability inventory. + * + * An unhealthy node contributes none: its sample is from before the check that + * failed, and a busy percentage that stopped moving reads as a live one. + */ +function describeLiveGPUs(node: StreamNode): NodeGPULiveDevice[] { + if (!node.healthy) { + return []; + } + const details = node.capabilities?.render_device_details ?? []; + return (node.last_stats?.gpu ?? []).map((stats) => describeLiveGPU(stats, details)); +} + +function describeLiveGPU( + stats: HostGPUStats, + details: readonly NodeRenderDevice[], +): NodeGPULiveDevice { + const device = stats.device?.trim() ?? ""; + // The inventory speaks /dev/dri paths; a sample for a GPU with no readable + // DRM node names a PCI address or a cuda index instead, which is why the + // fallback match exists rather than the path lookup alone. + const detail = + details.find((candidate) => candidate.path?.trim() === device && device !== "") ?? + details.find((candidate) => candidate.pci_address?.trim() === device && device !== ""); + const measured = isMeasuredGPUSource(stats.source); + const video = finiteNumber(stats.video_busy_pct); + const sessions = Math.max(0, Math.trunc(finiteNumber(stats.sessions) ?? 0)); + + return { + key: device || (detail?.path ?? "gpu"), + label: shortDeviceLabel(device, detail), + busy: measured && video != null ? `${clampPercent(video)}%` : DASH, + busyMuted: !measured || video == null, + sessions: sessions === 1 ? "1 session" : sessions === 0 ? "idle" : `${sessions} sessions`, + title: liveGPUTitle(stats, detail, device, measured), + }; +} + +function liveGPUTitle( + stats: HostGPUStats, + detail: NodeRenderDevice | undefined, + device: string, + measured: boolean, +): string { + const identity = [ + device || detail?.path?.trim() || "(unknown device)", + detail?.description?.trim(), + ] + .filter((part): part is string => !!part) + .join(" — "); + const lines = [identity]; + + if (measured) { + const engines = [ + formatEngine("video", stats.video_busy_pct), + formatEngine("render", stats.render_busy_pct), + ].filter((part): part is string => part !== null); + if (engines.length > 0) { + lines.push(engines.join(" · ")); + } + } else { + // Zeros with no measurement behind them must not read as an idle GPU. + lines.push("No source could measure this device on the last sample."); + } + + const whole = finiteNumber(stats.total_busy_pct); + if (whole != null) { + lines.push(`whole GPU ${clampPercent(whole)}% (all tenants)`); + } + const vram = formatUsedOfTotal(stats.vram_used_mb, stats.vram_total_mb, mebibytesToBytes); + if (vram) { + lines.push(`VRAM ${vram}`); + } + const source = stats.source?.trim(); + if (source) { + lines.push(`source: ${source}`); + } + return lines.join("\n"); +} + +function formatEngine(name: string, value: number | null | undefined): string | null { + const percent = finiteNumber(value); + return percent == null ? null : `${name} ${clampPercent(percent)}%`; +} + +function isMeasuredGPUSource(source: string | undefined): boolean { + const normalized = source?.trim().toLowerCase() ?? ""; + return normalized !== "" && normalized !== "unavailable"; +} + +function shortDeviceLabel(device: string, detail: NodeRenderDevice | undefined): string { + const path = device || detail?.path?.trim() || ""; + if (path === "") { + return "GPU"; + } + const tail = path.split("/").filter((part) => part !== ""); + return tail[tail.length - 1] ?? path; +} + function describeBackend(resolved: string, detected: readonly NodeDetected[]): NodeGPUBackendBadge { if (resolved === "" || resolved === "none") { + const skipped = detected.filter((entry) => entry.skipped); + if (skipped.length > 0 && skipped.length === detected.length) { + return badge( + "SW", + "none", + "Encoding in software — the configured GPU devices are not accessible on this node.", + ); + } return badge("SW", "none", "No hardware backend verified — encoding in software."); } @@ -121,7 +262,9 @@ function otherFailures(resolved: string, detected: readonly NodeDetected[]): Nod return detected .filter((entry) => { const backend = entry.backend?.trim().toLowerCase() ?? ""; - return !entry.verified && backend !== "" && backend !== resolved; + // Skipped entries were never probed (their devices are not accessible + // on this node) — expected on proxies, so they do not warrant a warning. + return !entry.verified && !entry.skipped && backend !== "" && backend !== resolved; }) .map((entry) => ({ label: (entry.backend?.trim() ?? "").toUpperCase(), @@ -203,3 +346,368 @@ function describeDeviceLine(device: NodeRenderDevice): string { const address = device.pci_address?.trim(); return address ? `${line} (${address})` : line; } + +// --- Host resource samples ------------------------------------------------- +// +// A node's last_stats and the API host's /admin/system/resources carry the same +// shapes, so both surfaces derive their numbers here. Nothing below invents a +// value: a field the sampler could not measure renders as a dash, never as a +// zero, because a zero on a dashboard is read as "measured and idle". + +/** Placeholder for a reading the sample does not carry. */ +const DASH = "—"; + +/** + * Fill percentage at which a mount is called out. A transcode scratch volume + * that fills stops transcodes with no other warning, and the last few percent + * of a volume disappear fast under a segment writer, so the threshold sits far + * enough below full to leave an operator time to act. + */ +export const DISK_FILL_WARNING_PCT = 85; + +/** One derived reading, ready to render in a table cell or a stat tile. */ +export interface ResourceMetric { + label: string; + /** Rendered value, or a dash when `muted`. */ + value: string; + /** Short secondary line for the dashboard tiles; empty when there is none. */ + detail: string; + /** Hover text explaining the value, or why there is none. */ + title: string; + /** Nothing measured this reading: render it in the muted color. */ + muted: boolean; + /** Past an attention threshold; render with the warning tint. */ + warning: boolean; +} + +export type NodeSystemPresentation = + | { + kind: "unreported"; + /** Dash, so the column keeps its shape. */ + label: string; + title: string; + } + | { + kind: "reported"; + cpu: ResourceMetric; + memory: ResourceMetric; + disk: ResourceMetric; + network: ResourceMetric; + }; + +/** + * Describe a node's System column from the sample its last health check + * carried. + * + * An unhealthy node reports nothing rather than its last numbers: the sample + * predates the check that failed, and a frozen CPU percentage is indis- + * tinguishable from a live one on screen. + */ +export function describeNodeSystem(node: StreamNode): NodeSystemPresentation { + const system = node.last_stats?.system; + if (!system) { + return { + kind: "unreported", + label: DASH, + title: node.healthy + ? "This node reported no resource sample. Sampling is Linux-only, and a node running a build from before resource sampling reports none." + : "This node is not answering health checks, so it has no current resource sample.", + }; + } + if (!node.healthy) { + return { + kind: "unreported", + label: DASH, + title: + "The last health check did not reach this node, so its most recent resource sample is no longer current.", + }; + } + return describeSystemStats(system); +} + +/** Derive the four host readings from one system sample. */ +export function describeSystemStats(system: HostSystemStats): { + kind: "reported"; + cpu: ResourceMetric; + memory: ResourceMetric; + disk: ResourceMetric; + network: ResourceMetric; +} { + return { + kind: "reported", + cpu: describeCPU(system), + memory: describeMemory(system), + disk: describeWorstDisk(system.disks ?? []), + network: describeNetwork(system), + }; +} + +function describeCPU(system: HostSystemStats): ResourceMetric { + const cpu = finiteNumber(system.cpu_pct); + const cores = finiteNumber(system.cores); + const load1 = finiteNumber(system.load1); + if (cpu == null) { + return mutedMetric("CPU", "This sample carries no CPU reading."); + } + + const percent = clampPercent(cpu); + const detail = [ + cores != null && cores > 0 ? (cores === 1 ? "1 core" : `${cores} cores`) : null, + load1 != null ? `load ${load1.toFixed(2)}` : null, + ] + .filter((part): part is string => part !== null) + .join(" · "); + + const title = [ + `${percent}% busy across all cores over the last sampling interval.`, + cores != null && cores > 0 + ? `${cores} CPU(s) available to this host — its cgroup quota where one is set.` + : null, + load1 != null + ? `1-minute load ${load1.toFixed(2)}, which also counts tasks blocked on storage.` + : null, + ] + .filter((part): part is string => part !== null) + .join(" "); + + return { label: "CPU", value: `${percent}%`, detail, title, muted: false, warning: false }; +} + +function describeMemory(system: HostSystemStats): ResourceMetric { + const used = finiteNumber(system.mem_used_mb); + const total = finiteNumber(system.mem_total_mb); + if (used == null || total == null || total <= 0) { + return mutedMetric("RAM", "This sample carries no memory reading."); + } + + const value = formatUsedOfTotal(used, total, mebibytesToBytes) ?? DASH; + const percent = clampPercent((used / total) * 100); + return { + label: "RAM", + value, + detail: `${percent}% used`, + title: `${value} used (${percent}%). Under a cgroup this is the container's limit and working set, not the host's.`, + muted: false, + warning: false, + }; +} + +/** + * The fullest sampled mount, which for a transcode node is its scratch dir — + * the only mount it samples. Reporting the worst rather than the first keeps + * the same rule working on the API host, which also samples the media roots. + */ +export function describeWorstDisk(disks: readonly HostDiskStats[]): ResourceMetric { + const measured = disks + .map((disk) => ({ disk, fill: diskFillPercent(disk) })) + .filter((entry): entry is { disk: HostDiskStats; fill: number } => entry.fill !== null); + + const title = disks.length > 0 ? disks.map(describeDiskLine).join("\n") : ""; + if (measured.length === 0) { + // Paths that exist but could not be measured are still worth naming in the + // tooltip, so an operator sees which mount went away rather than a dash + // with no explanation. + return mutedMetric("Disk", title === "" ? "This sample carries no disk reading." : title); + } + + const worst = measured.reduce((a, b) => (b.fill > a.fill ? b : a)); + const path = worst.disk.path?.trim() ?? ""; + return { + label: "Disk", + value: `${worst.fill}%`, + detail: path === "" ? "full" : path, + title, + muted: false, + warning: worst.fill >= DISK_FILL_WARNING_PCT, + }; +} + +function describeNetwork(system: HostSystemStats): ResourceMetric { + const rx = formatBitsPerSecond(system.net_rx_bps); + const tx = formatBitsPerSecond(system.net_tx_bps); + if (rx === null && tx === null) { + return mutedMetric("Net", "This sample carries no network reading."); + } + + return { + label: "Net", + value: `↓ ${rx ?? DASH} · ↑ ${tx ?? DASH}`, + detail: "rx · tx", + title: `Aggregate throughput with loopback excluded: ${rx ?? DASH} in, ${tx ?? DASH} out. In a container this is the container's own network namespace.`, + muted: false, + warning: false, + }; +} + +export type ResourceSamplePresentation = + | { + kind: "unavailable"; + title: string; + } + | { + kind: "sampled"; + cpu: ResourceMetric; + memory: ResourceMetric; + disk: ResourceMetric; + network: ResourceMetric; + /** Busiest GPU's video engine; null when this host reports no GPU. */ + gpu: ResourceMetric | null; + /** When the sample was taken, for a freshness label; null when unstamped. */ + sampledAt: string | null; + }; + +/** + * Describe the API host's own sample. A server predating the endpoint answers + * 404 and leaves `resources` undefined, which is the same story as a host that + * cannot be sampled: no numbers, said plainly, rather than an error. + */ +export function describeResourceSample( + resources: SystemResources | undefined | null, +): ResourceSamplePresentation { + const system = resources?.system; + if (!resources || resources.available !== true || !system) { + return { + kind: "unavailable", + title: + "This host is not being sampled. Resource sampling is Linux-only, and the first sample lands a few seconds after startup.", + }; + } + + return { + ...describeSystemStats(system), + kind: "sampled", + gpu: describeGPUBusy(resources.gpu ?? []), + sampledAt: resources.sampled_at?.trim() || null, + }; +} + +/** + * The busiest GPU's video engine, which is the one an operator asks about when + * transcodes queue. Averaging would hide a saturated card behind an idle one. + * Null when the host reports no GPU at all, so the tile is omitted rather than + * showing a zero. + */ +export function describeGPUBusy(gpu: readonly HostGPUStats[]): ResourceMetric | null { + if (gpu.length === 0) { + return null; + } + + const sessions = gpu.reduce((total, stats) => total + (finiteNumber(stats.sessions) ?? 0), 0); + const sessionLabel = sessions === 1 ? "1 session" : `${sessions} sessions`; + const title = gpu + .map((stats) => { + const device = stats.device?.trim() || "(unknown device)"; + const measured = isMeasuredGPUSource(stats.source); + const video = finiteNumber(stats.video_busy_pct); + const reading = measured && video != null ? `video ${clampPercent(video)}%` : "not measured"; + const count = Math.max(0, Math.trunc(finiteNumber(stats.sessions) ?? 0)); + return `${device} — ${reading} · ${count === 1 ? "1 session" : `${count} sessions`}`; + }) + .join("\n"); + + const busiest = gpu + .filter((stats) => isMeasuredGPUSource(stats.source)) + .map((stats) => finiteNumber(stats.video_busy_pct)) + .filter((value): value is number => value !== null) + .reduce((best, value) => (best === null || value > best ? value : best), null); + + if (busiest === null) { + return mutedMetric("GPU", title === "" ? "No GPU reading in this sample." : title); + } + + const label = gpu.length === 1 ? "video engine" : `busiest of ${gpu.length} GPUs`; + return { + label: "GPU", + value: `${clampPercent(busiest)}%`, + detail: `${label} · ${sessionLabel}`, + title, + muted: false, + warning: false, + }; +} + +function describeDiskLine(disk: HostDiskStats): string { + const path = disk.path?.trim() || "(unknown path)"; + if (disk.unavailable) { + return `${path} — unavailable on this host`; + } + const fill = diskFillPercent(disk); + if (fill === null) { + return `${path} — no capacity reported`; + } + const capacity = formatUsedOfTotal(disk.used_gb, disk.total_gb, gibibytesToBytes); + const line = `${path} — ${fill}% full${capacity ? ` (${capacity})` : ""}`; + // "Real but old" is the normal reading for a network mount whose server went + // away, and it is a different fact from "never measured". + return disk.stale ? `${line}, carried over from an earlier pass` : line; +} + +function diskFillPercent(disk: HostDiskStats): number | null { + if (disk.unavailable) { + return null; + } + const used = finiteNumber(disk.used_gb); + const total = finiteNumber(disk.total_gb); + if (used == null || total == null || total <= 0) { + return null; + } + return clampPercent((used / total) * 100); +} + +/** "12.4 GiB of 31.3 GiB", or null when either side is missing. */ +function formatUsedOfTotal( + used: number | null | undefined, + total: number | null | undefined, + toBytes: (value: number) => number, +): string | null { + const usedValue = finiteNumber(used); + const totalValue = finiteNumber(total); + if (usedValue == null || totalValue == null || totalValue <= 0) { + return null; + } + const usedLabel = formatFileSize(toBytes(usedValue), { iecUnits: true, fallback: "0 B" }); + const totalLabel = formatFileSize(toBytes(totalValue), { iecUnits: true }); + return `${usedLabel} of ${totalLabel}`; +} + +/** + * Humanize a *bits*-per-second rate, which is the unit the sampler reports and + * the unit egress is already quoted in elsewhere in the cluster. Decimal scale, + * because network rates are decimal everywhere they are quoted. + */ +export function formatBitsPerSecond(bps: number | null | undefined): string | null { + const value = finiteNumber(bps); + if (value == null || value < 0) { + return null; + } + if (value >= 1e9) { + return `${(value / 1e9).toFixed(1)} Gbps`; + } + if (value >= 1e6) { + return `${(value / 1e6).toFixed(1)} Mbps`; + } + if (value >= 1e3) { + return `${Math.round(value / 1e3)} kbps`; + } + return `${Math.round(value)} bps`; +} + +function mebibytesToBytes(value: number): number { + return value * 1024 ** 2; +} + +function gibibytesToBytes(value: number): number { + return value * 1024 ** 3; +} + +function mutedMetric(label: string, title: string): ResourceMetric { + return { label, value: DASH, detail: "", title, muted: true, warning: false }; +} + +function finiteNumber(value: number | null | undefined): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function clampPercent(value: number): number { + return Math.min(100, Math.max(0, Math.round(value))); +} From 10e6cbb156fa69f502077400dca3dd8a47989c5b Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:59:12 -0400 Subject: [PATCH 005/163] feat(nodepool): per-node hw_accel and hw_device overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New nullable stream_nodes columns hw_accel_override (enum-checked) and hw_device_override let a heterogeneous pool configure acceleration per node; NULL inherits the cluster-wide playback settings. A node overlays its own row's overrides onto the cluster config on every nodeconfig reload, matched by NODE_URL against the unique url column (trailing-slash tolerant, deterministic on ties, conservative on lookup failure). Remote transcode dispatch sends Node.EffectiveHWAccel — the override when set, else the cluster value — so auto still reaches the node for live resolution and a stale capability report can never pin a backend; jellycompat dispatch gains the same rule via a planner node lookup. PUT /admin/nodes/{id} accepts both fields with explicit-null clearing and case-insensitive enum validation (400 on bad values). The node edit dialog gains "Inherit cluster setting" controls, the GPU cell shows the override source, and the Playback Settings divergence warning now points at per-node overrides. Changes hot-apply in stages (dispatch immediately, node config within a reload, snapshots within 15m); only boot-time warmup and in-flight sessions wait for a restart — documented in docs/admin-api.md. Phase 4 of the node GPU observability plan. Related issue: #780 Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 33 +- docs/admin-api.md | 54 +++- internal/api/handlers/nodes.go | 4 + internal/api/handlers/nodes_test.go | 116 ++++++- internal/api/handlers/playback_v3.go | 5 +- .../handlers/playback_v3_node_hwaccel_test.go | 116 +++++++ internal/jellycompat/handlers_playback.go | 30 +- .../remote_dispatch_hwaccel_test.go | 105 +++++++ internal/nodeconfig/watcher.go | 173 ++++++++++- .../nodeconfig/watcher_overrides_db_test.go | 95 ++++++ internal/nodeconfig/watcher_overrides_test.go | 188 ++++++++++++ internal/nodeconfig/watcher_test.go | 13 +- internal/nodepool/planner.go | 15 + internal/nodepool/repository.go | 154 +++++++++- .../nodepool/repository_capabilities_test.go | 11 +- .../nodepool/repository_hw_overrides_test.go | 284 ++++++++++++++++++ .../sql/20260827025521_node_hw_overrides.sql | 32 ++ web/src/api/types.ts | 13 + web/src/hooks/queries/admin/nodes.ts | 12 +- web/src/pages/AdminNodes.tsx | 99 +++++- .../admin-settings/PlaybackSettings.test.tsx | 52 ++++ .../pages/admin-settings/PlaybackSettings.tsx | 11 +- web/src/pages/adminNodesPresentation.test.ts | 65 ++++ web/src/pages/adminNodesPresentation.ts | 74 +++++ 24 files changed, 1705 insertions(+), 49 deletions(-) create mode 100644 internal/api/handlers/playback_v3_node_hwaccel_test.go create mode 100644 internal/jellycompat/remote_dispatch_hwaccel_test.go create mode 100644 internal/nodeconfig/watcher_overrides_db_test.go create mode 100644 internal/nodeconfig/watcher_overrides_test.go create mode 100644 internal/nodepool/repository_hw_overrides_test.go create mode 100644 migrations/sql/20260827025521_node_hw_overrides.sql diff --git a/cmd/silo/main.go b/cmd/silo/main.go index fe01914b7..fb6c6cf9f 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -850,12 +850,31 @@ func main() { streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, redisClient) streamTelemetryRegistry.Start(appCtx) + // Resolved before the watcher starts: NODE_URL is this process's + // stream_nodes identity, and the watcher needs it on its very first + // load to overlay the node's own acceleration overrides. + nodeURL := os.Getenv("NODE_URL") + nodeName := os.Getenv("NODE_NAME") + if nodeURL == "" { + nodeURL = "http://localhost" + cfg.Server.Listen + // The guess is this process's whole identity: it keys session + // bookkeeping *and* selects the stream_nodes row whose acceleration + // overrides this node adopts. Two nodes listening on the same port + // guess the same URL and would share both. + slog.Warn("NODE_URL not set, using listen address — session keys may collide across nodes, and this node adopts the acceleration overrides of whichever stream_nodes row carries that URL", + "node_url", nodeURL) + } + if nodeName == "" { + nodeName = mode + } + bootstrap := nodeconfig.BootstrapOverrides{ Listen: cfg.Server.Listen, Mode: cfg.Server.Mode, DatabaseURL: cfg.Database.URL, JFListen: cfg.JellyfinCompat.Listen, RedisURL: bc.RedisURL, + NodeURL: nodeURL, } watcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, bootstrap) if err := watcher.Start(appCtx); err != nil { @@ -863,16 +882,6 @@ func main() { os.Exit(1) } - nodeURL := os.Getenv("NODE_URL") - nodeName := os.Getenv("NODE_NAME") - if nodeURL == "" { - nodeURL = "http://localhost" + cfg.Server.Listen - slog.Warn("NODE_URL not set, using listen address — session keys may collide across nodes") - } - if nodeName == "" { - nodeName = mode - } - tracker := nodesessions.NewTracker(redisClient, nodeURL, nodeName, mode) tracker.StartRefresh(appCtx) defer func() { @@ -949,7 +958,9 @@ func main() { // Hot-reload config watcher for integrated/api mode. Reloads on // EventSettingsChanged (Redis) with a 60s poll fallback, so settings // changes apply without restart even on Redis-less deployments. The - // watcher's config supersedes the startup snapshot from here on. + // watcher's config supersedes the startup snapshot from here on. No + // NodeURL: an API host is not a stream node and has no row whose + // acceleration overrides could apply to it. configWatcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, nodeconfig.BootstrapOverrides{ Listen: bc.Listen, Mode: bc.Mode, diff --git a/docs/admin-api.md b/docs/admin-api.md index 970469ddd..ee9aa6658 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -31,6 +31,45 @@ Always `200 OK` with a JSON array. | `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | | `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | +| `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | + +### Acceleration overrides + +`hw_accel_override` and `hw_device_override` override the cluster-wide +`playback.hw_accel` and `playback.hw_device` settings for one node. +`hw_accel_override` takes the same values as the cluster setting — `auto`, +`qsv`, `vaapi`, `nvenc`, `none`. Absent means inherit; there is no separate +"inherit" value to set. + +They exist for a heterogeneous deployment: one CPU-only node in a QSV cluster +sets `none` for itself instead of forcing every node onto the lowest common +denominator. A homogeneous deployment should leave both unset and configure +`playback.hw_accel` once. + +A node finds its own row by URL: `NODE_URL` on the node is matched against +`stream_nodes.url`, ignoring a trailing slash on either side. Set `NODE_URL` +explicitly on every node. Without it a node guesses `http://localhost:` +and adopts whatever row carries that URL, which on a multi-node deployment can +be a different machine's policy. + +The node overlays its row onto the cluster-wide playback settings on every +config reload, so the override is what that node probes with, advertises in +`capabilities.resolved`, and falls back to when a start request names no +backend. The API dispatches remote transcodes with the node's +`hw_accel_override` in preference to its own cluster setting, so the request +agrees with what the node would have run anyway. Dispatch reads the override +column, not `capabilities.resolved`: a node inheriting `auto` is dispatched +`auto` so it resolves against live hardware at session start rather than +against a snapshot. + +**A changed override applies without a restart, but not all at once.** Dispatch +picks it up as soon as the update returns, because the pools are reloaded. The +node applies it to new transcodes on its next config reload (within 60 seconds) +and re-advertises `capabilities.resolved` at its next capability snapshot +(every 15 minutes). Two things do wait for a restart: the hardware encoder +warmup that ran at boot, which stays primed for the old backend, and sessions +already transcoding, which keep the backend they started with. Restart the node +when you want all four in agreement immediately. ### `last_stats` @@ -127,8 +166,19 @@ Updates a node's mutable fields. Every field is optional; an omitted field is left unchanged. An empty-string `group` clears the group, and a non-positive `max_jobs` or `max_bandwidth_kbps` clears that cap. -`200 OK` with the updated node, `404 Not Found` for an unknown id. The node -pools are reloaded afterwards. +`hw_accel_override` and `hw_device_override` are writable here. Either `null` +or an empty string clears one, restoring inheritance of the cluster-wide +setting; an omitted field leaves it alone. Clearing an override is a real +change with a real effect, so it is deliberately expressible rather than being +indistinguishable from omission. + +`200 OK` with the updated node, `404 Not Found` for an unknown id, +`400 Bad Request` when `hw_accel_override` is not one of `auto`, `qsv`, +`vaapi`, `nvenc`, `none` (matched case-insensitively and stored lowercase, as +`playback.hw_accel` is). The node pools are reloaded afterwards, so remote +dispatch honors a new override immediately; the target node itself picks it up +on its next config reload — see "Acceleration overrides" above for what waits +for a restart. Capability fields are not writable here. They are owned by the health sweep, because only the node can say what hardware it has. diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index cd74105c8..c98c940ec 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -200,6 +200,10 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusNotFound, "not_found", "Node not found") return } + if errors.Is(err, nodepool.ErrInvalidNodeInput) { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } slog.ErrorContext(r.Context(), "updating node", "component", "api", "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update node") return diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 6164d8423..d0be01da4 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -6,10 +6,12 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" "testing" "time" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/go-chi/chi/v5" ) // An NVIDIA uuid identifies a card wherever it is plugged in; a PCI address @@ -63,6 +65,10 @@ func TestPhysicalGPUKeys(t *testing.T) { type stubNodeRepository struct { nodes []*nodepool.Node + // updateResult is what Update returns once validation passes; nil keeps the + // default "unknown node" answer. + updateResult *nodepool.Node + updated *nodepool.UpdateNodeInput } func (s *stubNodeRepository) List(context.Context) ([]*nodepool.Node, error) { return s.nodes, nil } @@ -75,8 +81,17 @@ func (s *stubNodeRepository) Create(context.Context, nodepool.CreateNodeInput) ( return nil, nodepool.ErrNodeNotFound } -func (s *stubNodeRepository) Update(context.Context, int, nodepool.UpdateNodeInput) (*nodepool.Node, error) { - return nil, nodepool.ErrNodeNotFound +// Update mirrors Repository.Update's order of operations — validate, then +// write — so handler tests see the same errors production returns. +func (s *stubNodeRepository) Update(_ context.Context, _ int, input nodepool.UpdateNodeInput) (*nodepool.Node, error) { + if err := input.Validate(); err != nil { + return nil, err + } + s.updated = &input + if s.updateResult == nil { + return nil, nodepool.ErrNodeNotFound + } + return s.updateResult, nil } func (s *stubNodeRepository) Delete(context.Context, int) error { return nil } @@ -142,3 +157,100 @@ func TestHandleListNodesIncludesCapabilities(t *testing.T) { } } } + +func updateNodeRequest(t *testing.T, body string) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/admin/nodes/1", strings.NewReader(body)) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("id", "1") + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) +} + +// A per-node override may only name a backend the cluster-wide setting could +// also name. Rejecting it here is what turns a CHECK-constraint violation into +// an answer the admin UI can show. +func TestHandleUpdateNodeRejectsUnknownHWAccelOverride(t *testing.T) { + repo := &stubNodeRepository{updateResult: &nodepool.Node{ID: 1, Name: "gpu-1"}} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleUpdateNode(recorder, updateNodeRequest(t, `{"hw_accel_override":"videotoolbox"}`)) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body = %s", recorder.Code, recorder.Body.String()) + } + if repo.updated != nil { + t.Fatalf("rejected input still reached the store: %+v", repo.updated) + } + if !strings.Contains(recorder.Body.String(), "hw_accel_override") { + t.Fatalf("error body does not name the field: %s", recorder.Body.String()) + } +} + +// Setting an override and clearing it again are both ordinary updates; the +// clear has to survive JSON decoding as a clear rather than as "unchanged". +func TestHandleUpdateNodeAcceptsAndClearsHWOverrides(t *testing.T) { + accel, device := "vaapi", "/dev/dri/renderD129" + tests := []struct { + name string + body string + wantAccel *string + wantDevice *string + }{ + { + name: "sets both", + body: `{"hw_accel_override":"vaapi","hw_device_override":"/dev/dri/renderD129"}`, + wantAccel: &accel, + wantDevice: &device, + }, + { + name: "explicit null clears both", + body: `{"hw_accel_override":null,"hw_device_override":null}`, + wantAccel: new(string), + wantDevice: new(string), + }, + { + name: "omitted leaves both alone", + body: `{"name":"gpu-1"}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stored := "qsv" + repo := &stubNodeRepository{updateResult: &nodepool.Node{ID: 1, Name: "gpu-1", HWAccelOverride: &stored}} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleUpdateNode(recorder, updateNodeRequest(t, test.body)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", recorder.Code, recorder.Body.String()) + } + if repo.updated == nil { + t.Fatal("update never reached the store") + } + if !equalStringPointer(repo.updated.HWAccelOverride, test.wantAccel) { + t.Fatalf("HWAccelOverride = %v, want %v", repo.updated.HWAccelOverride, test.wantAccel) + } + if !equalStringPointer(repo.updated.HWDeviceOverride, test.wantDevice) { + t.Fatalf("HWDeviceOverride = %v, want %v", repo.updated.HWDeviceOverride, test.wantDevice) + } + // The response is the stored row, so the admin UI sees the effective + // policy without a second read. + var response map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response["hw_accel_override"] != "qsv" { + t.Fatalf("response hw_accel_override = %v, want the stored value", response["hw_accel_override"]) + } + }) + } +} + +func equalStringPointer(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 70cdaa127..21e3f182d 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -2825,7 +2825,10 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla } sourceMetadata := sourceExecutionMetadataV3(file, result) sourceProfile, sourceBitDepth := sourceVideoTranscodeFactsV3(file, result) - hwAccel := h.playbackConfig().HWAccel + // The node's own override wins over this host's cluster-wide setting; every + // other node gets the cluster value verbatim, "auto" included, so the node + // still resolves it against live hardware at session start. + hwAccel := node.EffectiveHWAccel(h.playbackConfig().HWAccel) toneMapFilter := "" if result.ToneMapMode != "" { capabilities, err := h.remoteToneMapCapabilitiesV3(r.Context(), node.URL, false) diff --git a/internal/api/handlers/playback_v3_node_hwaccel_test.go b/internal/api/handlers/playback_v3_node_hwaccel_test.go new file mode 100644 index 000000000..da5e7d3a1 --- /dev/null +++ b/internal/api/handlers/playback_v3_node_hwaccel_test.go @@ -0,0 +1,116 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// A node under a per-node acceleration override resolves to its own backend no +// matter what the request says, so a request carrying this API host's +// cluster-wide value would be silently corrected on arrival — and the recipe +// card would then describe an encode that never ran. Every other node keeps the +// cluster value verbatim: a node honors a named backend without re-checking it, +// so "auto" has to survive dispatch to reach live detection on the node, and a +// stale capability report must never stand in for it. +func TestPrepareRemoteTransportV3DispatchesTheNodesEffectiveBackend(t *testing.T) { + override := func(value string) *string { return &value } + tests := []struct { + name string + cluster string + override *string + capabilities string + want string + }{ + { + name: "node overridden to software wins over a qsv cluster", + cluster: "qsv", + override: override("none"), + want: "none", + }, + { + name: "node overridden to other hardware wins too", + cluster: "qsv", + override: override("nvenc"), + want: "nvenc", + }, + { + name: "an override lands immediately, ahead of the stale report it contradicts", + cluster: "qsv", + override: override("none"), + // The node was still reporting qsv when the operator disabled it. + capabilities: `{"resolved":"qsv"}`, + want: "none", + }, + { + name: "a node with no override keeps the cluster value", + cluster: "qsv", + want: "qsv", + }, + { + name: "a blank override is not an override", + cluster: "qsv", + override: override(" "), + want: "qsv", + }, + { + name: "auto reaches the node so it resolves against live hardware", + cluster: "auto", + // A boot-time probe that ran before the render devices were + // attached must not pin later sessions to software. + capabilities: `{"resolved":"none","render_devices":[]}`, + want: "auto", + }, + { + name: "a stale report never overrides the cluster value", + cluster: "qsv", + capabilities: `{"resolved":"none"}`, + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var got transcodenode.TranscodeStartRequest + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/transcode/start" { + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decode remote start: %v", err) + } + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{SessionID: got.SessionID, Status: "started", AudioRecipeVersion: got.AudioRecipeVersion}) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer node.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{TranscodeEnabled: true, HWAccel: test.cluster} + } + pooled := &nodepool.Node{URL: node.URL, HWAccelOverride: test.override} + if test.capabilities != "" { + pooled.Capabilities = json.RawMessage(test.capabilities) + } + transport, transportErr := handler.prepareRemoteTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-node-hwaccel", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), remoteHLSResultV3(), + nodepool.Plan{TranscodeNode: pooled}, preparedTimelineV3{}, mediaAuthModeV3{}, + ) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + if got.HWAccel != test.want { + t.Fatalf("dispatched hw_accel = %q, want %q", got.HWAccel, test.want) + } + }) + } +} diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index f1ca50b20..24f105472 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -145,6 +145,14 @@ type compatTranscodeNodeHealth interface { TranscodeNodeHealthy(nodeURL string) bool } +// compatTranscodeNodeLookup resolves the pooled record behind a transcode node +// URL, which carries that node's own acceleration override. Optional, like the +// enumerators above: without it dispatch falls back to the cluster-wide +// acceleration setting. *nodepool.Planner implements it. +type compatTranscodeNodeLookup interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) +} + // transcodeStreamDetailsSetter is implemented by the native SessionManager. // Optional (like sessionStarterContext) so lightweight test fakes don't have // to; without it the session keeps transport-level defaults only. @@ -1032,6 +1040,26 @@ func clampSeekSeconds(seekSeconds float64, sources []PlaybackMediaSource) float6 return seekSeconds } +// remoteDispatchHWAccel picks the acceleration backend to name in a start +// request to one node: that node's own hw_accel_override when it carries one, +// and otherwise the cluster-wide setting this host runs under. A node under an +// override resolves to its own answer regardless, so naming it keeps the +// request and what runs in agreement. The cluster value passes through +// untouched otherwise — "auto" included, because the node honors a named +// backend verbatim and must be left to resolve it against live hardware. This +// mirrors the v1 dispatch path in internal/api/handlers/playback_v3.go. +func (h *PlaybackHandler) remoteDispatchHWAccel(nodeURL string) string { + lookup, ok := h.NodePlanner.(compatTranscodeNodeLookup) + if !ok { + return h.HWAccel + } + node, found := lookup.TranscodeNodeByURL(nodeURL) + if !found { + return h.HWAccel + } + return node.EffectiveHWAccel(h.HWAccel) +} + // startRemoteTranscode submits a frozen compatibility recipe to a selected node. func (h *PlaybackHandler) startRemoteTranscode( ctx context.Context, @@ -1177,7 +1205,7 @@ func (h *PlaybackHandler) startRemoteTranscodeWithToneMapMode( TargetCodecVideo: compatTargetVideoCodec, TargetCodecAudio: compatTargetAudioCodec, SegmentDuration: segmentDuration, - HWAccel: h.HWAccel, + HWAccel: h.remoteDispatchHWAccel(transcodeNodeURL), AudioTrackIndex: compatAudioTrackIndexOrDefault(source), SourceAudioChannels: compatSourceAudioChannels(source), TotalDuration: float64(source.Version.Duration), diff --git a/internal/jellycompat/remote_dispatch_hwaccel_test.go b/internal/jellycompat/remote_dispatch_hwaccel_test.go new file mode 100644 index 000000000..ba8478bad --- /dev/null +++ b/internal/jellycompat/remote_dispatch_hwaccel_test.go @@ -0,0 +1,105 @@ +package jellycompat + +import ( + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/nodepool" +) + +// nodeLookupPlannerStub is a planner that can also resolve a pooled node from +// its URL, which is what carries the node's own acceleration override. +type nodeLookupPlannerStub struct { + node *nodepool.Node +} + +func (s *nodeLookupPlannerStub) PlanSession(string, string, bool, int) nodepool.Plan { + return nodepool.Plan{TranscodeNode: s.node} +} + +func (s *nodeLookupPlannerStub) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if s.node == nil || s.node.URL != nodeURL { + return nil, false + } + return s.node, true +} + +// planner without a lookup, standing in for a test fake or a future planner +// that cannot resolve nodes. +type plainPlannerStub struct{} + +func (plainPlannerStub) PlanSession(string, string, bool, int) nodepool.Plan { return nodepool.Plan{} } + +// The Jellyfin surface dispatches to the same nodes as the native API, so it +// has to name the same backend: the node's own override when it has one, and +// the cluster value untouched otherwise. +func TestRemoteDispatchHWAccelPrefersTheNodesOverride(t *testing.T) { + override := func(value string) *string { return &value } + overriddenNode := func(cluster string, node *nodepool.Node) *PlaybackHandler { + return &PlaybackHandler{HWAccel: cluster, NodePlanner: &nodeLookupPlannerStub{node: node}} + } + tests := []struct { + name string + handler *PlaybackHandler + nodeURL string + want string + }{ + { + name: "node overridden to software wins over a qsv cluster", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", HWAccelOverride: override("none")}), + nodeURL: "http://node-1", + want: "none", + }, + { + name: "an override beats the stale report it contradicts", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", + HWAccelOverride: override("none"), + Capabilities: json.RawMessage(`{"resolved":"qsv"}`)}), + nodeURL: "http://node-1", + want: "none", + }, + { + name: "a node with no override keeps the cluster value", + handler: overriddenNode("qsv", &nodepool.Node{URL: "http://node-1"}), + nodeURL: "http://node-1", + want: "qsv", + }, + { + // The node re-resolves auto against live hardware at session start; + // a report from its last snapshot must not stand in for that. + name: "auto reaches the node even when the last report says otherwise", + handler: overriddenNode("auto", &nodepool.Node{ + URL: "http://node-1", Capabilities: json.RawMessage(`{"resolved":"none"}`)}), + nodeURL: "http://node-1", + want: "auto", + }, + { + name: "unknown node keeps the cluster value", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", HWAccelOverride: override("none")}), + nodeURL: "http://node-2", + want: "qsv", + }, + { + name: "planner without a lookup keeps the cluster value", + handler: &PlaybackHandler{HWAccel: "qsv", NodePlanner: plainPlannerStub{}}, + nodeURL: "http://node-1", + want: "qsv", + }, + { + name: "no planner at all keeps the cluster value", + handler: &PlaybackHandler{HWAccel: "qsv"}, + nodeURL: "http://node-1", + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.handler.remoteDispatchHWAccel(test.nodeURL); got != test.want { + t.Fatalf("remoteDispatchHWAccel(%q) = %q, want %q", test.nodeURL, got, test.want) + } + }) + } +} diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index ebc070b28..9567b9db1 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -2,6 +2,7 @@ package nodeconfig import ( "context" + "errors" "fmt" "log/slog" "reflect" @@ -24,8 +25,27 @@ type BootstrapOverrides struct { DatabaseURL string // from DATABASE_URL env JFListen string // from JF_PORT env RedisURL string // from REDIS_URL env + // NodeURL is this process's own stream_nodes identity (from NODE_URL env), + // set only in proxy/transcode mode. It is what lets a node find its own row + // and overlay that row's acceleration overrides onto the cluster-wide + // playback settings. Empty on an API host, which has no stream_nodes row. + NodeURL string } +// nodeHWOverrides is one node's own acceleration policy, as stored on its +// stream_nodes row. A nil field means the node inherits the cluster-wide +// playback setting. +type nodeHWOverrides struct { + HWAccel *string + HWDevice *string +} + +// loadNodeHWOverrides reads one node's overrides by URL. found is false when +// the node has no stream_nodes row at all — a legitimate deployment (a node +// nobody registered yet), not an error. It is a field on the Watcher so the +// overlay can be exercised without a database. +type loadNodeHWOverrides func(ctx context.Context, nodeURL string) (overrides nodeHWOverrides, found bool, err error) + // Watcher watches for configuration changes in the database and // automatically reloads the Config when changes are detected. type Watcher struct { @@ -37,6 +57,18 @@ type Watcher struct { bootstrap BootstrapOverrides onChange []func(old, updated *config.Config) reloadCh chan struct{} // buffered(1), event bus writes here + + // loadOverrides reads this node's own acceleration overrides; see the + // overlay in applySettings. + loadOverrides loadNodeHWOverrides + // overrides is the last successfully read overlay, kept so a database + // hiccup during a reload cannot silently flip a node back onto the + // cluster-wide backend. overridesLoaded distinguishes "read, and there is + // nothing to overlay" from "never read". + overrides nodeHWOverrides + overridesLoaded bool + missingRowLogged bool + duplicateRowLogged bool } // NewWatcher creates a new config watcher. Call Start to begin watching. The @@ -44,13 +76,15 @@ type Watcher struct { // before they reach config.LoadFromDB, so a hot reload never feeds ciphertext // into the live config (which would, e.g., break JWT validation). func NewWatcher(pool *pgxpool.Pool, cipher *secret.Cipher, eventBus cache.EventBus, bootstrap BootstrapOverrides) *Watcher { - return &Watcher{ + w := &Watcher{ pool: pool, cipher: cipher, eventBus: eventBus, bootstrap: bootstrap, reloadCh: make(chan struct{}, 1), } + w.loadOverrides = w.queryNodeHWOverrides + return w } // Config returns the current config. Safe for concurrent use. @@ -128,7 +162,7 @@ func (w *Watcher) reload(ctx context.Context) error { if err != nil { return err } - return w.applySettings(m) + return w.applySettings(ctx, m) } // fetchSettings reads all server_settings rows and decrypts sensitive values. @@ -161,9 +195,10 @@ func (w *Watcher) fetchSettings(ctx context.Context) (map[string]string, error) } // applySettings builds a Config from a plaintext settings map, re-applies -// bootstrap overrides, swaps the config pointer, and notifies OnChange -// callbacks when the config actually changed. -func (w *Watcher) applySettings(m map[string]string) error { +// bootstrap overrides, overlays this node's own acceleration policy, swaps the +// config pointer, and notifies OnChange callbacks when the config actually +// changed. +func (w *Watcher) applySettings(ctx context.Context, m map[string]string) error { newCfg, err := config.LoadFromDB(m) if err != nil { return fmt.Errorf("parse config: %w", err) @@ -186,6 +221,10 @@ func (w *Watcher) applySettings(m map[string]string) error { newCfg.Redis.URL = w.bootstrap.RedisURL } + // Last word, after the bootstrap re-apply: the node's own row decides its + // acceleration policy, and nothing above may put the cluster value back. + w.applyNodeHWOverrides(ctx, newCfg) + w.mu.Lock() old := w.cfg w.cfg = newCfg @@ -206,6 +245,130 @@ func (w *Watcher) applySettings(m map[string]string) error { return nil } +// applyNodeHWOverrides overlays this node's stored acceleration policy onto a +// freshly built config, so everything downstream — probes, warmup, the node's +// own fallback when a start request omits a backend — reads one effective +// value rather than consulting the row separately. +// +// It is a no-op on a host with no node identity (the API server, which has no +// stream_nodes row). Failure is deliberately conservative: a node that cannot +// read its row keeps the overlay it last read, because a database hiccup is +// not evidence that an operator cleared the override. +func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) { + if w.bootstrap.NodeURL == "" || w.loadOverrides == nil || cfg == nil { + return + } + + overrides, found, err := w.loadOverrides(ctx, w.bootstrap.NodeURL) + switch { + case err != nil: + slog.WarnContext(ctx, "node acceleration override lookup failed; keeping the previous effective policy", + "component", "nodeconfig", "error", err) + var loaded bool + w.mu.RLock() + overrides, loaded = w.overrides, w.overridesLoaded + w.mu.RUnlock() + if !loaded { + // Never read one: the cluster-wide settings stand as they are. + return + } + case !found: + // Logged once rather than on every 60s reload: an unregistered node is + // a standing condition, not an event. + w.mu.Lock() + first := !w.missingRowLogged + w.missingRowLogged = true + w.overrides, w.overridesLoaded = nodeHWOverrides{}, true + w.mu.Unlock() + if first { + slog.InfoContext(ctx, "no stream_nodes row for this node; inheriting the cluster acceleration settings", + "component", "nodeconfig", "node_url", w.bootstrap.NodeURL) + } + return + default: + w.mu.Lock() + w.overrides, w.overridesLoaded = overrides, true + w.mu.Unlock() + } + + if overrides.HWAccel != nil { + cfg.Playback.HWAccel = *overrides.HWAccel + } + if overrides.HWDevice != nil { + cfg.Playback.HWDevice = *overrides.HWDevice + } +} + +// queryNodeHWOverrides reads this node's own stream_nodes row. The URL is +// matched with trailing slashes ignored on both sides, because NODE_URL and +// the registered URL are typed by different people; the scan this costs is +// irrelevant next to getting the match wrong and silently inheriting the +// cluster policy. +// +// That tolerance can match two rows, though: stream_nodes.url is unique on the +// exact string, so "http://n1" and "http://n1/" are two legal registrations +// that rtrim collapses into one key. Ordering by id makes the winner the same +// on every reload — without it the seq scan returns whichever row it reached +// first, and the 30-second health sweep rewriting those rows would silently +// flip a node between two policies. The duplicate is reported rather than +// quietly resolved, because only an operator can fix it. +func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL string) (nodeHWOverrides, bool, error) { + if w.pool == nil { + return nodeHWOverrides{}, false, errors.New("no database pool") + } + rows, err := w.pool.Query(ctx, + `SELECT url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE rtrim(url, '/') = rtrim($1, '/') ORDER BY id LIMIT 2`, nodeURL) + if err != nil { + return nodeHWOverrides{}, false, fmt.Errorf("query node acceleration overrides: %w", err) + } + defer rows.Close() + + var ( + overrides nodeHWOverrides + matched []string + ) + for rows.Next() { + var ( + url string + row nodeHWOverrides + ) + if err := rows.Scan(&url, &row.HWAccel, &row.HWDevice); err != nil { + return nodeHWOverrides{}, false, fmt.Errorf("scan node acceleration overrides: %w", err) + } + if len(matched) == 0 { + overrides = row + } + matched = append(matched, url) + } + if err := rows.Err(); err != nil { + return nodeHWOverrides{}, false, fmt.Errorf("read node acceleration overrides: %w", err) + } + if len(matched) == 0 { + return nodeHWOverrides{}, false, nil + } + if len(matched) > 1 { + w.logDuplicateNodeRows(ctx, nodeURL, matched) + } + return overrides, true, nil +} + +// logDuplicateNodeRows warns, once per process, that more than one +// stream_nodes row claims this node's URL. Once rather than every 60s: the +// duplicate is a standing misconfiguration, and the lowest id keeps winning +// until an operator removes the other row. +func (w *Watcher) logDuplicateNodeRows(ctx context.Context, nodeURL string, matched []string) { + w.mu.Lock() + first := !w.duplicateRowLogged + w.duplicateRowLogged = true + w.mu.Unlock() + if !first { + return + } + slog.WarnContext(ctx, "several stream_nodes rows match this node's URL; using the lowest id", + "component", "nodeconfig", "node_url", nodeURL, "matched_urls", matched) +} + // poll runs the background loop that reloads config on timer or event. func (w *Watcher) poll(ctx context.Context) { ticker := time.NewTicker(60 * time.Second) diff --git a/internal/nodeconfig/watcher_overrides_db_test.go b/internal/nodeconfig/watcher_overrides_db_test.go new file mode 100644 index 000000000..f4d128851 --- /dev/null +++ b/internal/nodeconfig/watcher_overrides_db_test.go @@ -0,0 +1,95 @@ +package nodeconfig + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// newOverrideTestPool follows the repository-wide convention for tests that +// need a real database: skip unless one is configured, and skip again if it +// predates the migration under test rather than failing on a missing column. +func newOverrideTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + var columns int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM information_schema.columns + WHERE table_name = 'stream_nodes' AND column_name = 'hw_accel_override'`).Scan(&columns); err != nil || columns < 1 { + t.Skip("test database has not applied the stream_nodes override migration") + } + return pool +} + +func insertOverrideNode(t *testing.T, pool *pgxpool.Pool, url string, accel *string) int { + t.Helper() + ctx := context.Background() + var id int + if err := pool.QueryRow(ctx, + `INSERT INTO stream_nodes (name, type, url, hw_accel_override) + VALUES ($1, 'transcode', $2, $3) RETURNING id`, + fmt.Sprintf("override-%d", time.Now().UnixNano()), url, accel).Scan(&id); err != nil { + t.Fatalf("insert node %q: %v", url, err) + } + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `DELETE FROM stream_nodes WHERE id = $1`, id); err != nil { + t.Errorf("cleanup node %d: %v", id, err) + } + }) + return id +} + +// stream_nodes.url is unique on the exact string, so a node registered twice — +// once with a trailing slash — is two legal rows that the lookup's rtrim +// tolerance collapses into one key. The winner has to be the same on every +// reload: without an explicit order the seq scan returns whichever row it +// reaches first, and the 30-second health sweep rewriting those rows would +// silently flip the node between two acceleration policies mid-deployment. +func TestQueryNodeHWOverridesPicksTheSameRowAcrossReloads(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + base := fmt.Sprintf("http://dup-node-%d:8082", time.Now().UnixNano()) + + pinned := "none" + firstID := insertOverrideNode(t, pool, base, &pinned) + insertOverrideNode(t, pool, base+"/", nil) + + w := &Watcher{pool: pool} + assertPinned := func(stage string) { + t.Helper() + overrides, found, err := w.queryNodeHWOverrides(ctx, base) + if err != nil { + t.Fatalf("%s: lookup: %v", stage, err) + } + if !found { + t.Fatalf("%s: node row not found", stage) + } + if overrides.HWAccel == nil || *overrides.HWAccel != pinned { + t.Fatalf("%s: hw_accel override = %v, want the lowest-id row's %q", stage, overrides.HWAccel, pinned) + } + } + + assertPinned("initial load") + + // What the health sweep does every 30 seconds. It moves the tuple, and with + // it the physical order an unordered scan would have followed. + if _, err := pool.Exec(ctx, + `UPDATE stream_nodes SET healthy = NOT healthy, last_health_check = NOW() WHERE id = $1`, firstID); err != nil { + t.Fatalf("health sweep update: %v", err) + } + + assertPinned("after a health sweep rewrote the row") +} diff --git a/internal/nodeconfig/watcher_overrides_test.go b/internal/nodeconfig/watcher_overrides_test.go new file mode 100644 index 000000000..3f2d5abe5 --- /dev/null +++ b/internal/nodeconfig/watcher_overrides_test.go @@ -0,0 +1,188 @@ +package nodeconfig + +import ( + "context" + "errors" + "testing" +) + +// clusterSettings is what every node in a QSV deployment loads before its own +// row is consulted. +func clusterSettings() map[string]string { + return map[string]string{ + "playback.hw_accel": "qsv", + "playback.hw_device": "/dev/dri/renderD128", + } +} + +func newOverrideWatcher(t *testing.T, nodeURL string, load loadNodeHWOverrides) *Watcher { + t.Helper() + w := NewWatcher(nil, nil, nil, BootstrapOverrides{NodeURL: nodeURL}) + w.loadOverrides = load + return w +} + +// A node's own row wins over the cluster-wide settings: this is the whole +// mechanism behind a CPU-only box living in a hardware-accelerated cluster. +func TestApplySettingsOverlaysNodeHWOverrides(t *testing.T) { + accel, device := "none", "/dev/dri/renderD129" + tests := []struct { + name string + overrides nodeHWOverrides + wantAccel string + wantDevice string + }{ + { + name: "both overridden", + overrides: nodeHWOverrides{HWAccel: &accel, HWDevice: &device}, + wantAccel: "none", + wantDevice: "/dev/dri/renderD129", + }, + { + name: "backend only, device still inherited", + overrides: nodeHWOverrides{HWAccel: &accel}, + wantAccel: "none", + wantDevice: "/dev/dri/renderD128", + }, + { + name: "device only, backend still inherited", + overrides: nodeHWOverrides{HWDevice: &device}, + wantAccel: "qsv", + wantDevice: "/dev/dri/renderD129", + }, + { + name: "row with no overrides inherits both", + wantAccel: "qsv", + wantDevice: "/dev/dri/renderD128", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + return test.overrides, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != test.wantAccel || cfg.Playback.HWDevice != test.wantDevice { + t.Fatalf("effective policy = %q / %q, want %q / %q", + cfg.Playback.HWAccel, cfg.Playback.HWDevice, test.wantAccel, test.wantDevice) + } + }) + } +} + +// The API host has no stream_nodes row and must never pay for a lookup. +func TestApplySettingsSkipsOverlayWithoutNodeIdentity(t *testing.T) { + looked := false + w := newOverrideWatcher(t, "", func(context.Context, string) (nodeHWOverrides, bool, error) { + looked = true + return nodeHWOverrides{}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if looked { + t.Fatal("a host with no NodeURL queried stream_nodes") + } + if got := w.Config().Playback.HWAccel; got != "qsv" { + t.Fatalf("HWAccel = %q, want the cluster value", got) + } +} + +// An unregistered node keeps the cluster settings, and says so once rather +// than on every 60-second reload. +func TestApplySettingsMissingRowInheritsAndLogsOnce(t *testing.T) { + calls := 0 + w := newOverrideWatcher(t, "http://node-unregistered", func(context.Context, string) (nodeHWOverrides, bool, error) { + calls++ + return nodeHWOverrides{}, false, nil + }) + for range 3 { + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } + if calls != 3 { + t.Fatalf("lookups = %d, want one per reload", calls) + } + w.mu.RLock() + logged := w.missingRowLogged + w.mu.RUnlock() + if !logged { + t.Fatal("missing row was never recorded, so it would be logged on every reload") + } +} + +// A database hiccup must not flip a node back onto the cluster-wide backend: +// an unreadable row is not evidence that an operator cleared the override. +func TestApplySettingsKeepsPreviousOverrideWhenTheLookupFails(t *testing.T) { + accel := "nvenc" + fail := false + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + if fail { + return nodeHWOverrides{}, false, errors.New("connection refused") + } + return nodeHWOverrides{HWAccel: &accel}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("first apply: %v", err) + } + if got := w.Config().Playback.HWAccel; got != "nvenc" { + t.Fatalf("HWAccel = %q, want the stored override", got) + } + + fail = true + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply during outage: %v", err) + } + if got := w.Config().Playback.HWAccel; got != "nvenc" { + t.Fatalf("HWAccel = %q after a failed lookup, want the last known override", got) + } +} + +// Before any successful read there is nothing to keep, so the cluster settings +// stand rather than an invented value. +func TestApplySettingsFallsBackToClusterWhenNoOverrideWasEverRead(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{}, false, errors.New("connection refused") + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} + +// The overlay is the last word: a bootstrap re-apply happens before it, and +// nothing may put the cluster value back afterwards. +func TestApplySettingsOverlayOutlivesBootstrapReapply(t *testing.T) { + accel := "none" + w := NewWatcher(nil, nil, nil, BootstrapOverrides{ + NodeURL: "http://node-1", + Listen: ":9999", + Mode: "transcode", + }) + w.loadOverrides = func(context.Context, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + settings := clusterSettings() + settings["server.listen"] = ":8080" + if err := w.applySettings(context.Background(), settings); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Server.Listen != ":9999" || cfg.Server.Mode != "transcode" { + t.Fatalf("bootstrap overrides lost: listen %q mode %q", cfg.Server.Listen, cfg.Server.Mode) + } + if cfg.Playback.HWAccel != "none" { + t.Fatalf("HWAccel = %q, want the node override", cfg.Playback.HWAccel) + } +} diff --git a/internal/nodeconfig/watcher_test.go b/internal/nodeconfig/watcher_test.go index 4c67a764e..104fe007c 100644 --- a/internal/nodeconfig/watcher_test.go +++ b/internal/nodeconfig/watcher_test.go @@ -1,6 +1,7 @@ package nodeconfig import ( + "context" "testing" "github.com/Silo-Server/silo-server/internal/config" @@ -20,7 +21,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { }) settings := map[string]string{"server.log_level": "debug"} - if err := w.applySettings(settings); err != nil { + if err := w.applySettings(context.Background(), settings); err != nil { t.Fatalf("first apply: %v", err) } if calls != 1 { @@ -28,7 +29,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { } // Same settings again — pointer swaps, but callbacks must not fire. - if err := w.applySettings(settings); err != nil { + if err := w.applySettings(context.Background(), settings); err != nil { t.Fatalf("second apply: %v", err) } if calls != 1 { @@ -36,7 +37,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { } // A real change fires callbacks again. - if err := w.applySettings(map[string]string{"server.log_level": "warn"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "warn"}); err != nil { t.Fatalf("third apply: %v", err) } if calls != 2 { @@ -50,7 +51,7 @@ func TestApplySettingsReappliesBootstrapOverrides(t *testing.T) { RedisURL: "redis://env-host:6379", }) - err := w.applySettings(map[string]string{ + err := w.applySettings(context.Background(), map[string]string{ "server.listen": ":8080", "redis.url": "redis://db-host:6379", }) @@ -84,7 +85,7 @@ func TestRequestReloadCoalesces(t *testing.T) { func TestOnChangeAfterFirstApplySeesLaterChanges(t *testing.T) { w := newTestWatcher(t, BootstrapOverrides{}) - if err := w.applySettings(map[string]string{"server.log_level": "info"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "info"}); err != nil { t.Fatalf("initial apply: %v", err) } @@ -94,7 +95,7 @@ func TestOnChangeAfterFirstApplySeesLaterChanges(t *testing.T) { gotNew = updated.Server.LogLevel }) - if err := w.applySettings(map[string]string{"server.log_level": "error"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "error"}); err != nil { t.Fatalf("second apply: %v", err) } if gotOld != "info" || gotNew != "error" { diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 62d332135..21de9f504 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -125,6 +125,21 @@ func (p *Planner) TranscodeNode(nodeID int) (*Node, bool) { return nil, false } +// TranscodeNodeByURL returns the pooled record for a transcode node URL. It +// gives a dispatch path that node's own acceleration override, from the URL it +// is about to send a job to. Enabled or not: a caller holding the URL has +// already selected it. +func (p *Planner) TranscodeNodeByURL(nodeURL string) (*Node, bool) { + if p == nil || p.transcodes == nil || nodeURL == "" { + return nil, false + } + node := p.transcodes.FindByURL(normalizeNodeURL(nodeURL)) + if node == nil { + return nil, false + } + return node, true +} + // TranscodeNodeHealthy reports whether the pooled transcode node serving a URL // is currently healthy and enabled. Remote-start adoption gates its redirect // on this: a recipe another API server published is only trustworthy while diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 3a44c4422..432e94c2c 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -1,10 +1,12 @@ package nodepool import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -50,6 +52,32 @@ type Node struct { // is exactly as fresh as LastHealthCheck and never fresher. Absent for a // node that reports no sample. LastStats json.RawMessage `json:"last_stats,omitempty"` + // HWAccelOverride and HWDeviceOverride are this node's own acceleration + // policy. nil means the node inherits the cluster-wide playback.hw_accel / + // playback.hw_device settings, which is the normal case; a value here is + // what the node itself resolves against once it has reloaded its config. + HWAccelOverride *string `json:"hw_accel_override,omitempty"` + HWDeviceOverride *string `json:"hw_device_override,omitempty"` +} + +// EffectiveHWAccel is the acceleration backend this node runs under: its own +// override when it carries one, and otherwise the cluster-wide setting passed +// in. It is what a dispatch path names in a start request so the request, the +// recipe card, and what the node actually runs agree. +// +// Deliberately not derived from the node's stored capability report: that +// report is a snapshot up to a capability-refresh interval old, and naming its +// resolved backend would pin a stale answer *and* suppress the node's own +// start-time resolution — a node honors a named backend verbatim, so "auto" +// has to survive this far to reach live device enumeration on the node. +func (n *Node) EffectiveHWAccel(clusterHWAccel string) string { + if n == nil || n.HWAccelOverride == nil { + return clusterHWAccel + } + if override := strings.TrimSpace(*n.HWAccelOverride); override != "" { + return override + } + return clusterHWAccel } // CreateNodeInput holds the fields for creating a new node. @@ -78,8 +106,9 @@ func (i CreateNodeInput) Validate() error { // UpdateNodeInput holds the fields for updating a node. // The optional fields distinguish "leave unchanged" (nil) from "clear": -// an empty-string Group clears the group, and a non-positive MaxJobs or -// MaxBandwidthKbps clears that cap. +// an empty-string Group clears the group, an empty-string HWAccelOverride or +// HWDeviceOverride restores inheritance of the cluster-wide setting, and a +// non-positive MaxJobs or MaxBandwidthKbps clears that cap. type UpdateNodeInput struct { Name *string `json:"name,omitempty"` URL *string `json:"url,omitempty"` @@ -87,8 +116,77 @@ type UpdateNodeInput struct { Group *string `json:"group,omitempty"` MaxJobs *int `json:"max_jobs,omitempty"` MaxBandwidthKbps *int `json:"max_bandwidth_kbps,omitempty"` + HWAccelOverride *string `json:"hw_accel_override,omitempty"` + HWDeviceOverride *string `json:"hw_device_override,omitempty"` +} + +// UnmarshalJSON decodes an update body, mapping an explicit JSON null on the +// two acceleration overrides onto the empty-string clear sentinel the rest of +// this type uses. Plain decoding leaves a *string nil for an omitted field and +// for an explicit null alike, which would silently turn "go back to inheriting +// the cluster setting" into a no-op. Every other field decodes normally. +func (i *UpdateNodeInput) UnmarshalJSON(data []byte) error { + type plain UpdateNodeInput + var decoded plain + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + *i = UpdateNodeInput(decoded) + if isJSONNull(raw["hw_accel_override"]) { + i.HWAccelOverride = new(string) + } + if isJSONNull(raw["hw_device_override"]) { + i.HWDeviceOverride = new(string) + } + return nil +} + +// isJSONNull reports whether a field was present in the body with the literal +// value null (an absent field decodes to a nil RawMessage instead). +func isJSONNull(raw json.RawMessage) bool { + return string(bytes.TrimSpace(raw)) == "null" +} + +// Validate checks the values an update may set. Only the acceleration override +// has a closed set of values; the database CHECK enforces the same list, and +// rejecting here turns a constraint violation into an operator-readable error. +func (i UpdateNodeInput) Validate() error { + if i.HWAccelOverride == nil { + return nil + } + value := normalizeHWAccelOverride(*i.HWAccelOverride) + if value == nil { + // The clear sentinel: inherit the cluster-wide setting again. + return nil + } + if !slices.Contains(hwAccelOverrideValues, *value) { + return fmt.Errorf("%w: hw_accel_override must be one of %s", ErrInvalidNodeInput, + strings.Join(hwAccelOverrideValues, ", ")) + } + return nil } +// hwAccelOverrideValues mirrors the playback.hw_accel enum in +// internal/config/admin_settings.go and the CHECK constraint on +// stream_nodes.hw_accel_override: a per-node override may only name a backend +// the cluster-wide setting could also name. +var hwAccelOverrideValues = []string{hwAccelAuto, hwAccelQSV, hwAccelVAAPI, hwAccelNVENC, hwAccelNone} + +const ( + // hwAccelAuto asks the node to resolve its own backend against live + // hardware at session start; dispatch passes it through untouched for + // exactly that reason. + hwAccelAuto = "auto" + hwAccelQSV = "qsv" + hwAccelVAAPI = "vaapi" + hwAccelNVENC = "nvenc" + hwAccelNone = "none" +) + // normalizeGroup trims a group label and converts empty to NULL. func normalizeGroup(group string) *string { g := strings.TrimSpace(group) @@ -98,6 +196,26 @@ func normalizeGroup(group string) *string { return &g } +// normalizeOverride trims an override value and converts empty to NULL, which +// is how a node goes back to inheriting the cluster-wide setting. Case is +// preserved: a render device path is a filesystem path, not an enum. +func normalizeOverride(value string) *string { + v := strings.TrimSpace(value) + if v == "" { + return nil + } + return &v +} + +// normalizeHWAccelOverride is normalizeOverride for the acceleration enum, +// which is also lowercased. The cluster-wide playback.hw_accel accepts any +// casing (config.normalizeAdminEnum lowercases before comparing), and +// docs/admin-api.md promises the override takes the same values, so "QSV" from +// a third-party admin client must not be a 400 here when it is a 200 there. +func normalizeHWAccelOverride(value string) *string { + return normalizeOverride(strings.ToLower(value)) +} + // normalizeCap converts non-positive capacity values to NULL (unlimited). func normalizeCap(v *int) *int { if v == nil || *v <= 0 { @@ -116,7 +234,7 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats` +const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override` func scanNode(row pgx.Row) (*Node, error) { var n Node @@ -131,6 +249,7 @@ func scanNode(row pgx.Row) (*Node, error) { &n.LastHealthCheck, &n.CreatedAt, &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, &lastStats, + &n.HWAccelOverride, &n.HWDeviceOverride, ) if err != nil { return nil, err @@ -209,9 +328,12 @@ func (r *Repository) Create(ctx context.Context, input CreateNodeInput) (*Node, } // Update modifies a node's mutable fields. The optional fields use sentinel -// values to clear: an empty-string group and non-positive caps set the -// column to NULL (see UpdateNodeInput). +// values to clear: an empty-string group, an empty-string acceleration +// override, and non-positive caps set the column to NULL (see UpdateNodeInput). func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) (*Node, error) { + if err := input.Validate(); err != nil { + return nil, err + } var group *string if input.Group != nil { group = normalizeGroup(*input.Group) @@ -223,6 +345,13 @@ func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) if input.MaxBandwidthKbps != nil { maxBandwidth = normalizeCap(input.MaxBandwidthKbps) } + var hwAccelOverride, hwDeviceOverride *string + if input.HWAccelOverride != nil { + hwAccelOverride = normalizeHWAccelOverride(*input.HWAccelOverride) + } + if input.HWDeviceOverride != nil { + hwDeviceOverride = normalizeOverride(*input.HWDeviceOverride) + } row := r.pool.QueryRow(ctx, `UPDATE stream_nodes SET name = COALESCE($2, name), @@ -230,13 +359,17 @@ func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) enabled = COALESCE($4, enabled), node_group = CASE WHEN $5::boolean THEN $6::text ELSE node_group END, max_jobs = CASE WHEN $7::boolean THEN $8::integer ELSE max_jobs END, - max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END + max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END, + hw_accel_override = CASE WHEN $11::boolean THEN $12::text ELSE hw_accel_override END, + hw_device_override = CASE WHEN $13::boolean THEN $14::text ELSE hw_device_override END WHERE id = $1 RETURNING `+nodeColumns, id, input.Name, input.URL, input.Enabled, input.Group != nil, group, input.MaxJobs != nil, maxJobs, - input.MaxBandwidthKbps != nil, maxBandwidth) + input.MaxBandwidthKbps != nil, maxBandwidth, + input.HWAccelOverride != nil, hwAccelOverride, + input.HWDeviceOverride != nil, hwDeviceOverride) n, err := scanNode(row) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNodeNotFound @@ -298,4 +431,9 @@ func (r *Repository) UpdateCapabilities(ctx context.Context, id int, capabilitie } // Sentinel errors. -var ErrNodeNotFound = errors.New("stream node not found") +var ( + ErrNodeNotFound = errors.New("stream node not found") + // ErrInvalidNodeInput marks a caller-supplied value the store refuses, so + // an API layer can answer 400 without string-matching the message. + ErrInvalidNodeInput = errors.New("invalid node input") +) diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index 1acdc58eb..17ee629f5 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -27,11 +27,14 @@ func newNodeTestPool(t *testing.T) *pgxpool.Pool { t.Fatalf("connect test database: %v", err) } t.Cleanup(pool.Close) - var column *string + // Every read here selects the full column list, so the newest column is the + // one worth probing: a database missing it fails every test in the package. + var columns int if err := pool.QueryRow(ctx, - `SELECT column_name FROM information_schema.columns - WHERE table_name = 'stream_nodes' AND column_name = 'capabilities_hash'`).Scan(&column); err != nil { - t.Skip("test database has not applied the node capabilities migration") + `SELECT count(*) FROM information_schema.columns + WHERE table_name = 'stream_nodes' + AND column_name IN ('capabilities_hash', 'hw_accel_override')`).Scan(&columns); err != nil || columns < 2 { + t.Skip("test database has not applied the stream_nodes capability/override migrations") } return pool } diff --git a/internal/nodepool/repository_hw_overrides_test.go b/internal/nodepool/repository_hw_overrides_test.go new file mode 100644 index 000000000..4e102494d --- /dev/null +++ b/internal/nodepool/repository_hw_overrides_test.go @@ -0,0 +1,284 @@ +package nodepool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" +) + +// The two override fields are the whole point of per-node acceleration policy, +// so setting one, reading it back through an ordinary List/Get, and clearing it +// again all have to survive the round trip through Postgres. +func TestRepositoryUpdateHWOverridesRoundTrip(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node := createTestNode(t, repo, "hw-override") + + if node.HWAccelOverride != nil || node.HWDeviceOverride != nil { + t.Fatalf("new node already carries overrides: %+v", node) + } + + accel, device := "vaapi", "/dev/dri/renderD129" + updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{ + HWAccelOverride: &accel, + HWDeviceOverride: &device, + }) + if err != nil { + t.Fatalf("set overrides: %v", err) + } + if updated.HWAccelOverride == nil || *updated.HWAccelOverride != accel { + t.Fatalf("hw_accel_override = %v, want %q", updated.HWAccelOverride, accel) + } + if updated.HWDeviceOverride == nil || *updated.HWDeviceOverride != device { + t.Fatalf("hw_device_override = %v, want %q", updated.HWDeviceOverride, device) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.HWAccelOverride == nil || *reloaded.HWAccelOverride != accel { + t.Fatalf("reloaded hw_accel_override = %v, want %q", reloaded.HWAccelOverride, accel) + } + + // An unrelated update must not disturb the overrides: omitted means + // unchanged, exactly like the other optional fields. + renamed := node.Name + "-renamed" + untouched, err := repo.Update(ctx, node.ID, UpdateNodeInput{Name: &renamed}) + if err != nil { + t.Fatalf("rename node: %v", err) + } + if untouched.HWAccelOverride == nil || *untouched.HWAccelOverride != accel { + t.Fatalf("unrelated update dropped hw_accel_override: %v", untouched.HWAccelOverride) + } + + // The empty-string sentinel restores inheritance of the cluster setting. + cleared, err := repo.Update(ctx, node.ID, UpdateNodeInput{ + HWAccelOverride: new(string), + HWDeviceOverride: new(string), + }) + if err != nil { + t.Fatalf("clear overrides: %v", err) + } + if cleared.HWAccelOverride != nil || cleared.HWDeviceOverride != nil { + t.Fatalf("overrides after clear = %v / %v, want nil (inherit)", cleared.HWAccelOverride, cleared.HWDeviceOverride) + } +} + +// A value outside the enum must be refused before it reaches the CHECK +// constraint, so the operator sees which values are legal. +func TestRepositoryUpdateRejectsUnknownHWAccelOverride(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + node := createTestNode(t, repo, "hw-override-invalid") + + bogus := "videotoolbox" + _, err := repo.Update(context.Background(), node.ID, UpdateNodeInput{HWAccelOverride: &bogus}) + if !errors.Is(err, ErrInvalidNodeInput) { + t.Fatalf("err = %v, want ErrInvalidNodeInput", err) + } + + reloaded, err := repo.GetByID(context.Background(), node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.HWAccelOverride != nil { + t.Fatalf("rejected update still wrote %v", reloaded.HWAccelOverride) + } +} + +// A mixed-case override is accepted like the cluster-wide setting accepts one, +// and reaches the column lowercase — the only casing its CHECK allows. +func TestRepositoryUpdateLowercasesHWAccelOverride(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + node := createTestNode(t, repo, "hw-override-case") + + value, device := "QSV", "/dev/dri/renderD129" + updated, err := repo.Update(context.Background(), node.ID, UpdateNodeInput{ + HWAccelOverride: &value, + HWDeviceOverride: &device, + }) + if err != nil { + t.Fatalf("set overrides: %v", err) + } + if updated.HWAccelOverride == nil || *updated.HWAccelOverride != "qsv" { + t.Fatalf("hw_accel_override = %v, want %q", updated.HWAccelOverride, "qsv") + } + // The device is a path, not an enum: case survives. + if updated.HWDeviceOverride == nil || *updated.HWDeviceOverride != device { + t.Fatalf("hw_device_override = %v, want %q", updated.HWDeviceOverride, device) + } +} + +func createTestNode(t *testing.T, repo *Repository, prefix string) *Node { + t.Helper() + ctx := context.Background() + unique := time.Now().UnixNano() + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("%s-%d", prefix, unique), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://%s-%d", prefix, unique), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + return node +} + +// An admin UI clears a field by sending null. Standard decoding cannot tell +// that apart from an omitted field, so the type maps it onto the clear +// sentinel; getting this wrong makes "inherit again" a silent no-op. +func TestUpdateNodeInputDecodesExplicitNullAsClear(t *testing.T) { + tests := []struct { + name string + body string + wantAccel *string + wantDevice *string + }{ + {name: "omitted leaves both unchanged", body: `{"name":"gpu-1"}`}, + { + name: "explicit null clears the accel override", + body: `{"hw_accel_override":null}`, + wantAccel: new(string), + }, + { + name: "explicit null clears both", + body: `{"hw_accel_override":null,"hw_device_override":null}`, + wantAccel: new(string), + wantDevice: new(string), + }, + { + name: "values decode normally", + body: `{"hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}`, + wantAccel: ptrTo("qsv"), + wantDevice: ptrTo("/dev/dri/renderD128"), + }, + { + name: "empty string is the same clear sentinel", + body: `{"hw_accel_override":""}`, + wantAccel: new(string), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var input UpdateNodeInput + if err := json.Unmarshal([]byte(test.body), &input); err != nil { + t.Fatalf("decode: %v", err) + } + if !equalStringPtr(input.HWAccelOverride, test.wantAccel) { + t.Fatalf("HWAccelOverride = %v, want %v", input.HWAccelOverride, test.wantAccel) + } + if !equalStringPtr(input.HWDeviceOverride, test.wantDevice) { + t.Fatalf("HWDeviceOverride = %v, want %v", input.HWDeviceOverride, test.wantDevice) + } + }) + } +} + +// Decoding must not lose the fields it always carried. +func TestUpdateNodeInputKeepsExistingFields(t *testing.T) { + var input UpdateNodeInput + body := `{"name":"gpu-1","url":"http://gpu-1","enabled":true,"group":"rack-a","max_jobs":3,"max_bandwidth_kbps":0}` + if err := json.Unmarshal([]byte(body), &input); err != nil { + t.Fatalf("decode: %v", err) + } + if input.Name == nil || *input.Name != "gpu-1" || input.URL == nil || *input.URL != "http://gpu-1" { + t.Fatalf("identity fields lost: %+v", input) + } + if input.Enabled == nil || !*input.Enabled || input.Group == nil || *input.Group != "rack-a" { + t.Fatalf("enabled/group lost: %+v", input) + } + if input.MaxJobs == nil || *input.MaxJobs != 3 || input.MaxBandwidthKbps == nil || *input.MaxBandwidthKbps != 0 { + t.Fatalf("caps lost: %+v", input) + } +} + +func TestUpdateNodeInputValidate(t *testing.T) { + for _, value := range hwAccelOverrideValues { + if err := (UpdateNodeInput{HWAccelOverride: &value}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", value, err) + } + } + if err := (UpdateNodeInput{}).Validate(); err != nil { + t.Fatalf("Validate(omitted) = %v, want nil", err) + } + if err := (UpdateNodeInput{HWAccelOverride: new(string)}).Validate(); err != nil { + t.Fatalf("Validate(clear) = %v, want nil", err) + } + bogus := "cuda" + if err := (UpdateNodeInput{HWAccelOverride: &bogus}).Validate(); !errors.Is(err, ErrInvalidNodeInput) { + t.Fatalf("Validate(%q) = %v, want ErrInvalidNodeInput", bogus, err) + } +} + +// The cluster-wide playback.hw_accel accepts any casing, and the override is +// documented as taking the same values, so a third-party admin client must not +// get a 400 here for a body /admin/settings would have accepted. +func TestUpdateNodeInputValidateIgnoresCase(t *testing.T) { + for _, value := range []string{"QSV", " Vaapi ", "NONE", "Auto"} { + if err := (UpdateNodeInput{HWAccelOverride: &value}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", value, err) + } + } +} + +// Only the acceleration enum is case-folded on the way to the column, whose +// CHECK list is lowercase. A render device is a filesystem path and keeps its +// case. +func TestNormalizeHWAccelOverride(t *testing.T) { + if got := normalizeHWAccelOverride(" QSV "); got == nil || *got != "qsv" { + t.Fatalf("normalizeHWAccelOverride(%q) = %v, want %q", " QSV ", got, "qsv") + } + if got := normalizeHWAccelOverride(" "); got != nil { + t.Fatalf("blank override = %v, want nil (inherit)", got) + } + if got := normalizeOverride(" /dev/dri/renderD129 "); got == nil || *got != "/dev/dri/renderD129" { + t.Fatalf("device override = %v, want the path unchanged", got) + } +} + +// Dispatch names the node's own override, and otherwise the cluster value +// verbatim — "auto" included, so the node resolves it against live hardware at +// session start instead of inheriting a snapshot's answer. +func TestNodeEffectiveHWAccel(t *testing.T) { + tests := []struct { + name string + node *Node + cluster string + want string + }{ + {name: "no node at all", cluster: "qsv", want: "qsv"}, + {name: "no override inherits", node: &Node{}, cluster: "qsv", want: "qsv"}, + {name: "no override inherits auto", node: &Node{}, cluster: hwAccelAuto, want: hwAccelAuto}, + {name: "override wins", node: &Node{HWAccelOverride: ptrTo("none")}, cluster: "qsv", want: "none"}, + {name: "override wins over auto", node: &Node{HWAccelOverride: ptrTo(hwAccelNVENC)}, cluster: hwAccelAuto, want: hwAccelNVENC}, + {name: "blank override is not an override", node: &Node{HWAccelOverride: ptrTo(" ")}, cluster: "qsv", want: "qsv"}, + { + name: "a stale capability report is not consulted", + node: &Node{Capabilities: json.RawMessage(`{"resolved":"none"}`)}, + cluster: "qsv", + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.node.EffectiveHWAccel(test.cluster); got != test.want { + t.Fatalf("EffectiveHWAccel(%q) = %q, want %q", test.cluster, got, test.want) + } + }) + } +} + +func ptrTo(v string) *string { return &v } + +func equalStringPtr(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} diff --git a/migrations/sql/20260827025521_node_hw_overrides.sql b/migrations/sql/20260827025521_node_hw_overrides.sql new file mode 100644 index 000000000..6e6c23f8b --- /dev/null +++ b/migrations/sql/20260827025521_node_hw_overrides.sql @@ -0,0 +1,32 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN hw_accel_override text + CHECK (hw_accel_override IN ('auto', 'qsv', 'vaapi', 'nvenc', 'none')), + ADD COLUMN hw_device_override text; + +COMMENT ON COLUMN stream_nodes.hw_accel_override IS + 'Per-node hardware acceleration backend, overriding the cluster-wide ' + 'playback.hw_accel setting for this node only. NULL means inherit the ' + 'cluster value — the normal case for a homogeneous deployment. Set it ' + 'when one node''s hardware differs from the rest (a CPU-only box in a QSV ' + 'cluster sets ''none''). The node reads its own row (matched on url ' + 'against its NODE_URL) on every config reload, so this is the value it ' + 'probes with and falls back to; the API dispatches remote transcodes with ' + 'it too, in preference to the cluster value. A change applies without a ' + 'restart, except to the boot-time encoder warmup and to sessions already ' + 'transcoding.'; + +COMMENT ON COLUMN stream_nodes.hw_device_override IS + 'Per-node hardware device (render node path or index) for the backend ' + 'above, overriding the cluster-wide playback.hw_device for this node ' + 'only. NULL means inherit the cluster value. Applies on the same terms as ' + 'hw_accel_override.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS hw_accel_override, + DROP COLUMN IF EXISTS hw_device_override; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 9a21ef85b..c20a735b0 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3935,6 +3935,15 @@ export interface StreamNode { physical_gpu_keys?: string[]; /** The node's resource sample from the last health check. */ last_stats?: NodeLastStats | null; + /** + * This node's own acceleration policy. Absent or null is the normal case: + * the node inherits the cluster-wide playback.hw_accel / playback.hw_device + * settings. A value here is what the node resolves against from its next + * config reload, and what remote transcodes to it are dispatched with. + */ + hw_accel_override?: string | null; + /** Comma-separated render device paths pinned to this node; null inherits. */ + hw_device_override?: string | null; } export interface CreateNodeRequest { @@ -3954,6 +3963,10 @@ export interface UpdateNodeRequest { group?: string; max_jobs?: number; max_bandwidth_kbps?: number; + // An omitted override leaves the stored value alone; an explicit null (or an + // empty string) restores inheritance of the cluster-wide playback setting. + hw_accel_override?: string | null; + hw_device_override?: string | null; } export interface CheckNodeResponse { diff --git a/web/src/hooks/queries/admin/nodes.ts b/web/src/hooks/queries/admin/nodes.ts index 007a3b183..25c861ac6 100644 --- a/web/src/hooks/queries/admin/nodes.ts +++ b/web/src/hooks/queries/admin/nodes.ts @@ -1,6 +1,11 @@ import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; import { api } from "@/api/client"; -import type { StreamNode, CreateNodeRequest, CheckNodeResponse } from "@/api/types"; +import type { + StreamNode, + CreateNodeRequest, + UpdateNodeRequest, + CheckNodeResponse, +} from "@/api/types"; import { adminKeys } from "../keys"; import { toast } from "sonner"; @@ -35,7 +40,10 @@ export function useCreateNode() { export function useUpdateNode() { const queryClient = useQueryClient(); return useMutation({ - mutationFn: ({ id, body }: { id: number; body: Record }) => + // The body is typed rather than a loose record so a null acceleration + // override — the value that restores inheritance of the cluster-wide + // setting — survives to the wire instead of being dropped as a typo. + mutationFn: ({ id, body }: { id: number; body: UpdateNodeRequest }) => api(`/admin/nodes/${id}`, { method: "PUT", body: JSON.stringify(body), diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 1706a27cf..445449922 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -1,6 +1,6 @@ import { useState } from "react"; import type { FormEvent, ReactNode } from "react"; -import type { StreamNode, CreateNodeRequest } from "@/api/types"; +import type { StreamNode, CreateNodeRequest, UpdateNodeRequest } from "@/api/types"; import { useAdminNodes, useCreateNode, @@ -23,12 +23,26 @@ import { TableRow, } from "@/components/ui/table"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Plus, Pencil, Trash2, RefreshCw, Info, AlertTriangle } from "lucide-react"; import { ConfirmDialog } from "@/components/ConfirmDialog"; import { formatDateTime } from "@/lib/datetime"; import { cn } from "@/lib/utils"; import type { ResourceMetric } from "./adminNodesPresentation"; -import { describeNodeGPU, describeNodeSystem } from "./adminNodesPresentation"; +import { + HW_ACCEL_INHERIT, + HW_ACCEL_OVERRIDE_OPTIONS, + describeNodeAccelerationOverride, + describeNodeGPU, + describeNodeSystem, + parseHWDeviceOverride, +} from "./adminNodesPresentation"; type NodeType = "proxy" | "transcode"; @@ -78,13 +92,29 @@ function NodeSystemCell({ node }: { node: StreamNode }) { ); } +/** The "override: qsv" line, or nothing on a node that inherits the cluster. */ +function NodeOverrideLine({ node }: { node: StreamNode }) { + const override = describeNodeAccelerationOverride(node); + if (!override) { + return null; + } + return ( +
+ {override.label} +
+ ); +} + function NodeGPUCell({ node }: { node: StreamNode }) { const gpu = describeNodeGPU(node); if (gpu.kind === "awaiting") { return ( - - {gpu.label} - +
+ + {gpu.label} + + +
); } @@ -122,6 +152,7 @@ function NodeGPUCell({ node }: { node: StreamNode }) { {gpu.deviceSummary}
)} + {gpu.live.map((device) => (
0 ? overrideDevices.join(",") : null, + }; + updateMutation.mutate({ id: node.id, body }, { onSuccess: onClose }); } else { const body: CreateNodeRequest = { type: nodeType, ...fields }; createMutation.mutate(body, { onSuccess: onClose }); @@ -450,6 +494,49 @@ function NodeForm({
)} + {/* Overrides are edit-only: the create endpoint takes no acceleration + fields, so offering them here would silently drop what was typed. */} + {node && ( + <> +
+ + +

+ Optional. Overrides the cluster-wide Hardware Acceleration setting for this node only + — use it when this node's hardware differs from the rest of the cluster. Applies to + new transcodes within a minute; restart the node to re-prime its encoder for the new + backend. +

+
+ +
+ + setHwDeviceOverride(e.target.value)} + placeholder="Inherit cluster setting" + /> +

+ Optional. Comma-separated render device paths this node transcodes on (e.g.{" "} + /dev/dri/renderD128,/dev/dri/renderD129). Leave + empty to inherit the cluster-wide device selection. +

+
+ + )} + diff --git a/web/src/pages/admin-settings/PlaybackSettings.test.tsx b/web/src/pages/admin-settings/PlaybackSettings.test.tsx index b4c3a91a7..9faf59edc 100644 --- a/web/src/pages/admin-settings/PlaybackSettings.test.tsx +++ b/web/src/pages/admin-settings/PlaybackSettings.test.tsx @@ -1,4 +1,5 @@ import { renderToStaticMarkup } from "react-dom/server"; +import { MemoryRouter } from "react-router"; import { beforeEach, describe, expect, it, vi } from "vitest"; import PlaybackSettings from "./PlaybackSettings"; @@ -167,3 +168,54 @@ describe("PlaybackSettings transcode tone mapping", () => { expect(toggle).not.toHaveAttribute("disabled"); }); }); + +describe("PlaybackSettings divergent node inventories", () => { + it("points at the per-node overrides on the Nodes page", () => { + useHWAccelDetectionMock.mockReturnValue({ + data: { + resolved: "qsv", + render_device_details: [{ path: "/dev/dri/renderD128", description: "Intel GPU" }], + nodes: [ + { node_url: "http://node-a", render_devices: ["/dev/dri/renderD128"] }, + { node_url: "http://node-b", render_devices: ["/dev/dri/renderD129"] }, + ], + }, + isLoading: false, + }); + useSettingsFormMock.mockReturnValue(makeForm({ "playback.hw_accel": "qsv" })); + + const markup = renderToStaticMarkup( + + + , + ); + + expect(markup).toContain("the nodes report different devices"); + expect(markup).toContain("set per-node overrides on the"); + expect(markup).toContain('href="/admin/nodes"'); + }); + + it("stays quiet while every node reports the same devices", () => { + useHWAccelDetectionMock.mockReturnValue({ + data: { + resolved: "qsv", + render_device_details: [{ path: "/dev/dri/renderD128", description: "Intel GPU" }], + nodes: [ + { node_url: "http://node-a", render_devices: ["/dev/dri/renderD128"] }, + { node_url: "http://node-b", render_devices: ["/dev/dri/renderD128"] }, + ], + }, + isLoading: false, + }); + useSettingsFormMock.mockReturnValue(makeForm({ "playback.hw_accel": "qsv" })); + + const markup = renderToStaticMarkup( + + + , + ); + + expect(markup).not.toContain("set per-node overrides on the"); + expect(markup).not.toContain('href="/admin/nodes"'); + }); +}); diff --git a/web/src/pages/admin-settings/PlaybackSettings.tsx b/web/src/pages/admin-settings/PlaybackSettings.tsx index 79ab91e85..1a76096b1 100644 --- a/web/src/pages/admin-settings/PlaybackSettings.tsx +++ b/web/src/pages/admin-settings/PlaybackSettings.tsx @@ -1,4 +1,5 @@ import { useMemo } from "react"; +import { Link } from "react-router"; import { useSettingsForm } from "@/hooks/useSettingsForm"; import { useHWAccelDetection } from "@/hooks/queries/admin/system"; import { Label } from "@/components/ui/label"; @@ -124,7 +125,15 @@ export default function PlaybackSettings() { {inventoriesDiverge && (

This setting applies to every transcode node, but the nodes report different - devices. Only paths present on all nodes are safe to select. + devices. Only paths present on all nodes are safe to select — for the rest, set + per-node overrides on the{" "} + + Nodes page + + .

)}
diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index 96641d0a7..f3209cd7f 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -4,10 +4,12 @@ import { CAPABILITY_STALE_AFTER_MS, DISK_FILL_WARNING_PCT, describeGPUBusy, + describeNodeAccelerationOverride, describeNodeGPU, describeNodeSystem, describeResourceSample, formatBitsPerSecond, + parseHWDeviceOverride, } from "./adminNodesPresentation"; const NOW = Date.parse("2026-08-26T12:00:00Z"); @@ -713,3 +715,66 @@ describe("describeResourceSample", () => { }); }); }); + +describe("describeNodeAccelerationOverride", () => { + it("renders nothing for a node that inherits the cluster-wide settings", () => { + expect(describeNodeAccelerationOverride(makeNode())).toBeNull(); + expect( + describeNodeAccelerationOverride( + makeNode({ hw_accel_override: null, hw_device_override: null }), + ), + ).toBeNull(); + // Whitespace is not an override. + expect(describeNodeAccelerationOverride(makeNode({ hw_device_override: " , " }))).toBeNull(); + }); + + it("names the backend a node is pinned to", () => { + const override = describeNodeAccelerationOverride(makeNode({ hw_accel_override: "qsv" })); + + expect(override?.label).toBe("override: qsv"); + expect(override?.title).toContain("Acceleration: qsv"); + expect(override?.title).toContain("GPU devices: inherited"); + }); + + it("calls a software override software rather than none", () => { + expect(describeNodeAccelerationOverride(makeNode({ hw_accel_override: "none" }))?.label).toBe( + "override: software", + ); + }); + + it("shows a single pinned device inline and counts several", () => { + expect( + describeNodeAccelerationOverride( + makeNode({ hw_accel_override: "vaapi", hw_device_override: "/dev/dri/renderD129" }), + )?.label, + ).toBe("override: vaapi · /dev/dri/renderD129"); + + const many = describeNodeAccelerationOverride( + makeNode({ hw_device_override: "/dev/dri/renderD128, /dev/dri/renderD129" }), + ); + expect(many?.label).toBe("override: 2 devices"); + expect(many?.title).toContain("GPU devices: /dev/dri/renderD128, /dev/dri/renderD129."); + expect(many?.title).toContain("Acceleration: inherited"); + }); + + it("says when the override takes effect", () => { + const title = describeNodeAccelerationOverride(makeNode({ hw_accel_override: "nvenc" }))?.title; + expect(title).toContain("applies to new transcodes within a minute"); + expect(title).toContain("sessions already running keep the backend they started with"); + }); +}); + +describe("parseHWDeviceOverride", () => { + it("splits, trims, and drops empty entries", () => { + expect(parseHWDeviceOverride(" /dev/dri/renderD128 ,, /dev/dri/renderD129,")).toEqual([ + "/dev/dri/renderD128", + "/dev/dri/renderD129", + ]); + }); + + it("treats absent and empty values as no devices", () => { + expect(parseHWDeviceOverride(null)).toEqual([]); + expect(parseHWDeviceOverride(undefined)).toEqual([]); + expect(parseHWDeviceOverride(" ")).toEqual([]); + }); +}); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index cd10f8f71..5246f86ba 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -347,6 +347,80 @@ function describeDeviceLine(device: NodeRenderDevice): string { return address ? `${line} (${address})` : line; } +// --- Per-node acceleration overrides --------------------------------------- + +/** + * Select value standing in for "no override". Radix rejects an empty option + * value, and the wire form of "inherit" is null, so the picker needs a + * sentinel of its own. + */ +export const HW_ACCEL_INHERIT = "inherit"; + +/** + * The acceleration choices a node may be pinned to, inherit first. The backend + * values mirror the playback.hw_accel enum: a per-node override may only name a + * backend the cluster-wide setting could also name. + */ +export const HW_ACCEL_OVERRIDE_OPTIONS: readonly { value: string; label: string }[] = [ + { value: HW_ACCEL_INHERIT, label: "Inherit cluster setting" }, + { value: "auto", label: "Auto" }, + { value: "qsv", label: "Intel Quick Sync (QSV)" }, + { value: "vaapi", label: "VA-API" }, + { value: "nvenc", label: "NVIDIA NVENC" }, + { value: "none", label: "Software" }, +]; + +/** A node's own acceleration policy, as rendered beside its GPU inventory. */ +export interface NodeAccelerationOverride { + /** Compact row text, e.g. "override: qsv · /dev/dri/renderD129". */ + label: string; + /** Hover text: both halves of the policy, and when a change lands. */ + title: string; +} + +/** Split a stored comma-separated device override into its paths. */ +export function parseHWDeviceOverride(value: string | null | undefined): string[] { + return (value ?? "") + .split(",") + .map((part) => part.trim()) + .filter((part) => part !== ""); +} + +/** + * Describe how a node resolves hardware acceleration, or null when it resolves + * the cluster-wide way. Null is the normal case and renders nothing: the Nodes + * table would otherwise repeat the Playback settings page on every row, and the + * point of the line is to mark the rows that do *not* follow it. + */ +export function describeNodeAccelerationOverride( + node: StreamNode, +): NodeAccelerationOverride | null { + const accel = node.hw_accel_override?.trim().toLowerCase() ?? ""; + const devices = parseHWDeviceOverride(node.hw_device_override); + if (accel === "" && devices.length === 0) { + return null; + } + + const accelLabel = accel === "none" ? "software" : accel; + const firstDevice = devices[0] ?? ""; + const deviceLabel = devices.length === 1 ? firstDevice : `${devices.length} devices`; + const label = [accelLabel, devices.length > 0 ? deviceLabel : ""] + .filter((part) => part !== "") + .join(" · "); + + const title = [ + accel === "" + ? "Acceleration: inherited from the cluster-wide Hardware Acceleration setting." + : `Acceleration: ${accelLabel}, overriding the cluster-wide Hardware Acceleration setting.`, + devices.length === 0 + ? "GPU devices: inherited from the cluster-wide setting." + : `GPU devices: ${devices.join(", ")}.`, + "A changed override applies to new transcodes within a minute; sessions already running keep the backend they started with.", + ].join("\n"); + + return { label: `override: ${label}`, title }; +} + // --- Host resource samples ------------------------------------------------- // // A node's last_stats and the API host's /admin/system/resources carry the same From 7e01b9951370ae7d9d9c297c28893924dd4afd60 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:49:02 -0400 Subject: [PATCH 006/163] feat(nodepool): prefer the least-loaded physical gpu across nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Physical GPU identity moves into nodepool: every stored or pooled node row derives physical_gpu_keys from its capability payload (NVIDIA uuid, else boot_id|pci_address; a device with neither — including a missing boot id — contributes no key rather than a cross-host-colliding one). Transcode selection gains a strictly secondary tie-break: when candidates are level on effective jobs, the node whose physical GPU group (itself plus every pooled node sharing a key) carries fewer total jobs wins. Primary least-jobs ordering, session soft-affinity, and proxy selection are unchanged; job counts only, no utilization input. The admin Nodes page shows a Shared GPU badge naming the nodes a card is shared with, across the transcode and proxy tables. docs/admin-api.md documents the derivation, its boot-scoped stability caveat, and the planner behavior. Phase 5 of the node GPU observability plan. Related issue: #780 Co-Authored-By: Claude Fable 5 --- docs/admin-api.md | 29 ++- internal/api/handlers/nodes.go | 73 +------ internal/api/handlers/nodes_test.go | 54 +----- internal/nodepool/gpuidentity.go | 80 ++++++++ internal/nodepool/gpuidentity_test.go | 135 +++++++++++++ internal/nodepool/planner.go | 102 ++++++++-- internal/nodepool/planner_test.go | 194 +++++++++++++++++++ internal/nodepool/proxy_pool.go | 8 +- internal/nodepool/repository.go | 12 ++ internal/nodepool/repository_scan_test.go | 58 ++++++ internal/nodepool/transcode_pool.go | 7 + web/src/pages/AdminNodes.tsx | 38 +++- web/src/pages/adminNodesPresentation.test.ts | 57 ++++++ web/src/pages/adminNodesPresentation.ts | 41 ++++ 14 files changed, 751 insertions(+), 137 deletions(-) create mode 100644 internal/nodepool/gpuidentity.go create mode 100644 internal/nodepool/gpuidentity_test.go create mode 100644 internal/nodepool/repository_scan_test.go diff --git a/docs/admin-api.md b/docs/admin-api.md index ee9aa6658..55cb1e59d 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -144,10 +144,31 @@ One key per render device in the stored report, deduplicated and sorted: - `|`, because a PCI slot only means the same hardware within one boot of one kernel. -A device with neither contributes no key rather than a synthetic one. Two nodes -sharing a key are backed by the same physical GPU — the case that makes -per-node capacity accounting wrong, and which no single node's report can -express. +A device with neither contributes no key rather than a synthetic one, and so +does a slot on a host that reported no `boot_id`: `boot_id` detection is +best-effort, and an unscoped slot is not an identity, since every host with an +Intel iGPU has one at `0000:00:02.0`. Two nodes sharing a key are backed by the +same physical GPU — the case that makes per-node capacity accounting wrong, and +which no single node's report can express. The keys are derived from the stored report on every read, so they are +present as soon as a report is, including immediately after an API restart. + +Caveats on what a key can prove: + +- A key is only stable within one boot of the host it came from. `boot_id` + changes on reboot, so a fallback key does too, and the same card looks like a + different GPU until every node on that host has re-reported. An NVIDIA + `gpu_uuid` has no such limit. +- Intel and AMD GPUs passed through to separate VMs cannot be correlated at + all: each guest reports its own `boot_id` and its own PCI topology, so two + guests on one card produce two unrelated keys. Sharing there is invisible to + the server, and stays a matter for how the host is partitioned. + +Node selection uses the same keys as a tie-breaker: among transcode nodes that +are otherwise level on effective job count, the one whose physical GPU group — +itself plus every pooled transcode node sharing a key with it — carries the +fewest jobs wins. It never overrides the job count itself or the soft affinity +that keeps a session on its current node, and it does not apply to proxy +selection, which is round-robin and does no GPU work. ## `POST /api/v1/admin/nodes` diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index c98c940ec..d76cc66f5 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -9,7 +9,6 @@ import ( "fmt" "log/slog" "net/http" - "slices" "strconv" "sync" "time" @@ -78,18 +77,11 @@ type checkNodeResult struct { CapabilitiesHash string `json:"capabilities_hash,omitempty"` } -// nodeListItem is a stored node plus the fields derived from its capability -// payload. The row is embedded rather than copied so the response keeps every -// existing field automatically as the node model grows. -type nodeListItem struct { - *nodepool.Node - // PhysicalGPUKeys identifies the actual GPUs behind this node. Two nodes - // sharing a key are sharing hardware — the case that makes independent - // capacity accounting wrong — which no per-node field can express. - PhysicalGPUKeys []string `json:"physical_gpu_keys,omitempty"` -} - // HandleListNodes handles GET /admin/nodes. +// +// The stored rows are served as they are: physical_gpu_keys is derived by the +// node store when a row is scanned, so the same identities the planner routes +// on are the ones an operator sees. func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { nodes, err := h.repo.List(r.Context()) if err != nil { @@ -97,61 +89,10 @@ func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list nodes") return } - - items := make([]nodeListItem, 0, len(nodes)) - for _, node := range nodes { - items = append(items, nodeListItem{ - Node: node, - PhysicalGPUKeys: physicalGPUKeys(node.Capabilities), - }) - } - writeJSON(w, http.StatusOK, items) -} - -// nodeGPUIdentity is the minimal projection needed to identify a node's GPUs -// out of its stored capability payload. -type nodeGPUIdentity struct { - BootID string `json:"boot_id"` - RenderDeviceDetails []struct { - PCIAddress string `json:"pci_address"` - GPUUUID string `json:"gpu_uuid"` - } `json:"render_device_details"` -} - -// physicalGPUKeys derives one stable key per GPU a node can see. An NVIDIA -// uuid is preferred because it follows the card between slots and hosts; the -// PCI address falls back to it, scoped by boot id because a device path and -// slot only mean the same hardware within one boot of one kernel. A device with -// neither is unidentifiable and contributes no key rather than a fake one. -func physicalGPUKeys(capabilities []byte) []string { - if len(capabilities) == 0 { - return nil - } - var identity nodeGPUIdentity - if err := json.Unmarshal(capabilities, &identity); err != nil { - return nil - } - seen := make(map[string]struct{}, len(identity.RenderDeviceDetails)) - keys := make([]string, 0, len(identity.RenderDeviceDetails)) - for _, device := range identity.RenderDeviceDetails { - key := device.GPUUUID - if key == "" { - if device.PCIAddress == "" { - continue - } - key = identity.BootID + "|" + device.PCIAddress - } - if _, duplicate := seen[key]; duplicate { - continue - } - seen[key] = struct{}{} - keys = append(keys, key) - } - if len(keys) == 0 { - return nil + if nodes == nil { + nodes = []*nodepool.Node{} } - slices.Sort(keys) - return keys + writeJSON(w, http.StatusOK, nodes) } // HandleCreateNode handles POST /admin/nodes. diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index d0be01da4..96885ebde 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "slices" "strings" "testing" "time" @@ -14,55 +13,6 @@ import ( "github.com/go-chi/chi/v5" ) -// An NVIDIA uuid identifies a card wherever it is plugged in; a PCI address -// only identifies a slot, and only within one boot of one kernel. Deriving the -// key this way is what lets an admin see that two nodes are sharing one GPU. -func TestPhysicalGPUKeys(t *testing.T) { - tests := []struct { - name string - capabilities string - want []string - }{ - { - name: "prefers gpu uuid over slot identity", - capabilities: `{"boot_id":"boot-1","render_device_details":[ - {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`, - want: []string{"GPU-aaa"}, - }, - { - name: "falls back to boot-scoped pci address", - capabilities: `{"boot_id":"boot-1","render_device_details":[ - {"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}]}`, - want: []string{"boot-1|0000:04:00.0"}, - }, - { - name: "mixed devices are deduped and sorted", - capabilities: `{"boot_id":"boot-1","render_device_details":[ - {"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}, - {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}, - {"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}]}`, - want: []string{"GPU-bbb", "boot-1|0000:05:00.0"}, - }, - { - name: "device with no identity contributes no key", - capabilities: `{"boot_id":"boot-1","render_device_details":[ - {"path":"/dev/dri/renderD128"},{"path":"/dev/dri/renderD129","gpu_uuid":"GPU-ccc"}]}`, - want: []string{"GPU-ccc"}, - }, - {name: "no capabilities stored", capabilities: "", want: nil}, - {name: "unparseable payload", capabilities: `not json`, want: nil}, - {name: "no render devices", capabilities: `{"boot_id":"boot-1","render_device_details":[]}`, want: nil}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := physicalGPUKeys([]byte(tt.capabilities)) - if !slices.Equal(got, tt.want) { - t.Fatalf("physicalGPUKeys() = %v, want %v", got, tt.want) - } - }) - } -} - type stubNodeRepository struct { nodes []*nodepool.Node // updateResult is what Update returns once validation passes; nil keeps the @@ -113,6 +63,10 @@ func TestHandleListNodesIncludesCapabilities(t *testing.T) { `{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`), CapabilitiesHash: &hash, CapabilitiesRefreshedAt: &refreshedAt, + // Production derives this in the node store's row scanner (covered + // by TestScanNodeDerivesPhysicalGPUKeys); the stub stands in for it + // so this test can assert the handler passes the field through. + PhysicalGPUKeys: []string{"GPU-aaa"}, }, {ID: 2, Name: "old-node", Type: nodepool.NodeTypeProxy, URL: "http://old", Enabled: true}, }} diff --git a/internal/nodepool/gpuidentity.go b/internal/nodepool/gpuidentity.go new file mode 100644 index 000000000..20aee8843 --- /dev/null +++ b/internal/nodepool/gpuidentity.go @@ -0,0 +1,80 @@ +package nodepool + +import ( + "encoding/json" + "slices" +) + +// gpuIdentityView is the minimal projection this package parses out of an +// otherwise opaque capability payload, in the same spirit as +// capabilityDriftView: nodepool must not depend on playback, and identifying a +// GPU only needs the host's boot id and each render device's own identity. +type gpuIdentityView struct { + BootID string `json:"boot_id"` + RenderDeviceDetails []struct { + PCIAddress string `json:"pci_address"` + GPUUUID string `json:"gpu_uuid"` + } `json:"render_device_details"` +} + +// physicalGPUKeys derives one stable key per GPU a node can see, deduplicated +// and sorted so two nodes' key sets can be compared directly. +// +// An NVIDIA uuid is preferred because it follows the card between slots and +// hosts; the PCI address falls back to it, scoped by boot id because a device +// path and a slot only mean the same hardware within one boot of one kernel. A +// device with neither is unidentifiable and contributes no key rather than a +// fake one — a synthetic key would claim two nodes share hardware (or do not) +// on no evidence, and both directions of that claim change how work is placed. +// +// The fallback needs a boot id as much as it needs a slot. Boot id detection is +// best-effort (it reads /proc, which a hardened or sandboxed host may hide even +// while sysfs stays readable), and an empty one scopes the key to nothing: two +// unrelated hosts whose iGPU sits at the near-universal 0000:00:02.0 would both +// derive "|0000:00:02.0" and be routed as one card. So an unscoped slot is +// treated like no identity at all — nodes on such a host are accounted +// independently, which is only what they got before GPU grouping existed. +// +// A payload that cannot be parsed yields no keys: an unreadable report is not +// evidence about hardware. +func physicalGPUKeys(capabilities []byte) []string { + if len(capabilities) == 0 { + return nil + } + var identity gpuIdentityView + if err := json.Unmarshal(capabilities, &identity); err != nil { + return nil + } + seen := make(map[string]struct{}, len(identity.RenderDeviceDetails)) + keys := make([]string, 0, len(identity.RenderDeviceDetails)) + for _, device := range identity.RenderDeviceDetails { + key := device.GPUUUID + if key == "" { + if device.PCIAddress == "" || identity.BootID == "" { + continue + } + key = identity.BootID + "|" + device.PCIAddress + } + if _, duplicate := seen[key]; duplicate { + continue + } + seen[key] = struct{}{} + keys = append(keys, key) + } + if len(keys) == 0 { + return nil + } + slices.Sort(keys) + return keys +} + +// applyPhysicalGPUKeys refreshes a node's derived GPU identities from the +// capability payload it currently carries. Every place a Node is built from +// stored bytes calls it, so the field is never a stale answer to an older +// payload: the row scanner, the pools' load path, and the capability writer. +func applyPhysicalGPUKeys(n *Node) { + if n == nil { + return + } + n.PhysicalGPUKeys = physicalGPUKeys(n.Capabilities) +} diff --git a/internal/nodepool/gpuidentity_test.go b/internal/nodepool/gpuidentity_test.go new file mode 100644 index 000000000..8ebbdcd06 --- /dev/null +++ b/internal/nodepool/gpuidentity_test.go @@ -0,0 +1,135 @@ +package nodepool + +import ( + "encoding/json" + "slices" + "testing" + "time" +) + +// An NVIDIA uuid identifies a card wherever it is plugged in; a PCI address +// only identifies a slot, and only within one boot of one kernel. Deriving the +// key this way is what lets the planner — and an admin — see that two nodes are +// sharing one GPU. +func TestPhysicalGPUKeys(t *testing.T) { + tests := []struct { + name string + capabilities string + want []string + }{ + { + name: "prefers gpu uuid over slot identity", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`, + want: []string{"GPU-aaa"}, + }, + { + name: "falls back to boot-scoped pci address", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}]}`, + want: []string{"boot-1|0000:04:00.0"}, + }, + { + name: "mixed devices are deduped and sorted", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}, + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}, + {"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}]}`, + want: []string{"GPU-bbb", "boot-1|0000:05:00.0"}, + }, + { + name: "device with no identity contributes no key", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128"},{"path":"/dev/dri/renderD129","gpu_uuid":"GPU-ccc"}]}`, + want: []string{"GPU-ccc"}, + }, + { + // Boot id detection is best-effort. Without one the slot is scoped + // to nothing, and "|0000:00:02.0" — where every Intel iGPU lives — + // would merge unrelated hosts into one GPU group. + name: "a slot with no boot id contributes no key", + capabilities: `{"render_device_details":[{"pci_address":"0000:00:02.0"}]}`, + want: nil, + }, + { + // A uuid is host-independent by construction, so it survives the + // missing boot id that disqualifies its slot. + name: "a uuid still identifies a device on a host with no boot id", + capabilities: `{"render_device_details":[{"pci_address":"0000:03:00.0","gpu_uuid":"GPU-ddd"}]}`, + want: []string{"GPU-ddd"}, + }, + {name: "no capabilities stored", capabilities: "", want: nil}, + {name: "unparseable payload", capabilities: `not json`, want: nil}, + {name: "payload of the wrong shape", capabilities: `{"render_device_details":"nope"}`, want: nil}, + {name: "no render devices", capabilities: `{"boot_id":"boot-1","render_device_details":[]}`, want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := physicalGPUKeys([]byte(tt.capabilities)) + if !slices.Equal(got, tt.want) { + t.Fatalf("physicalGPUKeys() = %v, want %v", got, tt.want) + } + }) + } +} + +const gpuAAACapabilities = `{"boot_id":"boot-1","render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}` + +// Keys must exist from the moment a stored row reaches a pool, not only after +// the next capability refetch: after an API restart the planner routes on the +// inventory the database already holds. +func TestPoolsDeriveGPUKeysOnLoad(t *testing.T) { + transcodes := NewTranscodePool() + transcodes.SetNodes([]*Node{{ + ID: 1, URL: "http://tc-1/", Enabled: true, Healthy: true, + Capabilities: json.RawMessage(gpuAAACapabilities), + }}) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("transcode pool load derived %v, want [GPU-aaa]", got) + } + // The URL normalization the same loop performs must still happen. + if got := transcodes.Nodes()[0].URL; got != "http://tc-1" { + t.Fatalf("transcode pool load left URL %q unnormalized", got) + } + + proxies := NewProxyPool() + proxies.SetNodes([]*Node{ + {ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true, Capabilities: json.RawMessage(gpuAAACapabilities)}, + {ID: 3, URL: "http://proxy-2", Enabled: true, Healthy: true}, + }) + if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("proxy pool load derived %v, want [GPU-aaa]", got) + } + if got := proxies.Nodes()[1].PhysicalGPUKeys; got != nil { + t.Fatalf("node without a stored report derived %v, want none", got) + } +} + +// A refetched report replaces the identities it describes; carrying the +// previous ones over would claim a GPU that the node no longer reports. +func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { + refreshedAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + + transcodes := NewTranscodePool() + transcodes.SetNodes([]*Node{{ID: 1, URL: "http://tc-1", Enabled: true, Healthy: true}}) + transcodes.ApplyCapabilities(1, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("transcode ApplyCapabilities derived %v, want [GPU-aaa]", got) + } + + // The card was passed through to another host: the node now reports a + // device it cannot identify, and must stop claiming the old key. + transcodes.ApplyCapabilities(1, []byte(`{"boot_id":"boot-2","render_device_details":[{"path":"/dev/dri/renderD128"}]}`), + "sha256:bbb", refreshedAt) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { + t.Fatalf("stale identities survived a new report: %v", got) + } + + proxies := NewProxyPool() + proxies.SetNodes([]*Node{{ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true}}) + proxies.ApplyCapabilities(2, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt) + if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("proxy ApplyCapabilities derived %v, want [GPU-aaa]", got) + } +} diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 21de9f504..8db34457f 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -221,17 +221,19 @@ func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTr estBitrateKbps = 0 } proxies := p.proxies.Nodes() - transcodes := p.transcodes.Nodes() + pooledTranscodes := p.transcodes.Nodes() + transcodes := pooledTranscodes // Group health is computed over the full pool before any narrowing: // eligibility restricts what may be selected, not co-location semantics. - groupHealthy := groupHealth(proxies, transcodes) + // Shared-GPU load is summed over the same full pool, for the same reason. + groupHealthy := groupHealth(proxies, pooledTranscodes) var plan Plan if needsTranscode { if eligible != nil { transcodes = filterNodes(transcodes, eligible) } - plan.TranscodeNode = p.pickTranscode(transcodes, proxies, groupHealthy, currentTranscodeURL, estBitrateKbps, now) + plan.TranscodeNode = p.pickTranscode(transcodes, pooledTranscodes, proxies, groupHealthy, currentTranscodeURL, estBitrateKbps, now) if plan.TranscodeNode != nil { plan.ProxyNode = p.pickProxy(proxies, groupHealthy, plan.TranscodeNode.Group, estBitrateKbps, now) } @@ -277,12 +279,13 @@ func (p *Planner) PlanTranscodeSessionWithLocalEgress(sessionID, currentTranscod p.pruneReservations(now) delete(p.reserved, sessionID) - transcodes := p.transcodes.Nodes() - groupHealthy := groupHealth(nil, transcodes) + pooledTranscodes := p.transcodes.Nodes() + transcodes := pooledTranscodes + groupHealthy := groupHealth(nil, pooledTranscodes) if eligible != nil { transcodes = filterNodes(transcodes, eligible) } - node := p.pickLocalEgressTranscode(transcodes, groupHealthy, currentTranscodeURL, now) + node := p.pickLocalEgressTranscode(transcodes, pooledTranscodes, groupHealthy, currentTranscodeURL, now) if node == nil { return Plan{} } @@ -464,8 +467,13 @@ func groupHealth(proxies, transcodes []*Node) map[string]bool { // the session on currentURL unless a candidate has at least two fewer jobs // (the historical soft-affinity rule). Shared by pickTranscode and // pickLocalEgressTranscode, which differ only in their eligibility predicate. -func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, eligible func(*Node) bool) *Node { +// +// tieBreak, when non-nil, scores candidates that are level on effective jobs; +// the lower score wins. It never overrides the job count or the affinity rule, +// so a caller that passes nil gets exactly the historical selection. +func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, eligible func(*Node) bool, tieBreak func(*Node) int) *Node { var best, current *Node + bestJobs := 0 for _, n := range nodes { if !eligible(n) { continue @@ -473,7 +481,11 @@ func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, elig if n.URL == currentURL { current = n } - if best == nil || p.effectiveJobs(n, now) < p.effectiveJobs(best, now) { + jobs := p.effectiveJobs(n, now) + switch { + case best == nil, jobs < bestJobs: + best, bestJobs = n, jobs + case jobs == bestJobs && tieBreak != nil && tieBreak(n) < tieBreak(best): best = n } } @@ -488,11 +500,75 @@ func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, elig // pickTranscode returns the eligible transcode node with the fewest effective // jobs, keeping the session on currentURL unless a candidate has at least two -// fewer jobs (the historical soft-affinity rule). -func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { +// fewer jobs (the historical soft-affinity rule). Candidates level on job count +// are separated by the load on the physical GPU behind them: two pooled nodes +// can be two containers on one card, and spreading jobs across node records +// that share silicon does not spread the work. +// +// pool is the full transcode pool the candidates were drawn from; shared-GPU +// load is summed over all of it, since a job on a node this plan may not select +// still occupies the same GPU. +func (p *Planner) pickTranscode(transcodes, pool, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, proxies, groupHealthy, estKbps, now) - }) + }, p.physicalGPULoadScore(pool, now)) +} + +// physicalGPULoadScore precomputes, once per pick, the total effective jobs +// running on each pooled node's physical GPU group: itself plus every pooled +// node sharing at least one GPU identity with it, healthy or not, because a job +// occupies a card regardless of whether the node running it may take another. A +// node with no derived identities is a group of one. +// +// Pools only hold enabled nodes, so a node disabled while its transcodes are +// still running leaves the pool and stops counting against its group — the same +// blind spot the primary least-jobs rule already has, since there is no drain +// state between enabled and gone. +// +// Job counts only. Live GPU utilization is a richer signal but a much worse +// tie-breaker: it lags a newly admitted job by a sampling interval, so a burst +// of starts would all see the same idle card. +func (p *Planner) physicalGPULoadScore(pool []*Node, now time.Time) func(*Node) int { + jobs := make(map[*Node]int, len(pool)) + byKey := make(map[string][]*Node) + for _, n := range pool { + if n == nil { + continue + } + jobs[n] = p.effectiveJobs(n, now) + for _, key := range n.PhysicalGPUKeys { + byKey[key] = append(byKey[key], n) + } + } + + loads := make(map[*Node]int, len(pool)) + for _, n := range pool { + if n == nil { + continue + } + total := jobs[n] + counted := map[*Node]struct{}{n: {}} + for _, key := range n.PhysicalGPUKeys { + for _, peer := range byKey[key] { + // A peer sharing several keys with n must only be counted once. + if _, seen := counted[peer]; seen { + continue + } + counted[peer] = struct{}{} + total += jobs[peer] + } + } + loads[n] = total + } + + return func(n *Node) int { + if load, ok := loads[n]; ok { + return load + } + // A candidate the pool snapshot does not contain (a concurrent pool + // swap) scores as its own load, which is the no-sharing answer. + return p.effectiveJobs(n, now) + } } // pickLocalEgressTranscode applies the transcode half of normal session @@ -501,10 +577,10 @@ func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[st // suppress an otherwise healthy transcode executor. Passing nil proxies to // transcodeEligible reduces it to exactly that: healthy, enabled, under cap, // and group-healthy, with no proxy partner required. -func (p *Planner) pickLocalEgressTranscode(transcodes []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { +func (p *Planner) pickLocalEgressTranscode(transcodes, pool []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, nil, groupHealthy, 0, now) - }) + }, p.physicalGPULoadScore(pool, now)) } // transcodeEligible reports whether a transcode node may take a new session: diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index 74fd50fa4..12a5b674d 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -1,7 +1,10 @@ package nodepool import ( + "encoding/json" + "fmt" "strconv" + "strings" "testing" "time" ) @@ -784,3 +787,194 @@ func TestProxyNodeURLsListsEnabledProxies(t *testing.T) { t.Fatalf("proxy urls = %v, want both pooled proxies", urls) } } + +// gpuCapabilities is a capability report naming one render device per uuid, so +// pool loading derives exactly those identities. Written as a payload rather +// than by setting the derived field directly: the pool re-derives it on load, +// which is the behavior these tests depend on. +func gpuCapabilities(uuids ...string) json.RawMessage { + devices := make([]string, 0, len(uuids)) + for i, uuid := range uuids { + devices = append(devices, fmt.Sprintf(`{"path":"/dev/dri/renderD%d","gpu_uuid":%q}`, 128+i, uuid)) + } + return json.RawMessage(`{"boot_id":"boot-1","render_device_details":[` + strings.Join(devices, ",") + `]}`) +} + +// gpuTranscodeNode is transcodeNode carrying a capability report for the named +// GPUs. +func gpuTranscodeNode(id int, url string, activeJobs int, uuids ...string) *Node { + n := transcodeNode(id, url, nil, activeJobs) + n.Capabilities = gpuCapabilities(uuids...) + return n +} + +// Two pooled nodes can be two containers on one card. Spreading jobs evenly +// across node records that share silicon does not spread the work, so equal job +// counts are broken toward the node whose physical GPU is doing less. +func TestEqualJobsPrefersTheIdlePhysicalGPU(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 2, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + // tc-b and tc-c are level on jobs and tc-b comes first in pool order, so + // only the shared-GPU tie-break can select tc-c. + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("expected tc-c on the idle GPU, got %+v", plan.TranscodeNode) + } + + // The same rule applies to the API-relayed route, which shares pickNode. + // s1's reservation is released first so the second pick starts from the + // same job counts rather than from tc-c already charged for s1. + f.planner.ReleaseSession("s1") + local := f.planner.PlanTranscodeSessionWithLocalEgress("s2", "", nil) + if local.TranscodeNode == nil || local.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("local-egress expected tc-c, got %+v", local.TranscodeNode) + } +} + +// Jobs occupy a card whether or not the node running them may take another, so +// an unhealthy sharer — which stays pooled, only ineligible — still counts +// against its group. +func TestSharedGPULoadCountsUnusableSharers(t *testing.T) { + busy := gpuTranscodeNode(2, "http://tc-a", 3, "GPU-shared") + busy.Healthy = false + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + busy, + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("expected tc-c, got %+v", plan.TranscodeNode) + } +} + +// Boot id detection is best-effort, so two unrelated hosts can each report an +// iGPU at the near-universal slot 0000:00:02.0 with no boot id to scope it. +// Those are two cards, and planning them as one would steer work away from a +// genuinely idle GPU. +func TestUnscopedSlotsDoNotShareAGroup(t *testing.T) { + iGPU := json.RawMessage(`{"render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:00:02.0"}]}`) + busy := transcodeNode(2, "http://tc-a", nil, 2) + busy.Capabilities = iGPU + idle := transcodeNode(3, "http://tc-b", nil, 0) + idle.Capabilities = iGPU + f := newFixture([]*Node{proxyNode(1, "http://proxy-1", nil)}, []*Node{busy, idle}) + + if got := f.transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { + t.Fatalf("an unscoped slot derived %v, want no key", got) + } + loads := f.planner.physicalGPULoadScore(f.transcodes.Nodes(), f.now) + if got := loads(f.transcodes.Nodes()[1]); got != 0 { + t.Fatalf("idle host's shared-GPU load = %d, want 0 (its own jobs only)", got) + } +} + +// A node sharing several keys with the same peer must not have that peer's jobs +// counted once per key; the group is a set of nodes, not of keys. +func TestSharedGPULoadCountsEachSharerOnce(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 1, "GPU-x", "GPU-y"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-x", "GPU-y"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-p", "GPU-q"), + }, + ) + // tc-b's group load is 1 (its own 0 plus tc-a's single job counted once), + // tc-c's is 0, so tc-c wins — but only by one job, which double counting + // would inflate without changing the winner. The assertion that matters is + // the score itself. + loads := f.planner.physicalGPULoadScore(f.transcodes.Nodes(), f.now) + if got := loads(f.transcodes.Nodes()[1]); got != 1 { + t.Fatalf("tc-b shared-GPU load = %d, want 1", got) + } + if got := loads(f.transcodes.Nodes()[0]); got != 1 { + t.Fatalf("tc-a shared-GPU load = %d, want 1", got) + } + if got := loads(f.transcodes.Nodes()[2]); got != 0 { + t.Fatalf("tc-c shared-GPU load = %d, want 0", got) + } +} + +// The tie-break only ranks candidates that are already level; a node with fewer +// jobs still wins outright, even when its GPU is the busier one. +func TestFewerJobsBeatsAnIdlePhysicalGPU(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 5, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 1, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("expected the least-loaded tc-b, got %+v", plan.TranscodeNode) + } +} + +// Soft affinity outranks the tie-break: moving a running session to another +// node costs a restart, which a shared GPU is not on its own reason enough for. +func TestSharedGPUTieBreakDoesNotBreakSoftAffinity(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 2, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "http://tc-b", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("expected affinity to keep tc-b, got %+v", plan.TranscodeNode) + } +} + +// A node with no identifiable GPU is a group of itself, so a pool that reports +// none selects exactly as it did before this rule existed: least jobs, then +// pool order. +func TestNodesWithoutGPUKeysKeepPoolOrderOnTies(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + transcodeNode(2, "http://tc-a", nil, 0), + transcodeNode(3, "http://tc-b", nil, 0), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-a" { + t.Fatalf("expected the first pooled node, got %+v", plan.TranscodeNode) + } +} + +// Proxies are picked round-robin, and egress is not GPU work: a shared card +// must not perturb that rotation. +func TestSharedGPUDoesNotAffectProxySelection(t *testing.T) { + busyPeer := proxyNode(1, "http://proxy-a", nil) + busyPeer.Capabilities = gpuCapabilities("GPU-shared") + busyPeer.ActiveJobs = 9 + quiet := proxyNode(2, "http://proxy-b", nil) + quiet.Capabilities = gpuCapabilities("GPU-shared") + f := newFixture([]*Node{busyPeer, quiet}, nil) + + first := f.planner.PlanSession("s1", "", false, 0).ProxyNode + second := f.planner.PlanSession("s2", "", false, 0).ProxyNode + if first == nil || second == nil || first.URL == second.URL { + t.Fatalf("expected round-robin across both proxies, got %+v then %+v", first, second) + } +} diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index b7bf6fa5e..f33aaf638 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -19,10 +19,16 @@ func NewProxyPool() *ProxyPool { return &ProxyPool{} } -// SetNodes replaces the node list. Called on startup and when admin changes nodes. +// SetNodes replaces the node list. Called on startup and when admin changes +// nodes. Physical GPU identities are derived from each node's stored capability +// report here, so they exist from the first load rather than only after a +// capability refetch. func (p *ProxyPool) SetNodes(nodes []*Node) { p.mu.Lock() defer p.mu.Unlock() + for _, n := range nodes { + applyPhysicalGPUKeys(n) + } p.nodes = nodes } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 432e94c2c..71951b959 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -58,6 +58,14 @@ type Node struct { // what the node itself resolves against once it has reloaded its config. HWAccelOverride *string `json:"hw_accel_override,omitempty"` HWDeviceOverride *string `json:"hw_device_override,omitempty"` + // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from + // Capabilities rather than stored: it is a pure function of that payload, so + // a column would only be a second copy that can disagree with it. Two nodes + // sharing a key are backed by the same card — the case that makes + // independent per-node capacity accounting wrong, and which no single node's + // report can express. Last in the struct so it stays last on the wire, where + // the admin node list has always carried it. + PhysicalGPUKeys []string `json:"physical_gpu_keys,omitempty"` } // EffectiveHWAccel is the acceleration backend this node runs under: its own @@ -260,6 +268,10 @@ func scanNode(row pgx.Row) (*Node, error) { if len(lastStats) > 0 { n.LastStats = json.RawMessage(lastStats) } + // Derived here so every reader of a stored row — the admin listing as much + // as a pool load — sees the same identities without parsing the payload + // again for itself. + applyPhysicalGPUKeys(&n) return &n, nil } diff --git a/internal/nodepool/repository_scan_test.go b/internal/nodepool/repository_scan_test.go new file mode 100644 index 000000000..40e34e682 --- /dev/null +++ b/internal/nodepool/repository_scan_test.go @@ -0,0 +1,58 @@ +package nodepool + +import ( + "errors" + "fmt" + "slices" + "strings" + "testing" +) + +// fakeRow exercises scanNode without a database. pgx.Row is a single Scan +// method, so a stored capability payload can be handed to the real row scanner +// by writing into the destination pointers pgx would have filled. +type fakeRow struct { + capabilities []byte +} + +func (r fakeRow) Scan(dest ...any) error { + if want := strings.Count(nodeColumns, ",") + 1; len(dest) != want { + return fmt.Errorf("scanNode passed %d destinations for %d columns in nodeColumns", len(dest), want) + } + // capabilities is the first of the two jsonb columns scanned as raw bytes; + // last_stats is the second. Everything else keeps its zero value. + for _, d := range dest { + if raw, ok := d.(*[]byte); ok { + *raw = r.capabilities + return nil + } + } + return errors.New("scanNode has no raw-bytes destination for capabilities") +} + +// The admin node list is served straight from stored rows, so scanNode is the +// only producer of the physical_gpu_keys an operator sees. Losing the +// derivation there empties every Shared GPU badge silently, because the pools +// derive their own keys and keep routing correctly. +func TestScanNodeDerivesPhysicalGPUKeys(t *testing.T) { + n, err := scanNode(fakeRow{capabilities: []byte(gpuAAACapabilities)}) + if err != nil { + t.Fatalf("scanNode: %v", err) + } + if got := n.PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("scanned row derived %v, want [GPU-aaa]", got) + } + if string(n.Capabilities) != gpuAAACapabilities { + t.Fatalf("stored payload was not carried through: %s", n.Capabilities) + } + + // A node that never reported capabilities carries no identities rather + // than empty ones a client would have to special-case. + bare, err := scanNode(fakeRow{}) + if err != nil { + t.Fatalf("scanNode: %v", err) + } + if bare.Capabilities != nil || bare.PhysicalGPUKeys != nil { + t.Fatalf("row without a stored report derived %v", bare.PhysicalGPUKeys) + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 0757fd90e..336281da3 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -22,12 +22,16 @@ func NewTranscodePool() *TranscodePool { // SetNodes replaces the node list. Node URLs are normalized (trailing slashes // trimmed) at the storage boundary so every consumer compares them // consistently, including TranscodeNodeHealthy and remote-start adoption. +// Physical GPU identities are derived here too, so the planner's shared-GPU +// accounting works from the stored capability report immediately after a +// restart, before any node has advertised a changed hash. func (p *TranscodePool) SetNodes(nodes []*Node) { p.mu.Lock() defer p.mu.Unlock() for _, n := range nodes { if n != nil { n.URL = normalizeNodeURL(n.URL) + applyPhysicalGPUKeys(n) } } p.nodes = nodes @@ -133,6 +137,9 @@ func applyNodeCapabilities(nodes []*Node, id int, capabilities []byte, hash stri clone.Capabilities = append(json.RawMessage(nil), capabilities...) clone.CapabilitiesHash = &hash clone.CapabilitiesRefreshedAt = &refreshedAt + // The GPU identities belong to the payload being replaced, so they are + // re-derived rather than carried over from the previous report. + applyPhysicalGPUKeys(&clone) nodes[i] = &clone return } diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 445449922..7eac5ca95 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -41,6 +41,7 @@ import { describeNodeAccelerationOverride, describeNodeGPU, describeNodeSystem, + describeSharedGPU, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -105,7 +106,28 @@ function NodeOverrideLine({ node }: { node: StreamNode }) { ); } -function NodeGPUCell({ node }: { node: StreamNode }) { +/** + * The "Shared GPU" marker, or nothing when this node's card is its own. Muted + * rather than tinted: sharing hardware is information an operator needs when + * reading job counts, not a fault. + */ +function NodeSharedGPUBadge({ node, allNodes }: { node: StreamNode; allNodes: StreamNode[] }) { + const shared = describeSharedGPU(node, allNodes); + if (!shared) { + return null; + } + return ( + + {shared.label} + + ); +} + +function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNode[] }) { const gpu = describeNodeGPU(node); if (gpu.kind === "awaiting") { return ( @@ -120,10 +142,11 @@ function NodeGPUCell({ node }: { node: StreamNode }) { return (
-
+
{gpu.backend.label} + {gpu.failures.length > 0 && ( void; @@ -186,6 +215,7 @@ interface NodeSectionProps { function NodeSection({ type, nodes, + allNodes, infoBanner, showJobs, onAdd, @@ -279,7 +309,7 @@ function NodeSection({ - + @@ -622,6 +652,7 @@ export default function AdminNodes() { handleAdd("proxy")} onEdit={handleEdit} @@ -640,6 +671,7 @@ export default function AdminNodes() { handleAdd("transcode")} onEdit={handleEdit} diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index f3209cd7f..09f2315c6 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -8,6 +8,7 @@ import { describeNodeGPU, describeNodeSystem, describeResourceSample, + describeSharedGPU, formatBitsPerSecond, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -481,6 +482,62 @@ describe("describeNodeGPU", () => { }); }); +describe("describeSharedGPU", () => { + const alone = makeNode({ id: 1, name: "transcode-1" }); + const nvidiaA = makeNode({ id: 2, name: "transcode-a", physical_gpu_keys: ["GPU-aaa"] }); + const nvidiaB = makeNode({ id: 3, name: "transcode-b", physical_gpu_keys: ["GPU-aaa"] }); + const unique = makeNode({ id: 4, name: "transcode-c", physical_gpu_keys: ["GPU-ccc"] }); + + it("says nothing about a node that reports no identifiable GPU", () => { + expect(describeSharedGPU(alone, [alone, nvidiaA, nvidiaB])).toBeNull(); + }); + + it("says nothing when a node's GPUs are its own", () => { + expect(describeSharedGPU(unique, [unique, nvidiaA, nvidiaB])).toBeNull(); + }); + + it("names the other node on the same card, from either side", () => { + const nodes = [nvidiaA, nvidiaB, unique]; + expect(describeSharedGPU(nvidiaA, nodes)).toEqual({ + label: "Shared GPU", + title: "Shares a physical GPU with: transcode-b", + }); + expect(describeSharedGPU(nvidiaB, nodes)).toEqual({ + label: "Shared GPU", + title: "Shares a physical GPU with: transcode-a", + }); + }); + + it("matches on one key of several, across node types", () => { + const dualGPU = makeNode({ + id: 5, + name: "transcode-dual", + physical_gpu_keys: ["GPU-aaa", "boot-1|0000:04:00.0"], + }); + const proxy = makeNode({ + id: 6, + name: "proxy-same-host", + type: "proxy", + physical_gpu_keys: ["boot-1|0000:04:00.0"], + }); + + expect(describeSharedGPU(dualGPU, [dualGPU, nvidiaA, proxy])).toEqual({ + label: "Shared GPU", + title: "Shares a physical GPU with: transcode-a, proxy-same-host", + }); + }); + + it("reports nothing for a server that predates the field", () => { + const olderA = makeNode({ id: 7, name: "old-a" }); + const olderB = makeNode({ id: 8, name: "old-b" }); + expect(describeSharedGPU(olderA, [olderA, olderB])).toBeNull(); + }); + + it("does not match a node against itself when the list repeats its id", () => { + expect(describeSharedGPU(nvidiaA, [nvidiaA, nvidiaA])).toBeNull(); + }); +}); + const FULL_SAMPLE: HostSystemStats = { cpu_pct: 42, load1: 1.35, diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index 5246f86ba..eaffec305 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -347,6 +347,47 @@ function describeDeviceLine(device: NodeRenderDevice): string { return address ? `${line} (${address})` : line; } +// --- Shared physical GPUs -------------------------------------------------- + +/** The "Shared GPU" marker on a node backed by hardware another node also uses. */ +export interface SharedGPUBadge { + label: string; + /** Hover text naming the other nodes on the same card(s). */ + title: string; +} + +/** + * Describe whether a node shares a physical GPU with any other node, or null + * when it does not — which is the normal case and renders nothing. + * + * `physical_gpu_keys` is derived server-side from each node's capability + * report, so a node that reports no identifiable GPU (and every node from a + * server predating the field) carries none and can never match: an unknown GPU + * is not evidence of sharing in either direction. + */ +export function describeSharedGPU( + node: StreamNode, + allNodes: readonly StreamNode[], +): SharedGPUBadge | null { + const keys = new Set(node.physical_gpu_keys ?? []); + if (keys.size === 0) { + return null; + } + + const others = allNodes + .filter((candidate) => candidate.id !== node.id) + .filter((candidate) => (candidate.physical_gpu_keys ?? []).some((key) => keys.has(key))) + .map((candidate) => candidate.name); + if (others.length === 0) { + return null; + } + + return { + label: "Shared GPU", + title: `Shares a physical GPU with: ${others.join(", ")}`, + }; +} + // --- Per-node acceleration overrides --------------------------------------- /** From 5a3420c7778007caabe10b248b4b1b7ce14833d3 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:00:29 -0400 Subject: [PATCH 007/163] fix(nodeconfig): fall back to node name for override row lookup On split-horizon topologies the registered stream_nodes.url is the public address the API dials while NODE_URL is the node's internal one, so the URL match for the per-node override row can never hit. When the URL matches no row, the watcher now falls back to matching NODE_NAME against the registered name; an ambiguous name (no unique constraint) matches nothing and warns once. URL matches keep precedence. docs/admin-api.md documents the identity contract: keep registered names unique and NODE_NAME equal to them. Found validating phase 4 on the shared dev deployment, where both identities diverged and overrides silently inherited. Related issue: #780 Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 1 + docs/admin-api.md | 19 +++- internal/nodeconfig/watcher.go | 99 ++++++++++++++----- .../nodeconfig/watcher_overrides_db_test.go | 81 ++++++++++++++- internal/nodeconfig/watcher_overrides_test.go | 71 +++++++++++-- 5 files changed, 234 insertions(+), 37 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index fb6c6cf9f..b34c0fcb1 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -875,6 +875,7 @@ func main() { JFListen: cfg.JellyfinCompat.Listen, RedisURL: bc.RedisURL, NodeURL: nodeURL, + NodeName: nodeName, } watcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, bootstrap) if err := watcher.Start(appCtx); err != nil { diff --git a/docs/admin-api.md b/docs/admin-api.md index 55cb1e59d..2e3d9de29 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -46,11 +46,20 @@ sets `none` for itself instead of forcing every node onto the lowest common denominator. A homogeneous deployment should leave both unset and configure `playback.hw_accel` once. -A node finds its own row by URL: `NODE_URL` on the node is matched against -`stream_nodes.url`, ignoring a trailing slash on either side. Set `NODE_URL` -explicitly on every node. Without it a node guesses `http://localhost:` -and adopts whatever row carries that URL, which on a multi-node deployment can -be a different machine's policy. +A node finds its own row by URL first: `NODE_URL` on the node is matched +against `stream_nodes.url`, ignoring a trailing slash on either side. Set +`NODE_URL` explicitly on every node. Without it a node guesses +`http://localhost:` and adopts whatever row carries that URL, which on a +multi-node deployment can be a different machine's policy. + +If the URL does not match, the node falls back to `NODE_NAME` against the +registered name. This covers split-horizon topologies, where the API reaches +a node at a public URL that is registered in `stream_nodes.url`, but the +node's own `NODE_URL` is an internal address that never equals it. `name` +carries no unique constraint, so an ambiguous match — more than one row +sharing that name — identifies nothing and adopts neither row's overrides; +registered names should be unique per node, and `NODE_NAME` should equal the +registered name. The node overlays its row onto the cluster-wide playback settings on every config reload, so the override is what that node probes with, advertises in diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index 9567b9db1..80bc24de0 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -30,6 +30,10 @@ type BootstrapOverrides struct { // and overlay that row's acceleration overrides onto the cluster-wide // playback settings. Empty on an API host, which has no stream_nodes row. NodeURL string + // NodeName is this process's own registered node name (from NODE_NAME + // env). It is the override-row fallback identity when the registered url + // and NODE_URL differ, as they do on split-horizon topologies. + NodeName string } // nodeHWOverrides is one node's own acceleration policy, as stored on its @@ -40,11 +44,12 @@ type nodeHWOverrides struct { HWDevice *string } -// loadNodeHWOverrides reads one node's overrides by URL. found is false when -// the node has no stream_nodes row at all — a legitimate deployment (a node -// nobody registered yet), not an error. It is a field on the Watcher so the -// overlay can be exercised without a database. -type loadNodeHWOverrides func(ctx context.Context, nodeURL string) (overrides nodeHWOverrides, found bool, err error) +// loadNodeHWOverrides reads one node's overrides, matching its registered row +// by URL first and by NODE_NAME as the fallback. found is false when the node +// has no stream_nodes row at all — a legitimate deployment (a node nobody +// registered yet), not an error. It is a field on the Watcher so the overlay +// can be exercised without a database. +type loadNodeHWOverrides func(ctx context.Context, nodeURL, nodeName string) (overrides nodeHWOverrides, found bool, err error) // Watcher watches for configuration changes in the database and // automatically reloads the Config when changes are detected. @@ -65,10 +70,11 @@ type Watcher struct { // hiccup during a reload cannot silently flip a node back onto the // cluster-wide backend. overridesLoaded distinguishes "read, and there is // nothing to overlay" from "never read". - overrides nodeHWOverrides - overridesLoaded bool - missingRowLogged bool - duplicateRowLogged bool + overrides nodeHWOverrides + overridesLoaded bool + missingRowLogged bool + duplicateRowLogged bool + ambiguousNameLogged bool } // NewWatcher creates a new config watcher. Call Start to begin watching. The @@ -255,11 +261,11 @@ func (w *Watcher) applySettings(ctx context.Context, m map[string]string) error // read its row keeps the overlay it last read, because a database hiccup is // not evidence that an operator cleared the override. func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) { - if w.bootstrap.NodeURL == "" || w.loadOverrides == nil || cfg == nil { + if (w.bootstrap.NodeURL == "" && w.bootstrap.NodeName == "") || w.loadOverrides == nil || cfg == nil { return } - overrides, found, err := w.loadOverrides(ctx, w.bootstrap.NodeURL) + overrides, found, err := w.loadOverrides(ctx, w.bootstrap.NodeURL, w.bootstrap.NodeName) switch { case err != nil: slog.WarnContext(ctx, "node acceleration override lookup failed; keeping the previous effective policy", @@ -282,7 +288,7 @@ func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) w.mu.Unlock() if first { slog.InfoContext(ctx, "no stream_nodes row for this node; inheriting the cluster acceleration settings", - "component", "nodeconfig", "node_url", w.bootstrap.NodeURL) + "component", "nodeconfig", "node_url", w.bootstrap.NodeURL, "node_name", w.bootstrap.NodeName) } return default: @@ -312,15 +318,54 @@ func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) // first, and the 30-second health sweep rewriting those rows would silently // flip a node between two policies. The duplicate is reported rather than // quietly resolved, because only an operator can fix it. -func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL string) (nodeHWOverrides, bool, error) { +func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { if w.pool == nil { return nodeHWOverrides{}, false, errors.New("no database pool") } - rows, err := w.pool.Query(ctx, + overrides, matched, err := w.queryOverrideRows(ctx, `SELECT url, hw_accel_override, hw_device_override FROM stream_nodes WHERE rtrim(url, '/') = rtrim($1, '/') ORDER BY id LIMIT 2`, nodeURL) if err != nil { - return nodeHWOverrides{}, false, fmt.Errorf("query node acceleration overrides: %w", err) + return nodeHWOverrides{}, false, err + } + if len(matched) > 1 { + w.logDuplicateNodeRows(ctx, nodeURL, matched) + } + if len(matched) > 0 { + return overrides, true, nil + } + + // The registered url is how the API reaches the node, which on a + // split-horizon topology (public CDN url registered, internal NODE_URL on + // the node) never equals the node's own address. The name is the identity + // an operator controls on both sides, so it is the fallback — but names + // carry no unique constraint, so an ambiguous match identifies nothing. + if nodeName == "" { + return nodeHWOverrides{}, false, nil + } + overrides, matched, err = w.queryOverrideRows(ctx, + `SELECT url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE name = $1 ORDER BY id LIMIT 2`, nodeName) + if err != nil { + return nodeHWOverrides{}, false, err + } + switch len(matched) { + case 0: + return nodeHWOverrides{}, false, nil + case 1: + return overrides, true, nil + default: + w.logAmbiguousNodeName(ctx, nodeName, matched) + return nodeHWOverrides{}, false, nil + } +} + +// queryOverrideRows runs one identity query and returns the first row plus the +// urls of every row it matched, so callers can act on ambiguity. +func (w *Watcher) queryOverrideRows(ctx context.Context, query, arg string) (nodeHWOverrides, []string, error) { + rows, err := w.pool.Query(ctx, query, arg) + if err != nil { + return nodeHWOverrides{}, nil, fmt.Errorf("query node acceleration overrides: %w", err) } defer rows.Close() @@ -334,7 +379,7 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL string) (nod row nodeHWOverrides ) if err := rows.Scan(&url, &row.HWAccel, &row.HWDevice); err != nil { - return nodeHWOverrides{}, false, fmt.Errorf("scan node acceleration overrides: %w", err) + return nodeHWOverrides{}, nil, fmt.Errorf("scan node acceleration overrides: %w", err) } if len(matched) == 0 { overrides = row @@ -342,15 +387,23 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL string) (nod matched = append(matched, url) } if err := rows.Err(); err != nil { - return nodeHWOverrides{}, false, fmt.Errorf("read node acceleration overrides: %w", err) - } - if len(matched) == 0 { - return nodeHWOverrides{}, false, nil + return nodeHWOverrides{}, nil, fmt.Errorf("read node acceleration overrides: %w", err) } - if len(matched) > 1 { - w.logDuplicateNodeRows(ctx, nodeURL, matched) + return overrides, matched, nil +} + +// logAmbiguousNodeName warns once per process: several registered nodes share +// this node's NODE_NAME, so the name identifies nothing and the overrides of +// none of them are adopted. +func (w *Watcher) logAmbiguousNodeName(ctx context.Context, nodeName string, matched []string) { + w.mu.Lock() + first := !w.ambiguousNameLogged + w.ambiguousNameLogged = true + w.mu.Unlock() + if first { + slog.WarnContext(ctx, "several stream_nodes rows share this node's name; ignoring their acceleration overrides", + "component", "nodeconfig", "node_name", nodeName, "matched_urls", matched) } - return overrides, true, nil } // logDuplicateNodeRows warns, once per process, that more than one diff --git a/internal/nodeconfig/watcher_overrides_db_test.go b/internal/nodeconfig/watcher_overrides_db_test.go index f4d128851..4c3d5e0b0 100644 --- a/internal/nodeconfig/watcher_overrides_db_test.go +++ b/internal/nodeconfig/watcher_overrides_db_test.go @@ -35,13 +35,21 @@ func newOverrideTestPool(t *testing.T) *pgxpool.Pool { } func insertOverrideNode(t *testing.T, pool *pgxpool.Pool, url string, accel *string) int { + t.Helper() + return insertOverrideNodeNamed(t, pool, fmt.Sprintf("override-%d", time.Now().UnixNano()), url, accel) +} + +// insertOverrideNodeNamed is insertOverrideNode with an explicit registered +// name, for the name-fallback tests below where the name (not the +// auto-generated one) is the thing under test. +func insertOverrideNodeNamed(t *testing.T, pool *pgxpool.Pool, name, url string, accel *string) int { t.Helper() ctx := context.Background() var id int if err := pool.QueryRow(ctx, `INSERT INTO stream_nodes (name, type, url, hw_accel_override) VALUES ($1, 'transcode', $2, $3) RETURNING id`, - fmt.Sprintf("override-%d", time.Now().UnixNano()), url, accel).Scan(&id); err != nil { + name, url, accel).Scan(&id); err != nil { t.Fatalf("insert node %q: %v", url, err) } t.Cleanup(func() { @@ -70,7 +78,7 @@ func TestQueryNodeHWOverridesPicksTheSameRowAcrossReloads(t *testing.T) { w := &Watcher{pool: pool} assertPinned := func(stage string) { t.Helper() - overrides, found, err := w.queryNodeHWOverrides(ctx, base) + overrides, found, err := w.queryNodeHWOverrides(ctx, base, "") if err != nil { t.Fatalf("%s: lookup: %v", stage, err) } @@ -93,3 +101,72 @@ func TestQueryNodeHWOverridesPicksTheSameRowAcrossReloads(t *testing.T) { assertPinned("after a health sweep rewrote the row") } + +// A split-horizon node — public url registered, internal NODE_URL on the box +// — has no url match at all, so the lookup falls back to the registered name. +func TestQueryNodeHWOverridesFallsBackToUniqueName(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + registeredURL := fmt.Sprintf("https://public-%d.example.com", time.Now().UnixNano()) + pinned := "vaapi" + insertOverrideNodeNamed(t, pool, name, registeredURL, &pinned) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, "http://10.0.4.7:8082", name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if !found { + t.Fatal("name-matched row not found") + } + if overrides.HWAccel == nil || *overrides.HWAccel != pinned { + t.Fatalf("hw_accel override = %v, want the name-matched row's %q", overrides.HWAccel, pinned) + } +} + +// Two rows sharing the fallback name identify nothing: names carry no unique +// constraint, so an ambiguous match must not silently adopt either row's +// overrides. +func TestQueryNodeHWOverridesAmbiguousNameIsNotFound(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + pinned := "none" + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://a-%d.example.com", time.Now().UnixNano()), &pinned) + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://b-%d.example.com", time.Now().UnixNano()), nil) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, "http://10.0.4.7:8082", name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if found { + t.Fatalf("ambiguous name matched, overrides = %+v", overrides) + } +} + +// The url match wins even when the name also identifies a different row: url +// is the primary identity, and name is only consulted when the url misses. +func TestQueryNodeHWOverridesURLWinsOverNameMatch(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + url := fmt.Sprintf("http://url-match-%d:8082", time.Now().UnixNano()) + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + byURL := "qsv" + byName := "nvenc" + insertOverrideNodeNamed(t, pool, fmt.Sprintf("other-%d", time.Now().UnixNano()), url, &byURL) + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://different-%d.example.com", time.Now().UnixNano()), &byName) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, url, name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if !found { + t.Fatal("url-matched row not found") + } + if overrides.HWAccel == nil || *overrides.HWAccel != byURL { + t.Fatalf("hw_accel override = %v, want the url-matched %q, not the name-matched row", overrides.HWAccel, byURL) + } +} diff --git a/internal/nodeconfig/watcher_overrides_test.go b/internal/nodeconfig/watcher_overrides_test.go index 3f2d5abe5..8a572bc07 100644 --- a/internal/nodeconfig/watcher_overrides_test.go +++ b/internal/nodeconfig/watcher_overrides_test.go @@ -58,7 +58,7 @@ func TestApplySettingsOverlaysNodeHWOverrides(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { return test.overrides, true, nil }) if err := w.applySettings(context.Background(), clusterSettings()); err != nil { @@ -76,7 +76,7 @@ func TestApplySettingsOverlaysNodeHWOverrides(t *testing.T) { // The API host has no stream_nodes row and must never pay for a lookup. func TestApplySettingsSkipsOverlayWithoutNodeIdentity(t *testing.T) { looked := false - w := newOverrideWatcher(t, "", func(context.Context, string) (nodeHWOverrides, bool, error) { + w := newOverrideWatcher(t, "", func(context.Context, string, string) (nodeHWOverrides, bool, error) { looked = true return nodeHWOverrides{}, true, nil }) @@ -84,18 +84,75 @@ func TestApplySettingsSkipsOverlayWithoutNodeIdentity(t *testing.T) { t.Fatalf("apply: %v", err) } if looked { - t.Fatal("a host with no NodeURL queried stream_nodes") + t.Fatal("a host with no NodeURL or NodeName queried stream_nodes") } if got := w.Config().Playback.HWAccel; got != "qsv" { t.Fatalf("HWAccel = %q, want the cluster value", got) } } +// A split-horizon node whose registered url differs from its own NODE_URL +// still finds its row by NODE_NAME — the fallback identity an operator +// controls on both sides. The fake models the DB-level fallback behavior: +// it only reports a hit when the name matches, and this test asserts the +// overlay guard passes both bootstrap values through to the loader. +func TestApplySettingsOverlaysNodeHWOverridesByName(t *testing.T) { + accel := "vaapi" + var gotURL, gotName string + w := NewWatcher(nil, nil, nil, BootstrapOverrides{ + NodeURL: "http://10.0.4.7:8082", + NodeName: "silo-dev-transcode-ns17", + }) + w.loadOverrides = func(_ context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { + gotURL, gotName = nodeURL, nodeName + if nodeName == "silo-dev-transcode-ns17" { + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + return nodeHWOverrides{}, false, nil + } + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if gotURL != "http://10.0.4.7:8082" || gotName != "silo-dev-transcode-ns17" { + t.Fatalf("loader received url %q name %q, want both bootstrap values", gotURL, gotName) + } + if got := w.Config().Playback.HWAccel; got != "vaapi" { + t.Fatalf("HWAccel = %q, want the name-matched override", got) + } +} + +// A node with only NODE_NAME set (no NODE_URL at all) still runs the +// overlay: the guard fires when either identity is present, not just the url. +func TestApplySettingsOverlayRunsWithNodeNameOnly(t *testing.T) { + accel := "none" + looked := false + w := NewWatcher(nil, nil, nil, BootstrapOverrides{NodeName: "silo-dev-transcode-ns17"}) + w.loadOverrides = func(_ context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { + looked = true + if nodeURL != "" { + t.Fatalf("nodeURL = %q, want empty", nodeURL) + } + if nodeName != "silo-dev-transcode-ns17" { + t.Fatalf("nodeName = %q, want the bootstrap value", nodeName) + } + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if !looked { + t.Fatal("a host with NodeName set skipped the overlay lookup") + } + if got := w.Config().Playback.HWAccel; got != "none" { + t.Fatalf("HWAccel = %q, want the overlay applied", got) + } +} + // An unregistered node keeps the cluster settings, and says so once rather // than on every 60-second reload. func TestApplySettingsMissingRowInheritsAndLogsOnce(t *testing.T) { calls := 0 - w := newOverrideWatcher(t, "http://node-unregistered", func(context.Context, string) (nodeHWOverrides, bool, error) { + w := newOverrideWatcher(t, "http://node-unregistered", func(context.Context, string, string) (nodeHWOverrides, bool, error) { calls++ return nodeHWOverrides{}, false, nil }) @@ -124,7 +181,7 @@ func TestApplySettingsMissingRowInheritsAndLogsOnce(t *testing.T) { func TestApplySettingsKeepsPreviousOverrideWhenTheLookupFails(t *testing.T) { accel := "nvenc" fail := false - w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { if fail { return nodeHWOverrides{}, false, errors.New("connection refused") } @@ -149,7 +206,7 @@ func TestApplySettingsKeepsPreviousOverrideWhenTheLookupFails(t *testing.T) { // Before any successful read there is nothing to keep, so the cluster settings // stand rather than an invented value. func TestApplySettingsFallsBackToClusterWhenNoOverrideWasEverRead(t *testing.T) { - w := newOverrideWatcher(t, "http://node-1", func(context.Context, string) (nodeHWOverrides, bool, error) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { return nodeHWOverrides{}, false, errors.New("connection refused") }) if err := w.applySettings(context.Background(), clusterSettings()); err != nil { @@ -170,7 +227,7 @@ func TestApplySettingsOverlayOutlivesBootstrapReapply(t *testing.T) { Listen: ":9999", Mode: "transcode", }) - w.loadOverrides = func(context.Context, string) (nodeHWOverrides, bool, error) { + w.loadOverrides = func(context.Context, string, string) (nodeHWOverrides, bool, error) { return nodeHWOverrides{HWAccel: &accel}, true, nil } settings := clusterSettings() From f449eff79c554a1b890a201e1d4ebb3f6f47a7b3 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:08:22 -0400 Subject: [PATCH 008/163] fix(nodemetrics): prefer lxcfs-scoped /host/proc files when mounted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node running in Docker nested inside an LXC sees the kernel's raw /proc (bare-metal totals) and an unlimited own cgroup, because the LXC's cap lives on an ancestor cgroup invisible from the nested namespace — so cpu, cores, load, and memory reported the host. The sampler now prefers /host/proc/{stat,loadavg,meminfo} when present, which deployments bind-mount from the LXC where lxcfs virtualizes them to the container's real limits. net/dev and per-PID fdinfo reads deliberately stay on the container's own /proc. Documented with a compose snippet in the docker guide. Related issue: #780 Co-Authored-By: Claude Fable 5 --- docs/admin-api.md | 7 ++ docs/wiki/deployment/docker.md | 28 ++++++ internal/nodemetrics/sampler.go | 12 ++- internal/nodemetrics/sampler_test.go | 142 +++++++++++++++++++++++++++ internal/nodemetrics/system.go | 25 ++++- 5 files changed, 210 insertions(+), 4 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 2e3d9de29..0877d5c69 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -90,6 +90,13 @@ same `streamapp_node_*` gauges, with disk series labeled by role rather than by path). A sample larger than 32 KiB is dropped rather than stored — the health verdict is what routes streams, and no honest sample comes close to that. +Cgroup correction alone is not always enough: a Docker container nested inside +an LXC container sees no limit on its own cgroup (the LXC's cap lives on an +ancestor cgroup outside its namespace), so `cpu_pct`, `cores`, `load1`, and the +memory fields below read as the bare-metal host's totals unless the deployment +bind-mounts lxcfs's virtualized `/proc` files in — see the LXC section of +[docs/wiki/deployment/docker.md](wiki/deployment/docker.md#node-metrics). + `last_stats.system`: | Field | Type | Meaning | diff --git a/docs/wiki/deployment/docker.md b/docs/wiki/deployment/docker.md index 260841ed9..bba1f603a 100644 --- a/docs/wiki/deployment/docker.md +++ b/docs/wiki/deployment/docker.md @@ -262,6 +262,34 @@ the API host. A mount that stops responding reports its last good numbers marked `stale` and never delays a health response; a path a node cannot see at all is reported `unavailable` rather than as an empty disk. +**LXC hosts running Docker nested inside them.** The cgroup correction above +only works when the limit is visible on *this* container's own cgroup. On an +LXC host, a Docker container nested inside the LXC sees the raw kernel's +`/proc/stat`, `/proc/loadavg`, and `/proc/meminfo` — the physical machine's +totals, not the LXC's — while its own cgroup shows no limit at all, because the +LXC's cap lives on an ancestor cgroup outside the nested container's namespace. +lxcfs, which every LXC container mounts for exactly these files, virtualizes +them to the LXC's own limits, so bind-mounting that virtualized view into the +Docker container fixes it: + +```yaml +volumes: + - /proc/meminfo:/host/proc/meminfo:ro + - /proc/stat:/host/proc/stat:ro + - /proc/loadavg:/host/proc/loadavg:ro +``` + +These three mounts are only useful when the Docker host is itself an LXC/lxcfs +container — a bare-metal or VM Docker host has nothing extra to gain from them, +since its own `/proc` is already correct or already cgroup-corrected. Without +them, a node running nested this way reports the bare-metal host's CPU, memory, +and load totals instead of its own. + +One caveat: lxcfs only virtualizes `/proc/loadavg` when it runs with loadavg +accounting enabled (its `-l` flag; on Proxmox, `lxcfs` defaults to off). With +it off, cores and memory are container-scoped but `load1` remains the physical +host's — visibly higher than the container's core count under host contention. + ## Optional Meilisearch PostgreSQL full-text search needs no extra service. To offer Meilisearch as an diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 758be16e4..ba7536b1c 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -64,7 +64,11 @@ type Sampler struct { // Path seams. Production values point at the real filesystem; tests point // them at a fake /proc tree. - procDir string + procDir string + // hostProcDir is where an LXC's lxcfs-virtualized /proc files can be + // bind-mounted when this sampler runs in Docker nested inside an LXC + // container. See procDirFor for why it takes priority when present. + hostProcDir string cgroupLimitPaths []string cgroupUsagePaths []cgroupUsagePath cgroupCPUPaths []cgroupCPUPath @@ -106,6 +110,7 @@ func NewSampler(opts Options) *Sampler { now = time.Now } procDir := "/proc" + hostProcDir := "/host/proc" ffmpegPIDs := opts.FFmpegChildren if ffmpegPIDs == nil { pid := os.Getpid() @@ -121,6 +126,7 @@ func NewSampler(opts Options) *Sampler { identities: opts.DeviceIdentities, ffmpegPIDs: ffmpegPIDs, procDir: procDir, + hostProcDir: hostProcDir, cgroupLimitPaths: CgroupMemoryLimitPaths(), cgroupUsagePaths: slices.Clone(cgroupMemoryUsagePaths), cgroupCPUPaths: slices.Clone(cgroupCPUPaths), @@ -217,7 +223,7 @@ func (s *Sampler) sampleSystem(ctx context.Context, now time.Time) *SystemStats return &SystemStats{ CPUPct: cpuPct, - Load1: readLoad1(s.procDir), + Load1: readLoad1(s.procDirFor("loadavg")), Cores: cores, MemUsedMB: bytesToMB(usedBytes), MemTotalMB: bytesToMB(totalBytes), @@ -236,7 +242,7 @@ func (s *Sampler) sampleSystem(ctx context.Context, now time.Time) *SystemStats // — and a node pinned at its quota, which is the state worth alerting on, would // look nearly idle. func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { - host, hostCores := readCPUTimes(s.procDir) + host, hostCores := readCPUTimes(s.procDirFor("stat")) busyPct, _ = cpuBusyPercent(s.prevCPU, host) if host.valid { s.prevCPU = host diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 67fe3844c..5fd7254cd 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -58,6 +58,11 @@ func newTestSampler(t *testing.T, tree *procTree, clock *fakeClock, opts Options s := NewSampler(opts) s.goos = "linux" s.procDir = tree.root + // Point hostProcDir at a path that does not exist by default so tests are + // deterministic regardless of whether the machine running them happens to + // have a real /host/proc; tests that exercise the lxcfs override set this + // explicitly to a tree that does. + s.hostProcDir = filepath.Join(tree.root, "no-such-host-proc") s.cgroupLimitPaths = nil s.cgroupUsagePaths = nil s.cgroupCPUPaths = nil @@ -532,6 +537,143 @@ func TestSnapshotJSONShape(t *testing.T) { } } +// On an LXC host running Docker nested inside it, this process's own /proc +// sees the raw kernel — the bare-metal core count, memory, and load — while +// its own cgroup is unlimited, because the LXC's cap lives on an ancestor +// cgroup outside its namespace. lxcfs virtualizes /proc/stat, /proc/loadavg, +// and /proc/meminfo to the LXC's own limits, so when those are bind-mounted +// in at hostProcDir, they must win over the container's raw /proc for exactly +// those three files. +func TestHostProcOverridesLxcfsScopedFiles(t *testing.T) { + tree := newProcTree(t) + hostTree := newProcTree(t) + clock := newFakeClock() + + // The nested container's own /proc: a busy 8-core bare-metal host, 64 GiB + // of memory, and a load average that belongs to every other tenant on the + // box too. + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n"+ + "cpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0\n"+ + "cpu4 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "40.00 38.00 35.00 12/900 5555\n") + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + // lxcfs's view, bind-mounted at hostProcDir: the LXC is capped at 2 cores + // and 2 GiB, and its own load average. + hostTree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + hostTree.write("loadavg", "1.50 1.20 0.80 1/64 42\n") + hostTree.write("meminfo", "MemTotal: 2097152 kB\nMemAvailable: 1048576 kB\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.hostProcDir = hostTree.root + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the lxcfs-scoped 2, not the host's 8", system.Cores) + } + if system.Load1 != 1.50 { + t.Fatalf("Load1 = %v, want the lxcfs-scoped 1.50, not the raw host's 40.00", system.Load1) + } + if system.MemTotalMB != 2048 { + t.Fatalf("MemTotalMB = %d, want the lxcfs-scoped 2048, not the raw host's 65536", system.MemTotalMB) + } + if system.MemUsedMB != 1024 { + t.Fatalf("MemUsedMB = %d, want 1024 (2048 total - 1024 available)", system.MemUsedMB) + } + + // A second sample drives the CPU busy-percent delta off the lxcfs stat + // file too: 400 more busy jiffies out of 1000 more total. + hostTree.write("stat", "cpu 300 0 300 1400 0 0 0 0\ncpu0 150 0 150 700 0 0 0 0\ncpu1 150 0 150 700 0 0 0 0\n") + tree.write("stat", "cpu 99999 0 0 1 0 0 0 0\n") // the raw host is pegged; must not be read + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system = s.Snapshot().System + if system.CPUPct != 40 { + t.Fatalf("CPUPct = %d, want 40 from the lxcfs stat deltas, not the raw host's near-100", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2 on the second sample too", system.Cores) + } +} + +// Without a bind-mounted lxcfs view — plain Docker, bare metal, or an LXC +// deployment that has not mounted /host/proc — every reading must fall back +// to the container's own /proc exactly as before this feature existed. +func TestHostProcAbsentFallsBackToProcDir(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + tree.write("loadavg", "3.20 1.10 0.90 2/512 9\n") + tree.write("meminfo", "MemTotal: 8388608 kB\nMemAvailable: 6291456 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + // newTestSampler already points hostProcDir at a nonexistent path; this + // test just makes that fallback explicit and asserts on it. + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2 from procDir", system.Cores) + } + if system.Load1 != 3.20 { + t.Fatalf("Load1 = %v, want 3.2 from procDir", system.Load1) + } + if system.MemTotalMB != 8192 { + t.Fatalf("MemTotalMB = %d, want 8192 from procDir", system.MemTotalMB) + } + if system.MemUsedMB != 2048 { + t.Fatalf("MemUsedMB = %d, want 2048 from procDir", system.MemUsedMB) + } +} + +// Only stat, loadavg, and meminfo are host-proc aware. net/dev is per-netns +// and must stay on the container's own /proc even when hostProcDir has its +// own net/dev sitting right next to the other three files — otherwise a +// nested node would report the LXC host's aggregate network traffic instead +// of its own. +func TestHostProcDoesNotAffectNetDev(t *testing.T) { + tree := newProcTree(t) + hostTree := newProcTree(t) + clock := newFakeClock() + + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + netDev := func(rx, tx int) string { + return "Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + + " eth0: " + itoa(rx) + " 1 0 0 0 0 0 0 " + itoa(tx) + " 1 0 0 0 0 0 0\n" + } + tree.write("net/dev", netDev(1000, 2000)) + + // hostTree has its own net/dev with very different counters. If it were + // ever consulted, the throughput computed below would not match. + hostTree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + hostTree.write("loadavg", "0 0 0 0/0 0\n") + hostTree.write("meminfo", "MemTotal: 1024 kB\n") + hostTree.write("net/dev", netDev(9_000_000, 9_000_000)) + + s := newTestSampler(t, tree, clock, Options{}) + s.hostProcDir = hostTree.root + s.sample(context.Background()) + + tree.write("net/dev", netDev(6000, 12000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.NetRxBps != 8000 { // 5000 bytes / 5s * 8, from tree's net/dev + t.Fatalf("NetRxBps = %d, want 8000 from procDir's net/dev, not hostProcDir's", system.NetRxBps) + } + if system.NetTxBps != 16000 { // 10000 bytes / 5s * 8 + t.Fatalf("NetTxBps = %d, want 16000 from procDir's net/dev, not hostProcDir's", system.NetTxBps) + } +} + // itoa avoids importing strconv into every fixture builder above. func itoa(v int) string { if v == 0 { diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go index fb38aab7f..25b5d69dd 100644 --- a/internal/nodemetrics/system.go +++ b/internal/nodemetrics/system.go @@ -167,6 +167,29 @@ func netThroughputBps(previous, current netCounters) (rxBps, txBps int64, ok boo return rate(previous.rx, current.rx), rate(previous.tx, current.tx), true } +// procDirFor resolves which proc tree to read name ("stat", "loadavg", or +// "meminfo") from — the three files a container's cgroup cannot itself +// correct. +// +// On an LXC host running Docker nested inside it, this process's own /proc is +// the raw kernel view: /proc/stat, /proc/loadavg, and /proc/meminfo describe +// the physical machine rather than the LXC, and this nested container's own +// cgroup shows no limit at all, because the LXC's cap lives on an ancestor +// cgroup outside this container's namespace that it cannot see or read. +// lxcfs, running on the LXC host, virtualizes those same three files to the +// LXC's own limits; when an operator bind-mounts that virtualized view in at +// hostProcDir/, it is the only correct source, so it wins whenever +// present. Otherwise procDir/ — the file this process actually sees — +// is the only option, exactly as on plain Docker or bare metal. +func (s *Sampler) procDirFor(name string) string { + if s.hostProcDir != "" { + if _, err := os.Stat(filepath.Join(s.hostProcDir, name)); err == nil { + return s.hostProcDir + } + } + return s.procDir +} + // memoryStats reports used and total bytes for this process's memory domain. // // /proc/meminfo describes the host even inside a container, so a cgroup limit — @@ -174,7 +197,7 @@ func netThroughputBps(previous, current netCounters) (rxBps, txBps int64, ok boo // it, and cgroup usage (page cache excluded) wins over the host's own // used figure for the same reason. func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { - fields, err := ReadMeminfoBytes(filepath.Join(s.procDir, "meminfo")) + fields, err := ReadMeminfoBytes(filepath.Join(s.procDirFor("meminfo"), "meminfo")) if err == nil { totalBytes = fields["MemTotal"] if available, ok := fields["MemAvailable"]; ok && totalBytes >= available { From 9e846c07a22a852cfbc6d59ed8f9faafe55f4774 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:18:09 -0400 Subject: [PATCH 009/163] =?UTF-8?q?feat(nodes):=20operational=20QoL=20?= =?UTF-8?q?=E2=80=94=20re-probe,=20scratch=20guard,=20drift=20surfacing,?= =?UTF-8?q?=20device=20picker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-probe: exported probe-cache invalidation in playback and tonemap, a bearer-authed POST /admin/reprobe-capabilities on transcode and proxy nodes (refuses with 409 while transcodes run so a loaded smoke encode can never publish a false hardware regression; 503 keeps the previous hash), and POST /api/v1/admin/nodes/{id}/reprobe which extends the connection write deadline past the probe budget and immediately refetches and persists the new report through the health checker's existing machinery. Scratch admission: the sampler marks the scratch disk in the payload and the planner soft-excludes transcode candidates at >=95% scratch fill — never emptying the candidate set (degraded beats down), conservative on missing or stale stats, with latched transition logging that says whether the guard actually excluded or was dropped. Drift: the capability drift the sweep already computes persists to a new nullable stream_nodes.capability_drift column (UTF-8-safe 512-byte note, cleared only when every attempted probe passes again) and surfaces as a warning badge on the Nodes page. UI: hw_device_override becomes a checkbox picker fed by the node's stored device inventory (shared parser extracted to lib/hwDevices, free-text fallback without inventory, unknown configured paths preserved) plus a per-row Re-probe action. New docs/wiki/admin/monitoring-nodes.md operator guide and admin-api.md coverage for the new endpoints and fields. Related issue: #780 Co-Authored-By: Claude Fable 5 --- docs/admin-api.md | 143 +++++++++ docs/wiki/admin/monitoring-nodes.md | 284 ++++++++++++++++++ docs/wiki/index.md | 2 + internal/api/handlers/nodes.go | 239 +++++++++++++++ internal/api/handlers/nodes_reprobe_test.go | 262 ++++++++++++++++ internal/api/handlers/nodes_test.go | 7 +- internal/api/router.go | 8 + internal/nodemetrics/collector.go | 6 +- internal/nodemetrics/disk.go | 4 +- internal/nodemetrics/disk_test.go | 58 ++++ internal/nodemetrics/sampler.go | 9 - internal/nodemetrics/snapshot.go | 8 + internal/nodepool/gpuidentity_test.go | 6 +- internal/nodepool/health.go | 265 ++++++++++++++-- internal/nodepool/health_drift_test.go | 252 ++++++++++++++++ internal/nodepool/planner.go | 163 +++++++++- internal/nodepool/planner_scratch_test.go | 237 +++++++++++++++ internal/nodepool/proxy_pool.go | 4 +- internal/nodepool/repository.go | 27 +- .../nodepool/repository_capabilities_test.go | 53 +++- internal/nodepool/scratchpressure.go | 79 +++++ internal/nodepool/scratchpressure_test.go | 81 +++++ internal/nodepool/transcode_pool.go | 11 +- internal/playback/gpudetect.go | 23 ++ .../playback/gpudetect_invalidate_test.go | 55 ++++ internal/playback/gpudetect_test.go | 7 +- internal/proxy/reprobe.go | 77 +++++ internal/proxy/reprobe_test.go | 73 +++++ internal/proxy/server.go | 1 + internal/proxy/testdata/media_routes.txt | 2 + internal/tonemap/probe.go | 20 ++ internal/tonemap/probe_invalidate_test.go | 53 ++++ internal/tonemap/probe_test.go | 8 +- internal/transcodenode/reprobe.go | 107 +++++++ internal/transcodenode/reprobe_test.go | 124 ++++++++ internal/transcodenode/server.go | 1 + .../transcodenode/testdata/media_routes.txt | 2 + .../20260827125333_node_capability_drift.sql | 25 ++ web/src/api/types.ts | 36 +++ web/src/hooks/queries/admin/nodes.ts | 41 +++ web/src/lib/hwDevices.ts | 39 +++ web/src/pages/AdminNodes.tsx | 172 +++++++++-- .../admin-settings/playbackSettings.utils.ts | 36 +-- web/src/pages/adminNodesPresentation.test.ts | 214 ++++++++++++- web/src/pages/adminNodesPresentation.ts | 191 +++++++++++- 45 files changed, 3387 insertions(+), 128 deletions(-) create mode 100644 docs/wiki/admin/monitoring-nodes.md create mode 100644 internal/api/handlers/nodes_reprobe_test.go create mode 100644 internal/nodepool/health_drift_test.go create mode 100644 internal/nodepool/planner_scratch_test.go create mode 100644 internal/nodepool/scratchpressure.go create mode 100644 internal/nodepool/scratchpressure_test.go create mode 100644 internal/playback/gpudetect_invalidate_test.go create mode 100644 internal/proxy/reprobe.go create mode 100644 internal/proxy/reprobe_test.go create mode 100644 internal/tonemap/probe_invalidate_test.go create mode 100644 internal/transcodenode/reprobe.go create mode 100644 internal/transcodenode/reprobe_test.go create mode 100644 migrations/sql/20260827125333_node_capability_drift.sql create mode 100644 web/src/lib/hwDevices.ts diff --git a/docs/admin-api.md b/docs/admin-api.md index 0877d5c69..cf9c1fefc 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -32,6 +32,7 @@ Always `200 OK` with a JSON array. | `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | | `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | +| `capability_drift` | string | Human-readable note describing how the node's hardware got worse at the last capability refetch. Omitted when the last refetch found no regression (see below). | ### Acceleration overrides @@ -116,6 +117,39 @@ Each entry in `disks`: | `used_gb`, `total_gb` | float | Capacity in GiB. Used counts filesystem-reserved blocks, matching `df`. | | `stale` | bool | The numbers are real but carried over from an earlier pass because the current probe has not returned — the normal reading for a network mount whose server went away. Omitted when false. | | `unavailable` | bool | The path has never been measured on this node (it does not exist here, or the first probe is still hanging). `used_gb`/`total_gb` are meaningless. Omitted when false. | +| `scratch` | bool | This is the node's transcode working directory. Set on at most one entry; a media root sharing that volume is deduplicated onto it. Omitted when false. | + +`scratch` exists because this server does not know a node's transcode directory — +the node does. It is the one mount whose filling up breaks transcoding rather +than browsing, so the entry has to identify itself. Node selection reads it: +see "Scratch admission" below. It is also what labels the node's own +`streamapp_node_disk_*` series `scratch` instead of `library-N`. + +### Scratch admission + +A transcode writes HLS segments to its node's scratch volume for the whole life +of the session, so admitting one onto a nearly full node produces a stream that +dies mid-playback — after the client has already committed to it. Transcode +selection therefore skips a node whose `scratch` entry reports **95% or more** +used, and prefers a node with headroom even when the full one carries fewer +jobs. + +The exclusion is soft in two directions: + +- If it would leave no eligible candidate at all, it is ignored and the ordinary + least-jobs selection stands. Degraded service beats no service, and 95% was + not chosen to be a kill switch for a whole cluster. +- A node whose fill cannot be read is never excluded: no sample, no `scratch` + entry (a node predating the flag), an unmeasurable path, or numbers the node + itself marked `stale`. Taking capacity away on a fill we cannot read would be + worse than the failure it prevents. + +Each transition into pressure is logged once per node (`component=nodepool`, +"scratch volume nearly full"), not once per session start. + +Nothing else routes on `last_stats`, and the guard applies to playback and +local-egress transcode selection only — not to proxy selection, and not to the +non-streaming transcode reservations used by prepared downloads. Each entry in `last_stats.gpu`: @@ -151,6 +185,48 @@ one at a time per node, so a slow capability probe cannot delay the health cadence of the other nodes; a new report can therefore land shortly after the check that noticed the change rather than with it. +An operator who cannot wait for the sweep — or whose node will never advertise a +changed hash because its probe results are cached for its process lifetime — uses +`POST /api/v1/admin/nodes/{id}/reprobe`, which stores the new report before it +answers. + +### `capability_drift` + +Set when a capability refetch shows the node's hardware got **worse** than the +report it replaced: a backend that used to pass its FFmpeg probe and now fails, +or a render device that is gone. It reads like +`verified hardware backends lost: qsv; render devices gone: /dev/dri/renderD128; +resolved backend qsv -> none`, and is capped at 512 characters. + +It exists because that regression is otherwise only a log line, and the node +stays `healthy` throughout: a driver that stopped working silently turns a GPU +transcoder into a CPU one, which shows up to users as slow or failing playback +long before anyone reads a log. + +Semantics worth knowing: + +- Setting it is a comparison; clearing it is not. The note appears when a refetch + loses something, and stays until a refetch produces a report whose probes all + pass. A refetch that finds nothing *newly* lost leaves it alone, because a + delta against an already-degraded report always finds nothing — a reboot moves + `boot_id`, a reworded FFmpeg failure moves the probe reason, and either would + otherwise erase a standing regression and report a broken node as repaired. +- A backend reported as `skipped` does not hold the note open. Skipping means + the node cannot open the devices, which is a statement about access rather + than about hardware. +- Only a loss is reported. Added hardware is not drift. +- A node's first stored report carries none — there is nothing to compare it + against. +- It is written in the same statement as `capabilities` and `capabilities_hash`, + so it always describes the report stored beside it. +- Nothing routes on it. Node selection reads `healthy`, capacity, capability + eligibility, and the scratch guard above — never this field. + +Refetches only happen when a node advertises a changed `capabilities_hash`, so a +node whose GPU broke while its process kept running may report nothing new: the +probe results are cached for its process lifetime. `POST +/api/v1/admin/nodes/{id}/reprobe` is what forces the question. + ### `physical_gpu_keys` One key per render device in the stored report, deduplicated and sorted: @@ -247,6 +323,73 @@ This is the node's *current* hash, not the stored one. A value here that differs from the `capabilities_hash` in the list response means the background sweep has a refetch pending; this route does not fetch capabilities itself. +## `POST /api/v1/admin/nodes/{id}/reprobe` + +Tells one node to discard its cached hardware-probe verdicts and re-verify +against live hardware, then refetches and stores the resulting inventory +immediately. + +This is the answer to hardware that stopped working underneath a running node. A +node caches a **successful** probe for its whole process lifetime — re-verifying +per request would put FFmpeg execs on the playback path — so a GPU that has since +been removed, or whose driver was replaced with one that cannot encode, keeps +reporting `verified: true` until the node restarts. That is not visible in a +health check, because the node is healthy either way. Use it after installing or +downgrading a GPU driver, after changing which devices a node's container can +open, after replacing an FFmpeg build in place, and to confirm a +`capability_drift` note is still true. + +The reverse needs no action: a **failed** GPU probe carries a 15-second negative +TTL and is retried on its own, so a repaired driver flips `verified` to `true`, +changes the node's `capabilities_hash`, and is refetched within one snapshot +interval. The exception is the tone-map matrix, which caches any non-empty +inventory for the process lifetime, so a node whose GPU was broken at start can +stay software-only for tone mapping until it is re-probed or restarted. + +Body: none. Always `200 OK`; a node that refused or could not be reached is +reported in the body rather than as an HTTP error status, matching +`{id}/check` and `{id}/force-reload`. `404 Not Found` for an unknown id. + +| Field | Type | Meaning | +|---|---|---| +| `node_id`, `node_name` | int, string | The node this action ran against. | +| `status` | string | `ok` or `error`. | +| `error` | string | Why the node failed. Omitted on success. | +| `resolved` | string | The backend the node picked after re-probing. Omitted on failure. | +| `capability_hash` | string | The snapshot the node published. Compare it against `capabilities_hash` from the list response taken *before* the call to see whether anything changed. Omitted on failure. | +| `capabilities_refreshed` | bool | Whether this server also stored the node's new inventory before answering. | + +`capabilities_refreshed: false` with `status: ok` means the node re-probed but +the stored row has not caught up yet — a refresh for that node was already +running, or this deployment has no health sweep. The next sweep stores it. + +A node whose probe could not complete answers `status: error` and **keeps its +previous capability report**: an unfinished probe is not evidence the hardware +changed, and publishing a partial one would announce a change that did not +happen. Nothing is stored in that case, so a failed re-probe never degrades what +the list shows. + +A node that is **transcoding refuses**, also as `status: error`, and keeps its +report. Every hardware probe ends in a real encode on the GPU; a card at its +concurrent encoder-session limit fails that encode with an error nothing can +distinguish from a missing device, and the resulting `verified: false` would be +stored as a hardware regression for a GPU that is at that moment encoding. +Disable the node or wait for it to drain, then re-probe. + +The call can take a while. The node is given the probe budget it advertises in +its own report (`probe_request_timeout_ms`, up to five minutes); a node that has +never been inventoried gets 150 seconds. The capability refetch that follows adds +up to two minutes. The connection's write deadline is extended to cover both, so +this route can legitimately outlive the API listener's ordinary 120-second +`WriteTimeout`; a client should allow for that rather than treating a long wait +as a hung request. This re-probes only — it does not reload configuration or tear +anything down. + +Under the hood this is a bearer-authenticated `POST +/admin/reprobe-capabilities` on the node's own listener, which both transcode +nodes and proxy nodes serve. That route is internal to the cluster and is not +part of any client contract. + ## `GET /api/v1/admin/system/hw-accel` Reports GPU hardware and acceleration capability. With healthy transcode nodes diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md new file mode 100644 index 000000000..27db232de --- /dev/null +++ b/docs/wiki/admin/monitoring-nodes.md @@ -0,0 +1,284 @@ +--- +title: Monitoring Stream Nodes +description: How to read the admin Nodes page, when to re-probe a node's hardware, and how scratch-disk pressure affects transcode placement. +summary: What each Nodes-page column means, the GPU re-probe action, scratch admission, and scraping node metrics with Prometheus. +tags: + - silo + - docs + - wiki + - nodes + - gpu + - monitoring +audience: + - operator +last_reviewed: 2026-08-27 +related: + - ../index.md + - ../deployment/docker.md +--- + +# Monitoring Stream Nodes + +**Settings -> Nodes** lists every registered proxy and transcode node with its +current health, its verified GPU capability, and its host resource usage. This +page explains what the columns mean, which of them affect where playback is +placed, and what to do when the page disagrees with the hardware you know is in +the machine. + +Everything here applies to a distributed deployment. A single-node install has no +registered nodes, and its own resource usage appears on the admin dashboard +instead. + +## The columns + +### Status and Health + +**Status** is whether the node is *enabled* — an administrator's switch. A +disabled node is in no pool and is never selected, and it stops counting against +its co-location group. + +**Health** is the result of the last check. Every node is polled every 30 +seconds; a node that does not answer is routed around immediately, and existing +streams on it fail over on their next segment request. **Last Check** is when +that poll ran. + +Health and capability are independent. A node whose GPU driver broke stays +perfectly healthy: it still answers, it just encodes in software now. That is +what the GPU column and the drift badge exist to surface. + +### GPU + +The GPU column reports what the node's FFmpeg could actually *do* the last time +it was asked, not what its configuration names. Silo verifies hardware by running +a real single-frame encode on each candidate device, so the states are evidence, +not guesses: + +| Reading | Meaning | +|---|---| +| **QSV / VAAPI / NVENC**, green | The backend passed its FFmpeg probe on this node. Hover for the device it passed on. | +| **QSV / VAAPI / NVENC**, amber | The backend is configured and in use, but its probe *failed*. Hover for the reason FFmpeg gave. Transcodes will attempt it anyway, because an explicitly configured backend is honored verbatim. | +| **QSV / VAAPI / NVENC**, plain | The backend is in use but the node reported no verification for it — normal when a backend is configured on a node with no candidate devices to probe. | +| **SW** | No hardware backend verified: this node encodes in software. | +| **SW**, "devices not accessible" on hover | Every candidate device was *skipped* rather than probed, because this process cannot open any of them. This is the normal reading for a proxy node reading a cluster-wide `playback.hw_device` that points at the transcode nodes' cards. It is not a driver failure. | + +Below the badge, each render device the node can see is listed with its live +engine busyness where a measurement source exists. + +**`stale`** on the GPU cell means no health check has *confirmed* this inventory +for more than ten minutes. Checks run every 30 seconds and every response carries +the node's current capability hash, so roughly twenty checks in a row have to go +missing before the marker appears — it says the confirming sweep stopped, not +that the report is old. An old report is normal and is deliberately not flagged: +a node recomputes its snapshot every 15 minutes, the server refetches only when +the advertised hash changes, and a node whose GPU has not changed in a week is +serving a week-old report that is still true. An unhealthy node is never marked +stale, since it cannot confirm a report at all. + +**`Shared GPU`** marks a node whose physical card is also visible to another +registered node — two containers on one GPU, most often. Silo detects this from +each report's device identity (NVIDIA GPU UUID where available, otherwise the PCI +slot scoped to the host's boot id) and uses it when placing work: among transcode +nodes level on job count, the one whose *card* carries the fewest jobs wins. +Without that, spreading sessions across node records that share silicon would not +spread the work at all. + +**Drift**, an amber badge, means a capability refetch found the node's hardware +got *worse* than the report it replaced: a backend that used to pass its probe +now fails, or a render device is gone. Hover for the note. The badge stays until +a refetch produces a report whose probes all pass — it is not erased by the next +refetch that merely loses nothing further, so a reboot or a reworded FFmpeg error +cannot make a standing regression look repaired. Re-probing the node is the +direct way to ask whether it is still true. It is a warning, not a routing input +— nothing in node selection reads it. + +### System + +CPU, memory, the fullest sampled disk, and network throughput, sampled by the +node itself every five seconds and carried on its health response. It is the +current sample only; Silo keeps no history (see [Scraping node +metrics](#scraping-node-metrics) below). + +An unhealthy node shows a dash rather than its last numbers: the sample predates +the check that failed, and a frozen CPU percentage looks exactly like a live one. +A dash on a *healthy* node means the node reported no sample — sampling is +Linux-only, and a node running an older build reports none. + +The disk reading is the fullest mount the node can see, and it tints when that +mount passes 85% used. A mount whose server stopped responding shows its last +good numbers rather than blocking the health response; a path the node cannot see +at all is reported as unmeasurable rather than as an empty disk. + +In a container these numbers are the *container's*, corrected against its cgroup +— except on an LXC host running Docker nested inside it, which needs three +bind-mounts to read its own limits instead of the physical machine's. See the LXC +notes in [Deploy Silo with Docker](../deployment/docker.md#node-metrics), +including the caveat that lxcfs only virtualizes `/proc/loadavg` when it runs +with loadavg accounting enabled (`lxcfs -l`; off by default on Proxmox), so +`load1` can remain the physical host's while CPU and memory are correct. + +## Re-probing a node's hardware + +Each node row has a **Re-probe** action beside Check. It tells that node to throw +away its cached hardware verdicts, re-verify against live hardware, and hand the +fresh inventory straight back to the server. + +It exists because a *successful* probe is cached for the node's whole process +lifetime. That is deliberate — re-verifying on every playback request would put +FFmpeg executions on the playback path — but it means a node goes on reporting +hardware that has since stopped working: + +- **After a GPU driver upgrade, downgrade, or reinstall.** A card that worked + when the node started keeps reading as verified until the node restarts, even + once the new driver cannot encode a frame. +- **After changing which devices a container can open** (removing a `/dev/dri` + passthrough, switching the NVIDIA overlay, changing group membership). +- **After replacing an FFmpeg build in place**, where the path did not change. +- **To confirm a drift badge is still true**, rather than a transient failure + during a driver reload. + +The opposite direction needs no action. A *failed* GPU probe is cached for only +15 seconds, so a card that was broken when the node started and has since been +repaired verifies on its own; that flips the node's capability hash and the +server picks it up within one 15-minute snapshot. The exception is the tone-map +matrix, which caches any non-empty result permanently: a host whose GPU was +broken at start can stay software-only for tone mapping until it is re-probed or +restarted. + +Re-probe does not restart anything and does not reload configuration. It is +**refused on a node that is transcoding**: every probe ends in a real encode on +the GPU, and a card at its concurrent session limit fails that encode with an +error nothing can tell apart from a broken driver — which would flag working +hardware as a regression and take the node's tone-map inventory down with it. +Disable the node or wait for it to drain, then re-probe. On an idle node the call +can take a couple of minutes, because it pays the full cold probe cost on +purpose. + +Two outcomes are worth knowing: + +- On success the row's capability report, verified backends, and drift note are + all updated before the action returns — you are not waiting for the next + 30-second sweep. +- If the node's probe cannot finish, the action reports an error and the node + **keeps its previous report**. An unfinished probe is not evidence that the + hardware changed, so nothing is overwritten. Re-running it after the node + settles is safe. + +Force reload is a different tool: it makes a node re-read its configuration (and +tears down its sessions to do so). Use it after changing a node's acceleration +overrides; use Re-probe after changing the hardware or the driver under it. It +has no button today — it is API-only, `POST +/api/v1/admin/nodes/{id}/force-reload`. + +## Scratch admission + +A transcode writes HLS segments to its node's scratch directory for the entire +life of a session. A node that is nearly full therefore does not fail fast: it +accepts the session, streams for a while, and then dies with a write error after +the client has already committed to it. + +To avoid that, transcode selection **skips a node whose scratch volume is 95% or +more full**, preferring a node with headroom even when the full one is carrying +fewer jobs. Each time a node crosses into that state the server logs it once +(`component=nodepool`, "scratch volume nearly full") — once per transition, not +once per session. + +The rule is deliberately forgiving in two ways: + +- If skipping would leave *no* usable transcode node, the rule is ignored and + selection proceeds normally. Degraded playback beats no playback, and 95% was + never meant to darken a whole cluster. The log says which of the two happened: + a node that was genuinely kept out reads "excluded from selection", while a + node the guard had to admit anyway reads "still selected because no eligible + node has scratch headroom" and is accompanied by "transcode scratch guard + ignored". The second is an outage in progress — sessions are landing on a disk + that will fail mid-stream — and is worth paging on where the first is not. +- A node whose scratch fill cannot be read is never skipped: no sample, an + unmeasurable path, or numbers the node itself flagged as carried over from an + earlier probe. Removing capacity on a reading we do not have would be worse + than the failure it prevents. + +If you see this in your logs, the fix is on the node: raise the volume's size, +lower segment retention, or clear stale artifacts from the transcode directory. + +## Scraping node metrics + +Every Silo process — the API host and each node — publishes its own sample as +`streamapp_node_*` gauges on `/metrics`, on the same listener it serves traffic +from. It is unauthenticated, matching the API listener, because a scrape target +that needs a credential is a scrape target that goes unmonitored; what it exposes +is host resource counters, not media. + +Disk series are labeled by **role**, not by path — `mount="scratch"` and +`mount="library-1"`, `library-2`, ... — so an anonymous scrape cannot enumerate +where your media lives. The real paths are behind admin authentication on +`GET /api/v1/admin/system/resources` and on the Nodes page. Library ordering is +stable for a given configuration, but it is positional: adding a library root can +renumber the series after it, so alert on `mount="scratch"` by name and on the +library mounts by aggregate. + +A minimal scrape config, with one job per role: + +```yaml +scrape_configs: + - job_name: silo-api + metrics_path: /metrics + static_configs: + - targets: ["silo-api.internal:8080"] + labels: + silo_role: api + + - job_name: silo-nodes + metrics_path: /metrics + static_configs: + - targets: + - "transcode-1.internal:8081" + - "transcode-2.internal:8081" + labels: + silo_role: transcode + - targets: + - "proxy-1.internal:8082" + labels: + silo_role: proxy +``` + +Replace the hostnames and ports with your own; the metrics path is the same on +every role. The two alerts worth having first are the scratch volume filling and +a node's GPU going quiet while its CPU does not: + +```yaml +groups: + - name: silo-nodes + rules: + - alert: SiloScratchVolumeFilling + expr: | + streamapp_node_disk_used_bytes{mount="scratch"} + / streamapp_node_disk_total_bytes{mount="scratch"} > 0.9 + for: 15m + annotations: + summary: "Silo scratch volume above 90% on {{ $labels.instance }}" + + - alert: SiloNodeCPUSaturated + expr: streamapp_node_cpu_percent > 90 + for: 15m + annotations: + summary: "Silo node CPU pegged on {{ $labels.instance }} — check the GPU column for a failed probe" +``` + +The GPU gauges (`streamapp_node_gpu_video_busy_percent`, +`streamapp_node_gpu_render_busy_percent`, `streamapp_node_gpu_sessions`, +`streamapp_node_gpu_vram_used_bytes`, `streamapp_node_gpu_vram_total_bytes`) are +labeled by `device`. On Intel and AMD they measure Silo's own FFmpeg processes +only, so a card shared with anything outside Silo reads as less busy than it is; +on NVIDIA they come from `nvidia-smi` and are whole-GPU. A device that nothing +could measure this interval publishes no VRAM series rather than a zero. + +## Source References + +- `internal/nodepool` — health sweep, capability refetch, drift detection, and + transcode placement including the scratch guard. +- `internal/nodemetrics` — host and GPU sampling, and the `streamapp_node_*` + collector. +- `internal/playback/gpudetect.go` — the hardware verification probes behind the + GPU column. +- [Admin API](../../admin-api.md) — the `GET /api/v1/admin/nodes` field table and + the re-probe endpoint. diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 28b5cc90a..7d870035e 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -37,6 +37,8 @@ rewriting. synced library collections sourced from TMDB, Trakt, and MDBList. - [Local NFO Metadata](admin/nfo-local-metadata.md) - Supported NFO sidecar fields, how they merge with online providers, and the naming-supplies-structure contract. +- [Monitoring Stream Nodes](admin/monitoring-nodes.md) - Reading the Nodes page columns, re-probing + a node's GPU after a driver change, scratch-disk admission, and scraping node metrics. ## Deployment diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index d76cc66f5..1b5f5b764 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -7,14 +7,18 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "net/http" "strconv" + "strings" "sync" "time" "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/go-chi/chi/v5" "github.com/redis/go-redis/v9" ) @@ -34,6 +38,17 @@ type NodeListEnabled interface { ListEnabled(ctx context.Context, nodeType string) ([]*nodepool.Node, error) } +// NodeCapabilityRefresher fetches, persists, and publishes one node's +// capability report immediately rather than waiting for the background sweep. +// +// It is an interface only so the re-probe handler can be tested without a live +// sweep; *nodepool.HealthChecker is the single implementation, and the handler +// deliberately does not reimplement any of the fetch, drift, or persist logic +// that lives behind it. +type NodeCapabilityRefresher interface { + RefreshNodeCapabilities(ctx context.Context, n *nodepool.Node) error +} + // NodeHandler handles CRUD operations and health checks for stream nodes. type NodeHandler struct { repo NodeRepository @@ -43,6 +58,20 @@ type NodeHandler struct { eventBus cache.EventBus redisClient *redis.Client // for reading session keys jwtSecret string // for bearer auth when calling force-reload on nodes + // capabilities refreshes a node's stored inventory on demand; nil in a + // deployment with no health checker, where a re-probe still runs on the node + // and the stored row catches up on the next sweep. + capabilities NodeCapabilityRefresher +} + +// SetCapabilityRefresher wires the on-demand capability refresh used after a +// re-probe. It is set after construction because the health checker is built +// before the router and owned by the process, not by this handler. +func (h *NodeHandler) SetCapabilityRefresher(refresher NodeCapabilityRefresher) { + if h == nil { + return + } + h.capabilities = refresher } // NewNodeHandler creates a new NodeHandler. @@ -315,6 +344,216 @@ func (h *NodeHandler) HandleForceReloadNode(w http.ResponseWriter, r *http.Reque }}}) } +// ReprobeNodeResult is the JSON response for an operator-triggered capability +// re-probe of one node. +type ReprobeNodeResult struct { + NodeID int `json:"node_id"` + NodeName string `json:"node_name"` + // Status is "ok" when the node re-probed successfully, "error" otherwise. + // A node that refused or could not be reached is reported here rather than + // as an HTTP error status, matching the per-node check and force-reload + // routes: the request to the API succeeded, the node is what failed. + Status string `json:"status"` + Error string `json:"error,omitempty"` + // Resolved is the backend the node picked after re-probing. + Resolved string `json:"resolved,omitempty"` + // CapabilityHash identifies the snapshot the node published. Comparing it + // against the hash in the node list before the call is what tells an + // operator whether the re-probe changed anything. + CapabilityHash string `json:"capability_hash,omitempty"` + // CapabilitiesRefreshed reports whether this server also refetched and + // stored the node's new inventory before answering. False means the stored + // row will catch up on a later health sweep instead. + CapabilitiesRefreshed bool `json:"capabilities_refreshed"` +} + +// nodeReprobeFallbackTimeout bounds the re-probe request to a node whose stored +// capability report does not advertise a probe budget — one that has never been +// inventoried, or that runs a build predating the advertisement. +// +// A re-probe deliberately discards the node's probe caches, so it pays the full +// cold cost: a hardware walk plus the whole tone-map matrix, each bounded by +// several ffmpeg execs. This is the same order of magnitude as the health +// sweep's own two-minute capability fetch bound, plus request slack, and is only +// a fallback: a node that has been inventoried once advertises its real budget. +const nodeReprobeFallbackTimeout = 150 * time.Second + +// nodeReprobeTimeout is how long the API waits for one node's re-probe. +// +// The node itself publishes the budget its probe matrix needs in every +// capability report (probe_request_timeout_ms), which is exactly what a cold +// capability fetch is given; a node with different hardware or a slower ffmpeg +// therefore gets its own number rather than a cluster-wide guess. The stored +// report is parsed minimally here for the same reason nodepool parses its own +// narrow views: this layer has no business decoding the whole inventory. +func nodeReprobeTimeout(n *nodepool.Node) time.Duration { + var advertised struct { + ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms"` + } + if n != nil && len(n.Capabilities) > 0 { + // A report that cannot be parsed leaves the zero value, which is the + // fallback; an unreadable report is not a reason to fail the action. + _ = json.Unmarshal(n.Capabilities, &advertised) + } + return playback.NormalizeProbeRequestTimeout(advertised.ProbeRequestTimeoutMillis, nodeReprobeFallbackTimeout) +} + +// nodeReprobeWriteSlack covers the repository round trips and the JSON write +// that bracket the two long calls this handler makes. +const nodeReprobeWriteSlack = 15 * time.Second + +// extendReprobeWriteDeadline lifts this connection's write deadline to cover the +// whole action: the node's own probe budget, then the capability refetch that +// stores the result. +// +// The API listener's WriteTimeout is 120 seconds, and this route can legitimately +// outlive it — a node with two render devices advertises a probe budget past two +// minutes on its own, before the refetch adds its bound. Without this the +// deadline expires after the node has already re-probed and this server has +// already persisted the report: the operator's browser sees a torn connection, +// the UI toasts a failure for an action that succeeded, and the obvious response +// is to run the whole cold FFmpeg matrix again. The diagnostics upload route +// extends its deadlines for exactly the same reason. +func extendReprobeWriteDeadline(w http.ResponseWriter, r *http.Request, probeBudget time.Duration) { + budget := probeBudget + nodepool.CapabilityRefreshTimeout + nodeReprobeWriteSlack + if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(budget)); err != nil { + // A ResponseWriter that cannot carry a deadline (a test recorder, a + // wrapper that does not unwrap) is not a reason to refuse the action. + slog.WarnContext(r.Context(), "node re-probe write deadline not extended", "component", "api", + "budget", budget, "error", err) + } +} + +// HandleReprobeNode handles POST /admin/nodes/{id}/reprobe — asks one node to +// discard its cached hardware-probe verdicts and re-verify against live +// hardware, then stores the resulting inventory immediately. +// +// This is the operator's answer to hardware that stopped working underneath a +// running node. A node caches a successful probe for its whole process lifetime +// (re-verifying per request would put ffmpeg execs on the playback path), so a +// GPU that has since been removed, or whose driver was replaced, keeps reading +// "verified" until the node restarts. Nothing about that is visible in a health +// check, because the node is perfectly healthy either way. The opposite +// direction self-heals: a failed GPU probe is retried after a short negative +// TTL, so a repaired driver reaches the list on the next capability snapshot +// without this action. +// +// A node that is transcoding refuses: the probe smoke-encodes on the GPU, and a +// busy encoder would report working hardware as failed. +// +// Both halves matter: the node recomputes, and this server then refetches and +// persists the report so the admin list, the pools, and the planner's GPU +// identities agree with it without waiting up to a sweep interval. The refresh +// reuses the health checker's own machinery, so drift detection and persistence +// have exactly one implementation. +func (h *NodeHandler) HandleReprobeNode(w http.ResponseWriter, r *http.Request) { + id, err := parseIDParam(r) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid node ID") + return + } + + node, err := h.repo.GetByID(r.Context(), id) + if err != nil { + if errors.Is(err, nodepool.ErrNodeNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Node not found") + return + } + slog.ErrorContext(r.Context(), "fetching node for re-probe", "component", "api", "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch node") + return + } + + extendReprobeWriteDeadline(w, r, nodeReprobeTimeout(node)) + + result := ReprobeNodeResult{NodeID: node.ID, NodeName: node.Name, Status: "ok"} + reprobed, err := h.reprobeNode(r.Context(), node) + if err != nil { + slog.WarnContext(r.Context(), "node capability re-probe failed", "component", "api", + "node_id", node.ID, "name", node.Name, "error", err) + result.Status = "error" + result.Error = err.Error() + writeJSON(w, http.StatusOK, result) + return + } + result.Resolved = reprobed.Resolved + result.CapabilityHash = reprobed.CapabilityHash + + // The node has already recomputed at this point, so a refresh failure is + // reported alongside a successful re-probe rather than turning it into one: + // the next sweep will store the report, and saying the re-probe failed + // would invite an operator to run it again for nothing. + if h.capabilities == nil { + writeJSON(w, http.StatusOK, result) + return + } + if err := h.capabilities.RefreshNodeCapabilities(r.Context(), node); err != nil { + slog.WarnContext(r.Context(), "storing re-probed node capabilities failed", "component", "api", + "node_id", node.ID, "name", node.Name, "error", err) + } else { + result.CapabilitiesRefreshed = true + } + writeJSON(w, http.StatusOK, result) +} + +// nodeReprobeResponse is the node's own answer to /admin/reprobe-capabilities. +type nodeReprobeResponse struct { + Resolved string `json:"resolved"` + CapabilityHash string `json:"capability_hash"` +} + +// reprobeNode performs the bearer-authenticated re-probe call against one node, +// mirroring the force-reload client. +func (h *NodeHandler) reprobeNode(ctx context.Context, node *nodepool.Node) (nodeReprobeResponse, error) { + timeout := nodeReprobeTimeout(node) + client := &http.Client{Timeout: timeout} + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/reprobe-capabilities", nil) + if err != nil { + return nodeReprobeResponse{}, err + } + req.Header.Set("Authorization", "Bearer "+h.jwtSecret) + + resp, err := client.Do(req) + if err != nil { + return nodeReprobeResponse{}, logredact.SanitizeURLError(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + // Read the small error body rather than discarding it: it carries the + // node's own explanation, and reading it also lets the transport reuse + // the connection. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + switch resp.StatusCode { + case http.StatusServiceUnavailable: + // The node's own degraded-snapshot answer: it kept its previous + // hash rather than publishing a partial probe. + return nodeReprobeResponse{}, errors.New("node could not complete its hardware probe; its previous capability report was kept") + case http.StatusConflict: + // The node refused because it is transcoding: a probe's smoke encode + // competes with live sessions for encoder slots, and losing that + // race would be published as failed hardware. + if message := strings.TrimSpace(string(body)); message != "" { + return nodeReprobeResponse{}, errors.New(message) + } + return nodeReprobeResponse{}, errors.New("node is busy transcoding; retry the re-probe when it is idle") + } + return nodeReprobeResponse{}, fmt.Errorf("node re-probe returned status %d", resp.StatusCode) + } + var decoded nodeReprobeResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, maxNodeReprobeResponseBytes)).Decode(&decoded); err != nil { + return nodeReprobeResponse{}, err + } + return decoded, nil +} + +// maxNodeReprobeResponseBytes bounds the node's answer. It carries two short +// strings; the bound is only there because a node is a worker that may run on +// remote hardware. +const maxNodeReprobeResponseBytes = 8 << 10 + // HandleListSessions handles GET /admin/nodes/sessions — lists active playback // sessions from Redis, optionally filtered by node_id query parameter. func (h *NodeHandler) HandleListSessions(w http.ResponseWriter, r *http.Request) { diff --git a/internal/api/handlers/nodes_reprobe_test.go b/internal/api/handlers/nodes_reprobe_test.go new file mode 100644 index 000000000..046af26ba --- /dev/null +++ b/internal/api/handlers/nodes_reprobe_test.go @@ -0,0 +1,262 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/go-chi/chi/v5" +) + +// stubCapabilityRefresher stands in for the health checker's on-demand refresh. +type stubCapabilityRefresher struct { + calls atomic.Int32 + err error +} + +func (s *stubCapabilityRefresher) RefreshNodeCapabilities(context.Context, *nodepool.Node) error { + s.calls.Add(1) + return s.err +} + +func reprobeRequest(t *testing.T) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/admin/nodes/1/reprobe", nil) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("id", "1") + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) +} + +// fakeNodeServer answers the node-side re-probe route with the given status and +// body, recording the bearer token it was given. +func fakeNodeServer(t *testing.T, status int, body string) (url string, authorization *string) { + t.Helper() + seen := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/admin/reprobe-capabilities" || r.Method != http.MethodPost { + w.WriteHeader(http.StatusNotFound) + return + } + seen = r.Header.Get("Authorization") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server.URL, &seen +} + +// The point of the API-side action is that the operator does not have to wait a +// sweep interval: the node re-probes, and this server immediately refetches and +// stores the report so the list, the pools, and the planner agree with it. +func TestHandleReprobeNodeRefreshesStoredCapabilities(t *testing.T) { + url, authorization := fakeNodeServer(t, http.StatusOK, + `{"resolved":"qsv","capability_hash":"sha256:new"}`) + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.Error != "" { + t.Fatalf("result = %+v, want a clean ok", result) + } + if result.Resolved != "qsv" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result did not carry the node's answer: %+v", result) + } + if !result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = false, want the stored row refreshed on success") + } + if got := refresher.calls.Load(); got != 1 { + t.Fatalf("refresh calls = %d, want exactly one", got) + } + if *authorization != "Bearer secret" { + t.Fatalf("node saw authorization %q, want the bearer secret", *authorization) + } +} + +// A node whose probe could not complete answers 503 and keeps its previous +// hash. That has to reach the operator as a named failure, and must not trigger +// a capability refetch — refetching would store the report the node explicitly +// declined to republish. +func TestHandleReprobeNodeSurfacesNodeFailure(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusServiceUnavailable, "capability probe unavailable\n") + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + // The API request itself succeeded; the node is what failed, exactly as the + // per-node check and force-reload routes report it. + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "error" || result.Error == "" { + t.Fatalf("result = %+v, want a reported node error", result) + } + if !strings.Contains(result.Error, "hardware probe") { + t.Fatalf("error = %q, want it to explain the degraded probe", result.Error) + } + if result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = true after a failed re-probe") + } + if got := refresher.calls.Load(); got != 0 { + t.Fatalf("refresh calls = %d, want none after a failed re-probe", got) + } +} + +// A refresh failure is reported beside a successful re-probe rather than turning +// it into one: the node has already recomputed, and the next sweep will store +// the report. +func TestHandleReprobeNodeReportsRefreshFailureSeparately(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusOK, `{"resolved":"none","capability_hash":"sha256:new"}`) + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{err: nodepool.ErrCapabilityRefreshInFlight} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result = %+v, want the re-probe itself reported as ok", result) + } + if result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = true despite a failed refresh") + } +} + +// A node that is transcoding refuses, because a probe's smoke encode competing +// with live sessions would be published as failed hardware. The operator has to +// read the node's own explanation, not a bare status code, or the only sensible +// next step — drain the node and retry — is invisible. +func TestHandleReprobeNodeSurfacesBusyNodeRefusal(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusConflict, + "node is running 2 transcode job(s); a re-probe smoke-encodes on the GPU. Retry when the node is idle.\n") + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "error" { + t.Fatalf("result = %+v, want the refusal reported as an error", result) + } + if !strings.Contains(result.Error, "idle") { + t.Fatalf("error = %q, want the node's own explanation", result.Error) + } + if got := refresher.calls.Load(); got != 0 { + t.Fatalf("refresh calls = %d, want none after a refused re-probe", got) + } +} + +// The action outlives the API listener's 120s WriteTimeout by design: the node's +// advertised probe budget reaches five minutes and the capability refetch adds +// two more. Without a deadline extension the response is written after the +// connection's deadline has passed, so the operator sees a torn connection and a +// "Re-probe failed" toast for an action that succeeded — and re-runs the whole +// cold FFmpeg matrix. This drives a real listener with a deadline far shorter +// than the node takes, which is the only way to observe the write actually +// landing. +func TestHandleReprobeNodeOutlivesTheListenerWriteTimeout(t *testing.T) { + const nodeDelay = 400 * time.Millisecond + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(nodeDelay) + _, _ = w.Write([]byte(`{"resolved":"qsv","capability_hash":"sha256:new"}`)) + })) + defer node.Close() + + // The node advertises a budget, so the handler knows how long to hold the + // connection open — exactly as a node that has been inventoried once does. + report, err := json.Marshal(playback.HWAccelInfo{ProbeRequestTimeoutMillis: 30_000}) + if err != nil { + t.Fatal(err) + } + repo := &stubNodeRepository{node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, + Capabilities: report, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(&stubCapabilityRefresher{}) + + router := chi.NewRouter() + router.Post("/admin/nodes/{id}/reprobe", handler.HandleReprobeNode) + api := httptest.NewUnstartedServer(router) + api.Config.WriteTimeout = nodeDelay / 4 + api.Start() + defer api.Close() + + resp, err := api.Client().Post(api.URL+"/admin/nodes/1/reprobe", "", nil) + if err != nil { + t.Fatalf("re-probe response lost to the listener write deadline: %v", err) + } + defer func() { _ = resp.Body.Close() }() + var result ReprobeNodeResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result = %+v, want the node's answer delivered intact", result) + } +} + +func TestHandleReprobeNodeUnknownNode(t *testing.T) { + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 for an unknown node", recorder.Code) + } +} + +// The node publishes the budget its own probe matrix needs, so a node with +// slower hardware is not abandoned on a cluster-wide guess. A node that has +// never been inventoried falls back to the generous constant. +func TestNodeReprobeTimeoutPrefersTheNodeAdvertisedBudget(t *testing.T) { + advertised := playback.HWAccelInfo{ProbeRequestTimeoutMillis: 111_000} + payload, err := json.Marshal(advertised) + if err != nil { + t.Fatal(err) + } + if got := nodeReprobeTimeout(&nodepool.Node{Capabilities: payload}); got != 111*time.Second { + t.Fatalf("timeout = %s, want the node-advertised 111s", got) + } + if got := nodeReprobeTimeout(&nodepool.Node{}); got != nodeReprobeFallbackTimeout { + t.Fatalf("timeout = %s, want the fallback %s for a node with no report", got, nodeReprobeFallbackTimeout) + } + if got := nodeReprobeTimeout(&nodepool.Node{Capabilities: json.RawMessage(`not json`)}); got != nodeReprobeFallbackTimeout { + t.Fatalf("timeout = %s, want the fallback for an unreadable report", got) + } +} diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 96885ebde..9ec61b9c2 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -19,12 +19,17 @@ type stubNodeRepository struct { // default "unknown node" answer. updateResult *nodepool.Node updated *nodepool.UpdateNodeInput + // node is what GetByID returns; nil keeps the default "unknown node" answer. + node *nodepool.Node } func (s *stubNodeRepository) List(context.Context) ([]*nodepool.Node, error) { return s.nodes, nil } func (s *stubNodeRepository) GetByID(context.Context, int) (*nodepool.Node, error) { - return nil, nodepool.ErrNodeNotFound + if s.node == nil { + return nil, nodepool.ErrNodeNotFound + } + return s.node, nil } func (s *stubNodeRepository) Create(context.Context, nodepool.CreateNodeInput) (*nodepool.Node, error) { diff --git a/internal/api/router.go b/internal/api/router.go index a59439ca3..02cf527f0 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3161,6 +3161,13 @@ func NewRouter(deps Dependencies) chi.Router { jwtSecret = deps.Config.Auth.JWTSecret } nodeHandler := handlers.NewNodeHandler(deps.NodeRepo, deps.ProxyPool, deps.TranscodePool, deps.NodeRepo, deps.EventBus, deps.RedisClient, jwtSecret) + // A re-probe stores the node's new inventory through the + // sweep's own refresh, so the drift and persist rules have + // one implementation. Without a health checker the node + // still re-probes and the row catches up on a later sweep. + if deps.NodeHealthChecker != nil { + nodeHandler.SetCapabilityRefresher(deps.NodeHealthChecker) + } r.Route("/nodes", func(r chi.Router) { r.Get("/", nodeHandler.HandleListNodes) r.Post("/", nodeHandler.HandleCreateNode) @@ -3169,6 +3176,7 @@ func NewRouter(deps Dependencies) chi.Router { r.Post("/{id}/check", nodeHandler.HandleCheckNode) r.Post("/force-reload", nodeHandler.HandleForceReloadNodes) r.Post("/{id}/force-reload", nodeHandler.HandleForceReloadNode) + r.Post("/{id}/reprobe", nodeHandler.HandleReprobeNode) }) // Live node sessions (reads from Redis) // Note: /admin/sessions is already used for playback sessions from PostgreSQL. diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index bf58e91b9..1a56e3b51 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -112,7 +112,7 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { // zero would read as an empty disk in every alert rule. continue } - label := diskSeriesLabel(c.sampler.scratchPath(), disk, &libraries) + label := diskSeriesLabel(disk, &libraries) gauge(descDiskUsed, disk.UsedGB*bytesPerGB, label) gauge(descDiskTotal, disk.TotalGB*bytesPerGB, label) } @@ -143,8 +143,8 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { // paths themselves stay behind auth. // // libraries counts the non-scratch mounts already labeled in this scrape. -func diskSeriesLabel(scratchDir string, disk DiskStats, libraries *int) string { - if scratchDir != "" && disk.Path == scratchDir { +func diskSeriesLabel(disk DiskStats, libraries *int) string { + if disk.Scratch { return "scratch" } *libraries++ diff --git a/internal/nodemetrics/disk.go b/internal/nodemetrics/disk.go index 2a3b0f6dd..f2338a65d 100644 --- a/internal/nodemetrics/disk.go +++ b/internal/nodemetrics/disk.go @@ -166,11 +166,12 @@ func (s *Sampler) diskStats(paths []string, now time.Time) []DiskStats { if entry == nil { continue } + scratch := s.scratchDir != "" && path == s.scratchDir if !entry.haveGood { // Report it rather than hiding it: a media root this node cannot // see is a deployment fact an operator needs, and silently dropping // it looks identical to the path not being configured. - out = append(out, DiskStats{Path: path, Unavailable: true}) + out = append(out, DiskStats{Path: path, Unavailable: true, Scratch: scratch}) } else { if entry.good.FSID != "" { if seenFS[entry.good.FSID] { @@ -183,6 +184,7 @@ func (s *Sampler) diskStats(paths []string, now time.Time) []DiskStats { UsedGB: bytesToGB(entry.good.UsedBytes), TotalGB: bytesToGB(entry.good.TotalBytes), Stale: entry.stale(now, s.interval), + Scratch: scratch, }) } // The cap applies to every entry, measured or not. A host whose library diff --git a/internal/nodemetrics/disk_test.go b/internal/nodemetrics/disk_test.go index 92e557d40..caa9ca4f4 100644 --- a/internal/nodemetrics/disk_test.go +++ b/internal/nodemetrics/disk_test.go @@ -147,6 +147,64 @@ func TestDiskStatsDeduplicatesByFilesystem(t *testing.T) { } } +// The API stores a node's sample opaquely and does not know its transcode +// directory, so the scratch entry has to identify itself: the admission guard +// and the Prometheus label both find it by this flag, not by matching a path. +func TestDiskStatsFlagsTheScratchMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media", "/missing") + f.answer("/transcode", fsStats{UsedBytes: 96 << 30, TotalBytes: 100 << 30, FSID: "a:1"}) + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 3) + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 3 { + t.Fatalf("disks = %+v, want three entries", disks) + } + if !disks[0].Scratch || disks[0].Path != "/transcode" { + t.Fatalf("disks[0] = %+v, want the scratch dir flagged and first", disks[0]) + } + for _, disk := range disks[1:] { + if disk.Scratch { + t.Fatalf("non-scratch mount flagged as scratch: %+v", disk) + } + } +} + +// A scratch dir that does not exist yet is still the scratch entry: an +// unavailable reading has to be distinguishable from a media root's, and the +// admission guard depends on telling "cannot measure" from "not the scratch". +func TestDiskStatsFlagsAnUnavailableScratchMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media") + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 2) + disks := f.sampleAndSettle(t, 2) + + if len(disks) == 0 || disks[0].Path != "/transcode" { + t.Fatalf("disks = %+v, want the scratch dir first", disks) + } + if !disks[0].Unavailable || !disks[0].Scratch { + t.Fatalf("disks[0] = %+v, want an unavailable scratch entry", disks[0]) + } +} + +// A sampler with no scratch dir — a proxy node — flags nothing, so a reader +// never mistakes a media root for the transcode volume. +func TestDiskStatsFlagsNoScratchWithoutAScratchDir(t *testing.T) { + f := newDiskFixture(t, "", "/media") + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 1) + disks := f.sampleAndSettle(t, 1) + + for _, disk := range disks { + if disk.Scratch { + t.Fatalf("mount flagged as scratch with no scratch dir configured: %+v", disk) + } + } +} + // The contract that matters most: a mount whose server died reports its last // good numbers marked stale, and never delays a sample. func TestDiskStatsReportsHungMountAsStaleWithoutBlocking(t *testing.T) { diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index ba7536b1c..bec76679e 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -439,15 +439,6 @@ func deviceSessions(sessions map[string]int, aliases []string, claimed map[strin return total } -// scratchPath returns the transcode scratch dir this sampler was told about, -// used to label its series without publishing the path itself. -func (s *Sampler) scratchPath() string { - if s == nil { - return "" - } - return s.scratchDir -} - func (s *Sampler) identityList() []DeviceIdentity { if s.identities == nil { return nil diff --git a/internal/nodemetrics/snapshot.go b/internal/nodemetrics/snapshot.go index bb02d6d0b..3036a4f7d 100644 --- a/internal/nodemetrics/snapshot.go +++ b/internal/nodemetrics/snapshot.go @@ -80,6 +80,14 @@ type DiskStats struct { // does not exist on this node, or the very first probe is still hanging). // UsedGB/TotalGB are meaningless when it is set. Unavailable bool `json:"unavailable,omitempty"` + // Scratch marks the transcode working directory's entry. It is the one mount + // whose filling up breaks transcoding rather than merely browsing, so a + // reader has to be able to find it without knowing the deployment's paths — + // which is exactly the position the API is in, since it stores a node's + // sample opaquely. Set on at most one entry per sample; a media root sharing + // the scratch volume is deduplicated onto the scratch entry, which is + // reported first. + Scratch bool `json:"scratch,omitempty"` } // vendorNVIDIA is the GPUStats.Vendor value nvidia-smi enrichment reports. diff --git a/internal/nodepool/gpuidentity_test.go b/internal/nodepool/gpuidentity_test.go index 8ebbdcd06..95778c1b4 100644 --- a/internal/nodepool/gpuidentity_test.go +++ b/internal/nodepool/gpuidentity_test.go @@ -113,7 +113,7 @@ func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { transcodes := NewTranscodePool() transcodes.SetNodes([]*Node{{ID: 1, URL: "http://tc-1", Enabled: true, Healthy: true}}) - transcodes.ApplyCapabilities(1, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt) + transcodes.ApplyCapabilities(1, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil) if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { t.Fatalf("transcode ApplyCapabilities derived %v, want [GPU-aaa]", got) } @@ -121,14 +121,14 @@ func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { // The card was passed through to another host: the node now reports a // device it cannot identify, and must stop claiming the old key. transcodes.ApplyCapabilities(1, []byte(`{"boot_id":"boot-2","render_device_details":[{"path":"/dev/dri/renderD128"}]}`), - "sha256:bbb", refreshedAt) + "sha256:bbb", refreshedAt, nil) if got := transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { t.Fatalf("stale identities survived a new report: %v", got) } proxies := NewProxyPool() proxies.SetNodes([]*Node{{ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true}}) - proxies.ApplyCapabilities(2, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt) + proxies.ApplyCapabilities(2, []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil) if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { t.Fatalf("proxy ApplyCapabilities derived %v, want [GPU-aaa]", got) } diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index e4d8807e3..d36c1c8b2 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -4,12 +4,15 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "log/slog" "net/http" "slices" + "strings" "sync" "time" + "unicode/utf8" ) // healthResponse is the JSON response from a node's /health endpoint. @@ -145,6 +148,12 @@ type CapabilityFetcher func(ctx context.Context, nodeURL string) (payload []byte // abandoning it every sweep. const capabilityFetchTimeout = 2 * time.Minute +// CapabilityRefreshTimeout is the bound RefreshNodeCapabilities puts on the +// fetch it performs. It is exported for the one caller that has to hold an HTTP +// connection open across that fetch and must therefore size its own write +// deadline to include it. +const CapabilityRefreshTimeout = capabilityFetchTimeout + // HealthChecker runs periodic health checks on all nodes in both pools, // updating in-memory state and optionally persisting to the database. type HealthChecker struct { @@ -231,7 +240,7 @@ func (hc *HealthChecker) Start(ctx context.Context) { type applyHealthFunc func(id int, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) // applyCapabilitiesFunc is a pool's copy-on-write capability writer. -type applyCapabilitiesFunc func(id int, capabilities []byte, hash string, refreshedAt time.Time) +type applyCapabilitiesFunc func(id int, capabilities []byte, hash string, refreshedAt time.Time, drift *string) func (hc *HealthChecker) checkAll(ctx context.Context) { var wg sync.WaitGroup @@ -291,7 +300,8 @@ func (hc *HealthChecker) startCapabilityRefresh(ctx context.Context, n *Node, ap go func() { defer hc.capabilityRefreshes.Done() defer hc.capabilityRefreshInFlight.Delete(n.ID) - hc.refreshCapabilities(ctx, n, applyCapabilities) + // Errors are already logged inside; the sweep has no caller to report to. + _ = hc.refreshCapabilities(ctx, n, applyCapabilities) }() } @@ -301,13 +311,66 @@ func (hc *HealthChecker) waitForCapabilityRefreshes() { hc.capabilityRefreshes.Wait() } +// RefreshNodeCapabilities fetches, stores, and publishes one node's capability +// report immediately, on the caller's goroutine, using exactly the machinery the +// background sweep uses. +// +// It exists for the operator-triggered re-probe: the node has just recomputed +// its inventory, and waiting up to a sweep interval for the API to notice would +// make the action look like it did nothing. Every rule the sweep applies still +// applies here — drift is computed and persisted the same way, a report without +// a hash is refused, a failed fetch leaves the stored row alone — so there is no +// second, divergent persist path. +// +// It participates in the sweep's per-node in-flight guard, so a manual refresh +// and a sweep refresh of the same node cannot run at once; the loser reports +// ErrCapabilityRefreshInFlight rather than starting a duplicate fetch. +func (hc *HealthChecker) RefreshNodeCapabilities(ctx context.Context, n *Node) error { + if hc == nil || n == nil { + return errors.New("no node health checker configured") + } + if fetch, _ := hc.hooks(); fetch == nil { + return errors.New("no node capability fetcher configured") + } + if _, loaded := hc.capabilityRefreshInFlight.LoadOrStore(n.ID, struct{}{}); loaded { + return ErrCapabilityRefreshInFlight + } + defer hc.capabilityRefreshInFlight.Delete(n.ID) + return hc.refreshCapabilities(ctx, n, hc.applyCapabilitiesFor(n)) +} + +// ErrCapabilityRefreshInFlight reports that a capability refresh for the node +// was already running, so the caller's own refresh was not started. The report +// in flight is at least as fresh as the one the caller wanted. +var ErrCapabilityRefreshInFlight = errors.New("node capability refresh already in flight") + +// applyCapabilitiesFor returns the pool writer for a node's type, or nil when +// this checker has no pool for it — which is the normal state for a node row +// that is disabled and therefore in no pool. +func (hc *HealthChecker) applyCapabilitiesFor(n *Node) applyCapabilitiesFunc { + switch n.Type { + case NodeTypeProxy: + if hc.proxyPool != nil { + return hc.proxyPool.ApplyCapabilities + } + case NodeTypeTranscode: + if hc.transcodePool != nil { + return hc.transcodePool.ApplyCapabilities + } + } + return nil +} + // refreshCapabilities fetches and stores one node's capability report. A // failure leaves the stored row alone and is retried on the next sweep, because // a fetch that failed is no evidence about what the node has. -func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) { +// +// The returned error is for a caller that asked for this refresh explicitly; the +// sweep ignores it, since every failure is already logged here. +func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) error { fetch, onChanged := hc.hooks() if fetch == nil { - return + return nil } fetchCtx, cancel := context.WithTimeout(ctx, capabilityFetchTimeout) defer cancel() @@ -315,7 +378,7 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply if err != nil { slog.WarnContext(ctx, "node capability fetch failed", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL, "error", err) - return + return err } if hash == "" || len(payload) == 0 { // A hash is what makes the payload trackable; storing one without it @@ -323,24 +386,30 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply // node is the only thing that knows what it hashed. slog.WarnContext(ctx, "node capability report carries no hash", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) - return + return errors.New("node capability report carries no hash") } + // Computed before the write, because the comparison is against the report + // this one replaces, and stored with it so a reader never sees a note + // describing a different payload. + drift, parsed := computeCapabilityDrift(n.Capabilities, payload) + note := resolveDriftNote(n.CapabilityDrift, drift, parsed, payload) refreshedAt := time.Now() if hc.repo != nil { - if err := hc.repo.UpdateCapabilities(ctx, n.ID, payload, hash, refreshedAt); err != nil { + if err := hc.repo.UpdateCapabilities(ctx, n.ID, payload, hash, refreshedAt, note); err != nil { slog.WarnContext(ctx, "failed to persist node capabilities", "component", "nodepool", "id", n.ID, "name", n.Name, "error", err) - return + return err } } - logCapabilityChange(ctx, n, payload) + logCapabilityChange(ctx, n, drift, parsed) if applyCapabilities != nil { - applyCapabilities(n.ID, payload, hash, refreshedAt) + applyCapabilities(n.ID, payload, hash, refreshedAt, note) } if onChanged != nil { onChanged(n.URL) } + return nil } func storedCapabilitiesHash(n *Node) string { @@ -359,46 +428,184 @@ type capabilityDriftView struct { DetectedBackends []struct { Backend string `json:"backend"` Verified bool `json:"verified"` + // Skipped reports that no probe was attempted because the node cannot + // open any of the backend's candidate devices. That is a statement + // about access, not about hardware, so it never counts as a failure. + Skipped bool `json:"skipped"` } `json:"detected_backends"` RenderDevices []string `json:"render_devices"` } -// logCapabilityChange records what changed between the stored report and the -// new one. Losing a verified backend or a render device is the case worth -// waking an operator for: it means a node that was picked for hardware work -// silently became less capable, which otherwise only shows up as slow or -// failing transcodes. -func logCapabilityChange(ctx context.Context, n *Node, payload []byte) { - if len(n.Capabilities) == 0 { - slog.InfoContext(ctx, "node capabilities stored", "component", "nodepool", - "id", n.ID, "name", n.Name, "url", n.URL) - return +// capabilityDrift is what a refetch lost relative to the report it replaces. +// The zero value means nothing was lost, which is also what a node's very first +// report produces. +type capabilityDrift struct { + // first reports that there was no stored report to compare against. + first bool + // lostBackends are backends that used to pass their probe and no longer do. + lostBackends []string + // lostDevices are render devices present in the previous report and absent + // from this one. + lostDevices []string + // previousResolved and resolved are the backend each report resolved to; + // carried for the log line, which is where an operator reads the effect. + previousResolved string + resolved string +} + +// regressed reports whether this refetch lost something worth telling an +// operator about. +func (d capabilityDrift) regressed() bool { + return len(d.lostBackends) > 0 || len(d.lostDevices) > 0 +} + +// maxCapabilityDriftNoteBytes bounds the stored note. The inputs are a node's +// own device and backend lists, so an honest note is a line long; the bound is +// only there because the lists come from a worker that may run elsewhere, and a +// text column echoed to every admin listing nodes is not the place to trust +// them. +const maxCapabilityDriftNoteBytes = 512 + +// persistedNote renders this refetch's regression for the +// stream_nodes.capability_drift column, or nil when this refetch lost nothing. +// nil is not by itself a reason to clear a note the node already carries — see +// resolveDriftNote, which owns that decision. +func (d capabilityDrift) persistedNote() *string { + if !d.regressed() { + return nil + } + parts := make([]string, 0, 3) + if len(d.lostBackends) > 0 { + parts = append(parts, "verified hardware backends lost: "+strings.Join(d.lostBackends, ", ")) + } + if len(d.lostDevices) > 0 { + parts = append(parts, "render devices gone: "+strings.Join(d.lostDevices, ", ")) + } + if d.previousResolved != d.resolved { + parts = append(parts, "resolved backend "+d.previousResolved+" -> "+d.resolved) + } + note := truncateDriftNote(strings.Join(parts, "; ")) + return ¬e +} + +// truncateDriftNote bounds a note to maxCapabilityDriftNoteBytes without ever +// cutting a rune in half. +// +// The column is Postgres text, which rejects invalid UTF-8 outright, and a +// rejected UPDATE takes capabilities and capabilities_hash down with it — the +// stored hash then never advances, so every later sweep refetches and fails +// again. Slicing bytes is exactly how the untrusted device and backend names +// this bound exists to contain would produce that. +func truncateDriftNote(note string) string { + if len(note) <= maxCapabilityDriftNoteBytes { + return note + } + cut := note[:maxCapabilityDriftNoteBytes] + // A trailing partial sequence decodes as RuneError with size 1; a real + // U+FFFD in the input decodes with size 3 and is left alone. + for len(cut) > 0 { + r, size := utf8.DecodeLastRuneInString(cut) + if r != utf8.RuneError || size > 1 { + break + } + cut = cut[:len(cut)-1] + } + return cut + "..." +} + +// resolveDriftNote decides what stream_nodes.capability_drift should say after +// this refetch, given the note it already carries. +// +// Setting the note is a delta — a regression against the report being replaced. +// Clearing it must not be, because a delta against an already-degraded report +// finds nothing newly lost and would erase a standing regression on the next +// unrelated hash change: a reboot moves boot_id, a reworded FFmpeg failure moves +// the probe reason, and the operator-triggered re-probe refetches +// unconditionally. That would tell an operator the node recovered while its +// backend is still failing its probe, which is the one reading this column must +// never produce. So the note is latched, and only a report whose probes all pass +// clears it. +func resolveDriftNote(stored *string, drift capabilityDrift, parsed bool, payload []byte) *string { + if note := drift.persistedNote(); note != nil { + return note + } + if stored == nil || strings.TrimSpace(*stored) == "" { + return nil + } + if !parsed || !hardwareProbesClean(payload) { + // Nothing new was lost, but this report is not evidence of recovery. + return stored + } + return nil +} + +// hardwareProbesClean reports whether every backend the node actually probed +// passed. A skipped backend is not a failure — it means the node cannot open the +// devices, which is the normal reading for a proxy pointed at a cluster-wide +// hw_device — and a report that cannot be parsed is not evidence of anything. +func hardwareProbesClean(payload []byte) bool { + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false + } + for _, backend := range current.DetectedBackends { + if !backend.Verified && !backend.Skipped { + return false + } + } + return true +} + +// computeCapabilityDrift compares the report a node just served against the one +// it replaces. parsed is false when either payload cannot be read, in which case +// the drift is unknown rather than empty and callers must not treat it as +// evidence of recovery. +func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, parsed bool) { + if len(stored) == 0 { + return capabilityDrift{first: true}, true } var previous, current capabilityDriftView - if json.Unmarshal(n.Capabilities, &previous) != nil || json.Unmarshal(payload, ¤t) != nil { - return + if json.Unmarshal(stored, &previous) != nil || json.Unmarshal(payload, ¤t) != nil { + return capabilityDrift{}, false } + drift.previousResolved = previous.Resolved + drift.resolved = current.Resolved verifiedNow := make(map[string]bool, len(current.DetectedBackends)) for _, backend := range current.DetectedBackends { verifiedNow[backend.Backend] = backend.Verified } - var lostBackends []string for _, backend := range previous.DetectedBackends { if backend.Verified && !verifiedNow[backend.Backend] { - lostBackends = append(lostBackends, backend.Backend) + drift.lostBackends = append(drift.lostBackends, backend.Backend) } } - var lostDevices []string for _, device := range previous.RenderDevices { if !slices.Contains(current.RenderDevices, device) { - lostDevices = append(lostDevices, device) + drift.lostDevices = append(drift.lostDevices, device) } } - if len(lostBackends) == 0 && len(lostDevices) == 0 { + return drift, true +} + +// logCapabilityChange records what changed between the stored report and the +// new one. Losing a verified backend or a render device is the case worth +// waking an operator for: it means a node that was picked for hardware work +// silently became less capable, which otherwise only shows up as slow or +// failing transcodes. +func logCapabilityChange(ctx context.Context, n *Node, drift capabilityDrift, parsed bool) { + if !parsed { + return + } + if drift.first { + slog.InfoContext(ctx, "node capabilities stored", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + return + } + if !drift.regressed() { return } slog.WarnContext(ctx, "node capability drift", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL, - "lost_verified_backends", lostBackends, "lost_render_devices", lostDevices, - "previous_resolved", previous.Resolved, "resolved", current.Resolved) + "lost_verified_backends", drift.lostBackends, "lost_render_devices", drift.lostDevices, + "previous_resolved", drift.previousResolved, "resolved", drift.resolved) } diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go new file mode 100644 index 000000000..076170465 --- /dev/null +++ b/internal/nodepool/health_drift_test.go @@ -0,0 +1,252 @@ +package nodepool + +import ( + "context" + "encoding/json" + "strings" + "testing" + "unicode/utf8" +) + +const degradedCapabilityPayload = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false}],"capability_hash":"sha256:degraded"}` + +// The drift log line only reaches an operator who is reading logs. Persisting +// the same finding is what puts it on the node list, so the pool copy has to +// carry it the moment the report is applied. +func TestHealthCheckerCarriesCapabilityDriftIntoThePool(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilityDrift == nil { + t.Fatal("pool copy carries no capability_drift after a regression") + } + note := *stored.CapabilityDrift + for _, want := range []string{"nvenc", "/dev/dri/renderD128", "none"} { + if !strings.Contains(note, want) { + t.Fatalf("drift note %q does not name %q", note, want) + } + } +} + +// A repaired node must stop being flagged. The note describes the last +// comparison, not a latched incident, so a clean report clears it — otherwise a +// one-off driver hiccup would mark a node broken forever. +func TestHealthCheckerClearsCapabilityDriftOnRecovery(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + previousNote := "verified hardware backends lost: nvenc" + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + }, fetcher) + + fixture.sweep() + + if stored := fixture.storedNode(t); stored.CapabilityDrift != nil { + t.Fatalf("capability_drift = %q after recovery, want it cleared", *stored.CapabilityDrift) + } +} + +// A still-degraded report is not a recovery. Drift is a delta, so the refetch +// after a regression finds nothing *newly* lost — and clearing the note there +// would tell an operator the node is fine while its backend is still failing its +// probe. Anything that moves the hash reaches this path: a reboot moves boot_id, +// a reworded FFmpeg failure moves the probe reason. +func TestHealthCheckerKeepsCapabilityDriftWhileTheReportIsStillDegraded(t *testing.T) { + const rebootedPayload = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false,"reason":"no such device"}],` + + `"boot_id":"after-reboot","capability_hash":"sha256:rebooted"}` + url := newHealthNode(t, "sha256:rebooted") + fetcher := &fakeCapabilityFetcher{payload: []byte(rebootedPayload), hash: "sha256:rebooted"} + previousNote := "verified hardware backends lost: nvenc" + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + }, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilityDrift == nil { + t.Fatal("capability_drift cleared by a refetch that found the node still degraded") + } + if *stored.CapabilityDrift != previousNote { + t.Fatalf("capability_drift = %q, want the standing note %q", *stored.CapabilityDrift, previousNote) + } +} + +// The operator-triggered re-probe refetches unconditionally, with no hash gate. +// It is the action the docs and the UI tooltip prescribe for checking whether a +// drift note is still true, so it must be able to answer "yes" — clearing the +// badge on the first click regardless of what the probe found would make the +// only tool for confirming the note the tool that destroys it. +func TestRefreshNodeCapabilitiesKeepsDriftWhenNothingRecovered(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + previousNote := "verified hardware backends lost: nvenc" + node := &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + } + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err != nil { + t.Fatalf("RefreshNodeCapabilities: %v", err) + } + + if stored := fixture.storedNode(t); stored.CapabilityDrift == nil { + t.Fatal("a re-probe that found the same failing probe cleared the drift note") + } +} + +// A backend that was skipped is a statement about device access, not about +// hardware: it is the normal reading for a proxy pointed at a cluster-wide +// hw_device. It must not hold a drift note open forever. +func TestResolveDriftNoteClearsOnASkippedButOtherwiseCleanReport(t *testing.T) { + const skippedPayload = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"vaapi","verified":false,"skipped":true}]}` + standing := "verified hardware backends lost: vaapi" + drift, parsed := computeCapabilityDrift([]byte(skippedPayload), []byte(skippedPayload)) + if got := resolveDriftNote(&standing, drift, parsed, []byte(skippedPayload)); got != nil { + t.Fatalf("capability_drift = %q, want a skipped backend to count as clean", *got) + } +} + +// A node's very first report has nothing to compare against, so it must not be +// flagged. +func TestHealthCheckerStoresNoDriftOnFirstReport(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + }, fetcher) + + fixture.sweep() + + if stored := fixture.storedNode(t); stored.CapabilityDrift != nil { + t.Fatalf("capability_drift = %q on a first report, want none", *stored.CapabilityDrift) + } +} + +// An unreadable payload means the drift is unknown, not that the node recovered. +func TestComputeCapabilityDriftReportsUnparseablePayloads(t *testing.T) { + if _, parsed := computeCapabilityDrift([]byte(testCapabilityPayload), []byte(`not json`)); parsed { + t.Fatal("an unreadable new report parsed") + } + if _, parsed := computeCapabilityDrift([]byte(`not json`), []byte(testCapabilityPayload)); parsed { + t.Fatal("an unreadable stored report parsed") + } + drift, parsed := computeCapabilityDrift(nil, []byte(testCapabilityPayload)) + if !parsed || !drift.first || drift.regressed() { + t.Fatalf("first report drift = %+v, parsed = %v", drift, parsed) + } +} + +// The note is echoed to every admin listing nodes, and its inputs come from a +// worker that may run on remote hardware. +func TestCapabilityDriftNoteIsBounded(t *testing.T) { + drift := capabilityDrift{} + for range 400 { + drift.lostDevices = append(drift.lostDevices, "/dev/dri/renderD128") + } + note := drift.persistedNote() + if note == nil { + t.Fatal("a regression produced no note") + } + if len(*note) > maxCapabilityDriftNoteBytes+3 { + t.Fatalf("note is %d bytes, want it bounded at %d", len(*note), maxCapabilityDriftNoteBytes) + } +} + +// capability_drift is a Postgres text column, which rejects invalid UTF-8 +// outright — and the rejected UPDATE takes capabilities and capabilities_hash +// with it, so the stored hash never advances and every later sweep refetches and +// fails again. Device names come from a worker that may run elsewhere, so the +// bound has to cut on a rune boundary at every alignment, not just the lucky +// ones. +func TestCapabilityDriftNoteStaysValidUTF8AtEveryTruncationOffset(t *testing.T) { + for pad := range 8 { + drift := capabilityDrift{} + for range 40 { + drift.lostDevices = append(drift.lostDevices, + "/dev/dri/"+strings.Repeat("x", pad)+strings.Repeat("é", 12)) + } + note := drift.persistedNote() + if note == nil { + t.Fatalf("pad %d: a regression produced no note", pad) + } + if len(*note) <= maxCapabilityDriftNoteBytes { + t.Fatalf("pad %d: note is %d bytes, the fixture must exceed the bound", pad, len(*note)) + } + if !utf8.ValidString(*note) { + t.Fatalf("pad %d: truncated note is not valid UTF-8: %q", pad, *note) + } + } +} + +// The operator-triggered re-probe stores the node's new report immediately, and +// must go through the same fetch, drift, and publish path the sweep uses rather +// than a second implementation. +func TestRefreshNodeCapabilitiesStoresImmediately(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + node := &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + } + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err != nil { + t.Fatalf("RefreshNodeCapabilities: %v", err) + } + + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:degraded" { + t.Fatalf("stored hash = %v, want the refetched report", stored.CapabilitiesHash) + } + if stored.CapabilityDrift == nil { + t.Fatal("an immediate refresh did not persist the drift the sweep would have") + } + // The capability cache must be told, exactly as on a sweep refresh. + if notifications := fixture.notifications(); len(notifications) != 1 || notifications[0] != url { + t.Fatalf("notifications = %v, want one for %s", notifications, url) + } + if got := fetcher.callCount(); got != 1 { + t.Fatalf("fetch calls = %d, want exactly one", got) + } +} + +// A refresh already running is at least as fresh as the one being asked for, so +// the second caller is told rather than starting a duplicate fetch. +func TestRefreshNodeCapabilitiesRefusesWhenAlreadyInFlight(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + node := &Node{ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true} + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + fixture.checker.capabilityRefreshInFlight.Store(node.ID, struct{}{}) + defer fixture.checker.capabilityRefreshInFlight.Delete(node.ID) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err == nil { + t.Fatal("a duplicate refresh was allowed to start") + } + if got := fetcher.callCount(); got != 0 { + t.Fatalf("fetch calls = %d, want none while a refresh is in flight", got) + } +} diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 8db34457f..f5e59cd63 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -2,6 +2,7 @@ package nodepool import ( "context" + "log/slog" "sync" "time" @@ -85,16 +86,31 @@ type Planner struct { rr map[string]int // per-group round-robin cursor; "" = global reserved map[string]*reservation // keyed by playback session ID now func() time.Time // overridable for tests + // scratchPressed latches, by node id, which nodes were under scratch + // pressure the last time a transcode selection looked. It exists only to + // make the operator warning fire once per transition into pressure: the + // eligibility path runs on every session start and every quality switch, so + // logging there unlatched would produce a line per playback event for as + // long as a disk stays full. Guarded by mu, which every selection path + // already holds. Entries are pruned as nodes leave the pool. + scratchPressed map[int]bool + // scratchGuardDropped latches whether the last transcode selection had to + // ignore the scratch guard because every eligible candidate was pressured. + // Latched for the same reason as scratchPressed, and separately from it + // because a cluster reaches "no headroom anywhere" on its own schedule. + // Guarded by mu. + scratchGuardDropped bool } // NewPlanner creates a planner over the given pools. func NewPlanner(proxies *ProxyPool, transcodes *TranscodePool) *Planner { return &Planner{ - proxies: proxies, - transcodes: transcodes, - rr: make(map[string]int), - reserved: make(map[string]*reservation), - now: time.Now, + proxies: proxies, + transcodes: transcodes, + rr: make(map[string]int), + reserved: make(map[string]*reservation), + now: time.Now, + scratchPressed: make(map[int]bool), } } @@ -509,11 +525,138 @@ func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, elig // load is summed over all of it, since a job on a node this plan may not select // still occupies the same GPU. func (p *Planner) pickTranscode(transcodes, pool, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { - return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { + return p.pickWithScratchGuard(transcodes, pool, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, proxies, groupHealthy, estKbps, now) }, p.physicalGPULoadScore(pool, now)) } +// pickWithScratchGuard is pickNode with the scratch-pressure exclusion applied +// as a *soft* filter: a node whose transcode scratch volume is at or past +// scratchPressureFillPercent is skipped, unless skipping leaves no candidate at +// all, in which case the guard is ignored and the ordinary pick stands. +// +// Soft is the whole point. The guard's job is to steer sessions away from a node +// that will die mid-stream while a healthy sibling exists; it is not a license +// to refuse playback. A cluster whose scratch volumes have all filled — one shared +// NFS export, a retention setting that is too generous everywhere — would +// otherwise go from degraded to dark on a threshold nobody chose for that +// purpose. Degraded service beats no service, and the WARN below is what tells +// an operator which it is. +// +// pool is the full transcode pool the candidates were drawn from; it scopes the +// warning latch, which tracks disks rather than the narrowed candidate set. +// +// Callers must hold mu. +func (p *Planner) pickWithScratchGuard(nodes, pool []*Node, currentURL string, now time.Time, eligible func(*Node) bool, tieBreak func(*Node) int) *Node { + excluded := false + guarded := func(n *Node) bool { + if !eligible(n) { + return false + } + if scratchPressured(n) { + excluded = true + return false + } + return true + } + picked := p.pickNode(nodes, currentURL, now, guarded, tieBreak) + guardDropped := false + if picked == nil && excluded { + // Without an exclusion the unguarded pick would return the same nil: + // nothing was eligible in the first place. + picked = p.pickNode(nodes, currentURL, now, eligible, tieBreak) + guardDropped = picked != nil + } + // Logged after the fallback decides, because what an operator needs to know + // is whether the pressured node was actually kept out of selection. + p.noteScratchPressure(pool, guardDropped) + return picked +} + +// noteScratchPressure logs each pooled node's transition into and out of +// scratch pressure exactly once, and separately latches the cluster-wide state +// where the guard had to give way. +// +// guardDropped reports that this pick found every eligible candidate pressured +// and admitted one anyway. It has to reach the log, because the two states the +// guard can be in are opposites for an operator: "sessions are being steered +// away from this disk" is a warning, while "sessions are landing on a disk that +// is about to fail mid-stream because nothing else is eligible" is an outage in +// progress. One invariant message asserting an exclusion cannot say which, and +// the pressure latch alone would keep asserting the exclusion long after the +// fallback started ignoring it. +// +// There is no context to log against: selection is synchronous inside +// PlanSession and friends, which take no ctx because they are called from +// several request paths and from reservation code that has none. The latch, +// not a context, is what keeps this out of the hot path's log volume — the +// eligibility check runs on every session start and every quality switch. +// +// Callers must hold mu. +func (p *Planner) noteScratchPressure(pool []*Node, guardDropped bool) { + seen := make(map[int]struct{}, len(pool)) + for _, n := range pool { + if n == nil { + continue + } + seen[n.ID] = struct{}{} + pressured := scratchPressured(n) + if pressured == p.scratchPressed[n.ID] { + continue + } + if pressured { + pct, _ := scratchDiskFillPercent(n) + attrs := []any{ + "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL, + "scratch_fill_pct", pct, "threshold_pct", scratchPressureFillPercent, + "scratch_guard_dropped", guardDropped, + } + if guardDropped { + slog.Warn("transcode node scratch volume nearly full, still selected because no eligible node has scratch headroom", attrs...) + } else { + slog.Warn("transcode node scratch volume nearly full, excluded from selection", attrs...) + } + p.scratchPressed[n.ID] = true + continue + } + slog.Info("transcode node scratch volume recovered", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + delete(p.scratchPressed, n.ID) + } + // A node that left the pool must not keep a latch entry: it would suppress + // the warning if the same id came back still full. + for id := range p.scratchPressed { + if _, ok := seen[id]; !ok { + delete(p.scratchPressed, id) + } + } + p.noteScratchGuardDropped(guardDropped) +} + +// noteScratchGuardDropped reports the transition into and out of the state where +// the scratch guard has nothing left to steer towards. +// +// It is latched separately from the per-node pressure warnings because the two +// move independently: a cluster can cross into "no headroom anywhere" long after +// its nodes' pressure transitions were logged, and without this the operator's +// only signal would be an exclusion message written before the exclusion stopped +// happening. +// +// Callers must hold mu. +func (p *Planner) noteScratchGuardDropped(dropped bool) { + if dropped == p.scratchGuardDropped { + return + } + p.scratchGuardDropped = dropped + if dropped { + slog.Warn("transcode scratch guard ignored: every eligible node is over the scratch threshold", + "component", "nodepool", "threshold_pct", scratchPressureFillPercent) + return + } + slog.Info("transcode scratch guard back in force: an eligible node has scratch headroom again", + "component", "nodepool") +} + // physicalGPULoadScore precomputes, once per pick, the total effective jobs // running on each pooled node's physical GPU group: itself plus every pooled // node sharing at least one GPU identity with it, healthy or not, because a job @@ -578,7 +721,7 @@ func (p *Planner) physicalGPULoadScore(pool []*Node, now time.Time) func(*Node) // transcodeEligible reduces it to exactly that: healthy, enabled, under cap, // and group-healthy, with no proxy partner required. func (p *Planner) pickLocalEgressTranscode(transcodes, pool []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { - return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { + return p.pickWithScratchGuard(transcodes, pool, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, nil, groupHealthy, 0, now) }, p.physicalGPULoadScore(pool, now)) } @@ -588,6 +731,12 @@ func (p *Planner) pickLocalEgressTranscode(transcodes, pool []*Node, groupHealth // its whole group healthy and — when the group contains proxies — at least // one of them with job and bandwidth headroom (a group's capacity is bounded // by its proxies). +// +// Scratch-volume pressure is deliberately *not* checked here. It is a soft +// exclusion applied by pickWithScratchGuard, which can drop it when it would +// empty the candidate set; a hard predicate could not. Callers that use this +// directly (ReserveTranscodeWorkWith's non-streaming reservations) intentionally +// keep the historical behavior. func (p *Planner) transcodeEligible(n *Node, proxies []*Node, groupHealthy map[string]bool, estKbps int, now time.Time) bool { if !n.Healthy || !n.Enabled || !p.underCap(n, now) { return false diff --git a/internal/nodepool/planner_scratch_test.go b/internal/nodepool/planner_scratch_test.go new file mode 100644 index 000000000..659b56469 --- /dev/null +++ b/internal/nodepool/planner_scratch_test.go @@ -0,0 +1,237 @@ +package nodepool + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +// captureScratchLogs redirects the default logger, which is what selection logs +// against: PlanSession takes no context, so there is no handler to inject. +func captureScratchLogs(t *testing.T) *bytes.Buffer { + t.Helper() + buffer := &bytes.Buffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(buffer, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return buffer +} + +// A transcode writes segments to its scratch volume for the life of the session, +// so admitting one onto a nearly full node buys a stream that dies mid-playback. +// With a healthy sibling available the pressured node must lose, even though it +// carries fewer jobs and would otherwise win outright. +func TestPlanSessionSkipsScratchPressuredTranscodeNode(t *testing.T) { + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(96, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 3) + roomy.LastStats = scratchStats(20, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{full, roomy}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("no transcode node selected") + } + if plan.TranscodeNode.URL != "http://tc-roomy" { + t.Fatalf("selected %s, want the node with scratch headroom", plan.TranscodeNode.URL) + } +} + +// The guard is soft on purpose. If every otherwise-eligible node is over the +// threshold, refusing to plan would turn a degraded cluster into a dark one, and +// the threshold was never chosen to be a kill switch. +func TestPlanSessionIgnoresScratchGuardWhenEveryNodeIsPressured(t *testing.T) { + first := transcodeNode(1, "http://tc-a", nil, 5) + first.LastStats = scratchStats(99, 100) + second := transcodeNode(2, "http://tc-b", nil, 1) + second.LastStats = scratchStats(97, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{first, second}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("scratch pressure emptied the pool; degraded service must beat no service") + } + // With the guard dropped the ordinary least-jobs rule decides. + if plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("selected %s, want the least-loaded node once the guard is ignored", plan.TranscodeNode.URL) + } + if plan.ProxyNode == nil { + t.Fatal("no proxy paired with the fallback transcode node") + } +} + +// The WARN is the operator's only signal about scratch pressure, so it must not +// claim an exclusion that did not happen. When every eligible node is over the +// threshold the guard gives way and the pressured node takes the session — the +// opposite of "excluded from selection", and a far more urgent state. +func TestScratchPressureWarningReportsTheDegradedAdmission(t *testing.T) { + logs := captureScratchLogs(t) + first := transcodeNode(1, "http://tc-a", nil, 5) + first.LastStats = scratchStats(99, 100) + second := transcodeNode(2, "http://tc-b", nil, 1) + second.LastStats = scratchStats(97, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{first, second}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("scratch pressure emptied the pool; degraded service must beat no service") + } + + output := logs.String() + if strings.Contains(output, "excluded from selection") { + t.Fatalf("logged an exclusion for a node it then selected (%s):\n%s", plan.TranscodeNode.URL, output) + } + if !strings.Contains(output, "still selected because no eligible node has scratch headroom") { + t.Fatalf("no degraded-admission warning:\n%s", output) + } + if !strings.Contains(output, "transcode scratch guard ignored") { + t.Fatalf("the cluster-wide guard-dropped state was never reported:\n%s", output) + } +} + +// With a sibling that has headroom the guard really does exclude, and the +// message has to say so — the two states are told apart by wording, so both have +// to be exercised. +func TestScratchPressureWarningReportsARealExclusion(t *testing.T) { + logs := captureScratchLogs(t) + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(99, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 3) + roomy.LastStats = scratchStats(20, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{full, roomy}) + + f.planner.PlanSession("session-1", "", true, 0) + + output := logs.String() + if !strings.Contains(output, "excluded from selection") { + t.Fatalf("no exclusion warning while a node with headroom existed:\n%s", output) + } + if strings.Contains(output, "transcode scratch guard ignored") { + t.Fatalf("reported the guard as dropped while it was honored:\n%s", output) + } +} + +// The guard-dropped state is latched like the per-node pressure warnings, and +// reports its way back out: an operator who saw the outage warning needs to know +// when steering resumed. +func TestScratchGuardDroppedStateLatchesAndRecovers(t *testing.T) { + logs := captureScratchLogs(t) + only := transcodeNode(1, "http://tc-a", nil, 0) + only.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{only}) + + for range 3 { + f.planner.PlanSession("session-1", "", true, 0) + } + if got := strings.Count(logs.String(), "transcode scratch guard ignored"); got != 1 { + t.Fatalf("guard-dropped warnings = %d across three plans, want exactly one", got) + } + + recovered := transcodeNode(1, "http://tc-a", nil, 0) + recovered.LastStats = scratchStats(10, 100) + f.transcodes.SetNodes([]*Node{recovered}) + f.planner.PlanSession("session-2", "", true, 0) + + if !strings.Contains(logs.String(), "transcode scratch guard back in force") { + t.Fatalf("the guard coming back into force was never reported:\n%s", logs.String()) + } +} + +// A node whose sample cannot be read must not be excluded: the guard fires on +// measured pressure only, and a node predating the scratch flag reports none. +func TestPlanSessionAdmitsNodeWithoutScratchStats(t *testing.T) { + unknown := transcodeNode(1, "http://tc-unknown", nil, 0) + pressured := transcodeNode(2, "http://tc-full", nil, 0) + pressured.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{unknown, pressured}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-unknown" { + t.Fatalf("selected %v, want the node with no scratch reading", plan.TranscodeNode) + } +} + +// Soft affinity keeps a session on its current node unless a candidate is two +// jobs better. Scratch pressure has to beat that: staying is what kills the +// stream, and the affinity rule exists to avoid gratuitous switches, not to +// defend a full disk. +func TestPlanSessionMovesOffAScratchPressuredCurrentNode(t *testing.T) { + current := transcodeNode(1, "http://tc-current", nil, 0) + current.LastStats = scratchStats(99, 100) + other := transcodeNode(2, "http://tc-other", nil, 0) + other.LastStats = scratchStats(10, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{current, other}) + + plan := f.planner.PlanSession("session-1", "http://tc-current", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-other" { + t.Fatalf("selected %v, want the session moved off the full-scratch node", plan.TranscodeNode) + } +} + +// The local-egress path admits transcodes without a proxy partner and must apply +// the same guard, since it is the same scratch volume being written. +func TestPlanTranscodeSessionWithLocalEgressAppliesScratchGuard(t *testing.T) { + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(96, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 2) + roomy.LastStats = scratchStats(20, 100) + f := newFixture(nil, []*Node{full, roomy}) + + plan := f.planner.PlanTranscodeSessionWithLocalEgress("session-1", "", nil) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-roomy" { + t.Fatalf("selected %v, want the node with scratch headroom", plan.TranscodeNode) + } +} + +// The warning is latched so a full disk does not produce a log line per session +// start. The latch is internal state, so this asserts the transitions it records +// rather than the log output. +func TestScratchPressureWarningLatchesPerTransition(t *testing.T) { + node := transcodeNode(1, "http://tc-a", nil, 0) + node.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{node}) + + for range 3 { + f.planner.PlanSession("session-1", "", true, 0) + } + f.planner.mu.Lock() + latched := len(f.planner.scratchPressed) + pressed := f.planner.scratchPressed[1] + f.planner.mu.Unlock() + if latched != 1 || !pressed { + t.Fatalf("latch = %v, want node 1 latched once", f.planner.scratchPressed) + } + + // Recovery clears the latch, so the next episode warns again. + recovered := transcodeNode(1, "http://tc-a", nil, 0) + recovered.LastStats = scratchStats(10, 100) + f.transcodes.SetNodes([]*Node{recovered}) + f.planner.PlanSession("session-2", "", true, 0) + + f.planner.mu.Lock() + latched = len(f.planner.scratchPressed) + f.planner.mu.Unlock() + if latched != 0 { + t.Fatalf("latch = %v after recovery, want it cleared", f.planner.scratchPressed) + } +} + +// A node that leaves the pool must not leave a latch entry behind: the same id +// coming back still full would then never warn. +func TestScratchPressureLatchIsPrunedWhenANodeLeavesThePool(t *testing.T) { + node := transcodeNode(1, "http://tc-a", nil, 0) + node.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{node}) + f.planner.PlanSession("session-1", "", true, 0) + + f.transcodes.SetNodes([]*Node{transcodeNode(2, "http://tc-b", nil, 0)}) + f.planner.PlanSession("session-2", "", true, 0) + + f.planner.mu.Lock() + _, stillLatched := f.planner.scratchPressed[1] + f.planner.mu.Unlock() + if stillLatched { + t.Fatal("latch entry survived the node leaving the pool") + } +} diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index f33aaf638..5c472cfe4 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -71,8 +71,8 @@ func (p *ProxyPool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int // ApplyCapabilities records a freshly fetched capability report by swapping the // node for an updated copy, keeping published *Node values immutable. -func (p *ProxyPool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time) { +func (p *ProxyPool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { p.mu.Lock() defer p.mu.Unlock() - applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt) + applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt, drift) } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 71951b959..74b9671d4 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -58,6 +58,14 @@ type Node struct { // what the node itself resolves against once it has reloaded its config. HWAccelOverride *string `json:"hw_accel_override,omitempty"` HWDeviceOverride *string `json:"hw_device_override,omitempty"` + // CapabilityDrift is an operator-facing note describing how this node's + // hardware got worse at the last capability refetch: a backend that used to + // pass its probe and now fails, or a render device that is gone. nil means + // the last refetch found no regression, which is also how a recovered node + // reads — the note is rewritten by every refetch and a clean report clears + // it. It is written beside Capabilities in one statement, so it always + // describes the report stored with it, and nothing routes on it. + CapabilityDrift *string `json:"capability_drift,omitempty"` // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from // Capabilities rather than stored: it is a pure function of that payload, so // a column would only be a second copy that can disagree with it. Two nodes @@ -242,7 +250,7 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override` +const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override, capability_drift` func scanNode(row pgx.Row) (*Node, error) { var n Node @@ -258,6 +266,7 @@ func scanNode(row pgx.Row) (*Node, error) { &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, &lastStats, &n.HWAccelOverride, &n.HWDeviceOverride, + &n.CapabilityDrift, ) if err != nil { return nil, err @@ -426,13 +435,19 @@ func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, act } // UpdateCapabilities persists a freshly fetched capability report together with -// the hash that identifies it. The three columns are written in one statement -// so a reader never sees a payload beside a hash from a different report. -func (r *Repository) UpdateCapabilities(ctx context.Context, id int, capabilities []byte, hash string, refreshedAt time.Time) error { +// the hash that identifies it and the drift note comparing it against the +// previous one. The four columns are written in one statement so a reader never +// sees a payload beside a hash — or a drift note — from a different report. +// +// A nil drift writes NULL, which is how a node that has recovered stops being +// flagged: the note describes the last comparison, not a latched incident, so +// carrying the previous value forward would keep a repaired driver on screen as +// broken. +func (r *Repository) UpdateCapabilities(ctx context.Context, id int, capabilities []byte, hash string, refreshedAt time.Time, drift *string) error { tag, err := r.pool.Exec(ctx, - `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4 + `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4, capability_drift = $5 WHERE id = $1`, - id, capabilities, hash, refreshedAt) + id, capabilities, hash, refreshedAt, drift) if err != nil { return fmt.Errorf("update node capabilities: %w", err) } diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index 17ee629f5..c7b42a058 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -33,7 +33,7 @@ func newNodeTestPool(t *testing.T) *pgxpool.Pool { if err := pool.QueryRow(ctx, `SELECT count(*) FROM information_schema.columns WHERE table_name = 'stream_nodes' - AND column_name IN ('capabilities_hash', 'hw_accel_override')`).Scan(&columns); err != nil || columns < 2 { + AND column_name IN ('capabilities_hash', 'hw_accel_override', 'capability_drift')`).Scan(&columns); err != nil || columns < 3 { t.Skip("test database has not applied the stream_nodes capability/override migrations") } return pool @@ -63,7 +63,7 @@ func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { payload := json.RawMessage(`{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"]}`) refreshedAt := time.Now().UTC().Truncate(time.Millisecond) - if err := repo.UpdateCapabilities(ctx, node.ID, payload, "sha256:abc", refreshedAt); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, payload, "sha256:abc", refreshedAt, nil); err != nil { t.Fatalf("update capabilities: %v", err) } @@ -89,9 +89,56 @@ func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { } } +// The drift note is what puts a silent hardware regression on the node list, so +// it has to survive a write and come back through an ordinary read — and a later +// clean refetch has to clear it, or a repaired node stays flagged forever. +func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("drift-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://drift-test-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + if node.CapabilityDrift != nil { + t.Fatalf("new node already carries drift: %q", *node.CapabilityDrift) + } + + note := "verified hardware backends lost: nvenc; resolved backend nvenc -> none" + payload := json.RawMessage(`{"resolved":"none"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, payload, "sha256:degraded", time.Now(), ¬e); err != nil { + t.Fatalf("update capabilities with drift: %v", err) + } + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.CapabilityDrift == nil || *reloaded.CapabilityDrift != note { + t.Fatalf("stored drift = %v, want %q", reloaded.CapabilityDrift, note) + } + + recovered := json.RawMessage(`{"resolved":"nvenc"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, recovered, "sha256:recovered", time.Now(), nil); err != nil { + t.Fatalf("update capabilities without drift: %v", err) + } + reloaded, err = repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.CapabilityDrift != nil { + t.Fatalf("drift = %q after recovery, want NULL", *reloaded.CapabilityDrift) + } +} + func TestRepositoryUpdateCapabilitiesUnknownNode(t *testing.T) { repo := NewRepository(newNodeTestPool(t)) - err := repo.UpdateCapabilities(context.Background(), -1, []byte(`{}`), "sha256:abc", time.Now()) + err := repo.UpdateCapabilities(context.Background(), -1, []byte(`{}`), "sha256:abc", time.Now(), nil) if !errors.Is(err, ErrNodeNotFound) { t.Fatalf("err = %v, want ErrNodeNotFound", err) } diff --git a/internal/nodepool/scratchpressure.go b/internal/nodepool/scratchpressure.go new file mode 100644 index 000000000..cf5316f9e --- /dev/null +++ b/internal/nodepool/scratchpressure.go @@ -0,0 +1,79 @@ +package nodepool + +import "encoding/json" + +// scratchStatsView is the minimal projection this package parses out of an +// otherwise opaque last_stats payload, in the same spirit as +// capabilityDriftView and gpuIdentityView: nodepool must not depend on +// nodemetrics, and admission only needs the scratch volume's fill. +// +// The scratch entry is found by its own flag rather than by matching a path, +// because the API does not know a node's transcode directory — the node does, +// and it says which entry is the one. +type scratchStatsView struct { + System struct { + Disks []struct { + Scratch bool `json:"scratch"` + UsedGB float64 `json:"used_gb"` + TotalGB float64 `json:"total_gb"` + Stale bool `json:"stale"` + Unavailable bool `json:"unavailable"` + } `json:"disks"` + } `json:"system"` +} + +// scratchPressureFillPercent is the scratch-volume fill at which a transcode +// node stops being offered new sessions. +// +// A transcode writes HLS segments to that volume for the life of the session, so +// a node that is nearly full does not fail fast: it accepts the session, streams +// for a while, and then dies mid-playback with a write error — the worst failure +// shape available, because the client has already committed to it. Five percent +// headroom is a few minutes of segments at any realistic bitrate, which is what +// makes it enough to notice and act on rather than a hard stop. +// +// It is deliberately high. A scratch volume sitting at 80% is a normal steady +// state for a node with a large segment retention, and excluding those would +// shrink a healthy cluster for no reason. +const scratchPressureFillPercent = 95 + +// scratchDiskFillPercent reports the fill percentage of a node's transcode +// scratch volume from its last health sample. +// +// ok is false whenever the answer would be a guess rather than a measurement: +// no sample, an unparseable one, no scratch entry (a node predating the flag, or +// a proxy with no scratch dir), a path the node could not measure, a capacity of +// zero, or numbers the node itself marked stale. Every one of those means the +// admission guard must not fire — excluding a node on a fill we cannot read +// would take capacity away on no evidence, and a full disk that is still being +// written to shows up as a failing transcode, which is recoverable, while an +// empty pool is not. +func scratchDiskFillPercent(n *Node) (pct int, ok bool) { + if n == nil || len(n.LastStats) == 0 { + return 0, false + } + var view scratchStatsView + if err := json.Unmarshal(n.LastStats, &view); err != nil { + return 0, false + } + for _, disk := range view.System.Disks { + if !disk.Scratch { + continue + } + if disk.Unavailable || disk.Stale || disk.TotalGB <= 0 || disk.UsedGB < 0 { + return 0, false + } + // Floored, so the threshold reads as "95% or more of the volume is + // used" rather than rounding a 94.6% volume into exclusion. + return int(disk.UsedGB / disk.TotalGB * 100), true + } + return 0, false +} + +// scratchPressured reports whether a node's scratch volume is too full to admit +// new work. A node whose fill cannot be read is never pressured; see +// scratchDiskFillPercent. +func scratchPressured(n *Node) bool { + pct, ok := scratchDiskFillPercent(n) + return ok && pct >= scratchPressureFillPercent +} diff --git a/internal/nodepool/scratchpressure_test.go b/internal/nodepool/scratchpressure_test.go new file mode 100644 index 000000000..50a80fd21 --- /dev/null +++ b/internal/nodepool/scratchpressure_test.go @@ -0,0 +1,81 @@ +package nodepool + +import ( + "encoding/json" + "fmt" + "testing" +) + +// scratchStats builds the last_stats blob a node's health response produces for +// a scratch volume at the given fill, in the shape nodemetrics publishes. +func scratchStats(usedGB, totalGB float64, flags ...string) json.RawMessage { + extra := "" + for _, flag := range flags { + extra += fmt.Sprintf(`,"%s":true`, flag) + } + return json.RawMessage(fmt.Sprintf( + `{"system":{"disks":[{"path":"/transcode","used_gb":%g,"total_gb":%g,"scratch":true%s},`+ + `{"path":"/media","used_gb":10,"total_gb":100}]}}`, usedGB, totalGB, extra)) +} + +func TestScratchDiskFillPercent(t *testing.T) { + tests := []struct { + name string + stats json.RawMessage + wantPct int + wantOK bool + }{ + {name: "measured", stats: scratchStats(50, 100), wantPct: 50, wantOK: true}, + {name: "at threshold", stats: scratchStats(95, 100), wantPct: 95, wantOK: true}, + // Floored rather than rounded, so "95%" means at least 95% used. + {name: "just under threshold", stats: scratchStats(94.99, 100), wantPct: 94, wantOK: true}, + {name: "full", stats: scratchStats(100, 100), wantPct: 100, wantOK: true}, + {name: "no stats at all", stats: nil}, + {name: "unparseable", stats: json.RawMessage(`not json`)}, + {name: "stale numbers", stats: scratchStats(99, 100, "stale")}, + {name: "unmeasurable path", stats: scratchStats(0, 0, "unavailable")}, + {name: "zero capacity", stats: scratchStats(0, 0)}, + { + name: "no scratch entry", + stats: json.RawMessage(`{"system":{"disks":[{"path":"/media","used_gb":99,"total_gb":100}]}}`), + }, + {name: "no system section", stats: json.RawMessage(`{"gpu":[]}`)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pct, ok := scratchDiskFillPercent(&Node{LastStats: test.stats}) + if ok != test.wantOK { + t.Fatalf("ok = %v, want %v (pct = %d)", ok, test.wantOK, pct) + } + if ok && pct != test.wantPct { + t.Fatalf("pct = %d, want %d", pct, test.wantPct) + } + }) + } + if _, ok := scratchDiskFillPercent(nil); ok { + t.Fatal("a nil node reported a readable scratch fill") + } +} + +// The threshold is the whole contract of the guard, so the boundary is asserted +// on the predicate the planner actually calls. +func TestScratchPressuredThresholdBoundary(t *testing.T) { + for _, test := range []struct { + usedGB float64 + want bool + }{ + {usedGB: 94, want: false}, + {usedGB: 94.99, want: false}, + {usedGB: 95, want: true}, + {usedGB: 99.5, want: true}, + } { + if got := scratchPressured(&Node{LastStats: scratchStats(test.usedGB, 100)}); got != test.want { + t.Fatalf("scratchPressured(%g%% used) = %v, want %v", test.usedGB, got, test.want) + } + } + // Missing evidence is never pressure: excluding a node on a fill we cannot + // read would take capacity away for nothing. + if scratchPressured(&Node{}) { + t.Fatal("a node with no resource sample was treated as under scratch pressure") + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 336281da3..dd5d455ed 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -94,10 +94,10 @@ func (p *TranscodePool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps // ApplyCapabilities records a freshly fetched capability report by swapping the // node for an updated copy, keeping published *Node values immutable. -func (p *TranscodePool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time) { +func (p *TranscodePool) ApplyCapabilities(id int, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { p.mu.Lock() defer p.mu.Unlock() - applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt) + applyNodeCapabilities(p.nodes, id, capabilities, hash, refreshedAt, drift) } // applyNodeHealth replaces the slice entry for id with an updated copy. @@ -128,7 +128,11 @@ func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps // applyNodeCapabilities replaces the slice entry for id with a copy carrying // the new capability report. The payload is cloned because the caller's buffer // (a decoded HTTP response) is not ours to publish. -func applyNodeCapabilities(nodes []*Node, id int, capabilities []byte, hash string, refreshedAt time.Time) { +// +// drift is set verbatim, nil included: the note describes the comparison that +// produced this payload, so a node whose hardware recovered must lose the note +// at the same moment it gains the clean report. +func applyNodeCapabilities(nodes []*Node, id int, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { for i, n := range nodes { if n.ID != id { continue @@ -137,6 +141,7 @@ func applyNodeCapabilities(nodes []*Node, id int, capabilities []byte, hash stri clone.Capabilities = append(json.RawMessage(nil), capabilities...) clone.CapabilitiesHash = &hash clone.CapabilitiesRefreshedAt = &refreshedAt + clone.CapabilityDrift = drift // The GPU identities belong to the payload being replaced, so they are // re-derived rather than carried over from the previous report. applyPhysicalGPUKeys(&clone) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 96654cbcb..b8fadeff2 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -502,6 +502,29 @@ func hwProbeCacheEntryCurrent(entry hwProbeCacheEntry, now time.Time) bool { return entry.result.available || now.Before(entry.expiresAt) } +// InvalidateHWProbeCache drops every cached backend verdict so the next +// detection walk re-runs its FFmpeg smoke encodes against live hardware. +// +// It exists because a positive verdict is cached for the whole process +// lifetime, which is right for routing — a GPU that encoded a frame does not +// stop being able to between two playback requests — but blind to the one event +// that legitimately changes the answer: an operator replacing a driver, moving +// a card, or changing device access underneath a running node. Without this the +// only way to re-verify is a restart. +// +// A probe already in flight is neither canceled nor discarded: it completes and +// writes its result, and a caller that re-probes while another goroutine is +// mid-probe therefore joins that flight and can observe the pre-invalidation +// verdict once. Canceling shared work would instead fail an unrelated playback +// request that is waiting on it, which is the worse trade. The operator-facing +// re-probe action runs the detection walk itself after invalidating, so in the +// ordinary single-caller case the cache is repopulated from a cold start. +func InvalidateHWProbeCache() { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + hwProbeCache.entries = make(map[string]hwProbeCacheEntry) +} + // hwProbeCacheKey separates results per backend and per candidate device on top // of the FFmpeg binary's identity. func hwProbeCacheKey(ffmpegPath, backend, device string) string { diff --git a/internal/playback/gpudetect_invalidate_test.go b/internal/playback/gpudetect_invalidate_test.go new file mode 100644 index 000000000..f66f578fa --- /dev/null +++ b/internal/playback/gpudetect_invalidate_test.go @@ -0,0 +1,55 @@ +package playback + +import ( + "os" + "strings" + "testing" + "time" +) + +// A verified backend is cached for the process lifetime, which is exactly the +// blind spot InvalidateHWProbeCache exists to close: without it, a driver +// upgraded underneath a running node can only be noticed by restarting. The +// observable contract is that ffmpeg is executed again, so this counts +// invocations of the fake binary rather than inspecting the cache. +func TestInvalidateHWProbeCacheForcesAnotherProbe(t *testing.T) { + setupHWAccelTest(t) + // This case counts commands rather than racing a deadline, so it opts out of + // the shared fixture's very short probe timeout: a fake ffmpeg killed by a + // loaded machine would fail here as a probe failure, which is not what is + // under test. Restored by the fixture's own cleanup. + hwProbeCommandTimeout = 5 * time.Second + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + + probeCommands := func() int { + data, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatalf("read probe log: %v", err) + } + return strings.Count(string(data), "\n") + } + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("first probe failed: %s", reason) + } + first := probeCommands() + if first == 0 { + t.Fatal("first probe ran no ffmpeg commands") + } + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("cached probe failed: %s", reason) + } + if got := probeCommands(); got != first { + t.Fatalf("cached probe ran %d commands, want the cached verdict reused (%d)", got-first, first) + } + + InvalidateHWProbeCache() + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("probe after invalidation failed: %s", reason) + } + if got := probeCommands(); got != first*2 { + t.Fatalf("probe after invalidation ran %d commands total, want %d", got, first*2) + } +} diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 58dc7eaab..be8faf4f2 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -1047,8 +1047,9 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary { return fakeFFmpegBinary{path: path, logPath: logPath} } +// resetHWProbeCacheForTest clears the probe cache between cases. It delegates +// to the exported invalidation so tests exercise the same seam the re-probe +// action uses rather than a second, drifting implementation. func resetHWProbeCacheForTest() { - hwProbeCache.Lock() - defer hwProbeCache.Unlock() - hwProbeCache.entries = make(map[string]hwProbeCacheEntry) + InvalidateHWProbeCache() } diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go new file mode 100644 index 000000000..7fd3982cb --- /dev/null +++ b/internal/proxy/reprobe.go @@ -0,0 +1,77 @@ +package proxy + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// reprobeCapabilitiesResponse mirrors the transcode node's re-probe answer, so +// an operator action does not have to know which node type it is talking to. +type reprobeCapabilitiesResponse struct { + // Resolved is the backend this proxy would now use. + Resolved string `json:"resolved"` + // CapabilityHash identifies the snapshot this re-probe published. + CapabilityHash string `json:"capability_hash"` +} + +// handleReprobeCapabilities discards this proxy's cached probe verdicts and +// rebuilds the capability snapshot against live hardware. +// +// A proxy runs ffmpeg too (remux, Dolby Vision RPU strip), and the probe caches +// behind its snapshot keep a successful verdict for the process lifetime. That +// is correct for playback and blind to hardware that has since stopped working +// underneath it, which no cache key can observe — see the transcode node's copy +// of this handler for the full reasoning. A rebuild that does not finish keeps +// the previously published hash. +// +// Unlike the transcode node this does not refuse while busy. That guard is about +// encoder sessions: a proxy's jobs are remuxes and RPU strips, which hold no +// encoder slot for a probe's smoke encode to lose a race against. +func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { + playback.InvalidateHWProbeCache() + tonemap.InvalidateProbeCache() + + ctx, cancel := context.WithTimeout(r.Context(), s.capabilityProbeBudget()) + defer cancel() + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + slog.WarnContext(r.Context(), "proxy capability re-probe incomplete", "component", "proxy", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return + } + previous := s.storedCapabilityHash() + s.storeCapabilityHash(info.CapabilityHash) + slog.InfoContext(r.Context(), "proxy capabilities re-probed", "component", "proxy", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(reprobeCapabilitiesResponse{ + Resolved: info.Resolved, + CapabilityHash: info.CapabilityHash, + }); err != nil { + slog.WarnContext(r.Context(), "encode proxy re-probe result", "component", "proxy", "error", err) + } +} + +// capabilityProbeBudget is the deadline one snapshot rebuild gets. +// +// A proxy's snapshot is the bounded hardware walk plus the transformation +// registry's own bounded commands — not the tone-map matrix — so this is an +// over-allowance rather than a measurement. It is deliberately the same number +// the transcode node uses: both node types advertise one probe budget to +// callers, and a second constant here would be a second thing to keep in step +// with the walk and registry timeouts. +func (s *Server) capabilityProbeBudget() time.Duration { + hwAccel := playback.HWAccelNone + hwDevice := "" + if cfg := s.watcher.Config(); cfg != nil { + hwAccel = cfg.Playback.HWAccel + hwDevice = cfg.Playback.HWDevice + } + return tonemap.ProbeEndpointTimeout(hwAccel, hwDevice) +} diff --git a/internal/proxy/reprobe_test.go b/internal/proxy/reprobe_test.go new file mode 100644 index 000000000..15adb48f8 --- /dev/null +++ b/internal/proxy/reprobe_test.go @@ -0,0 +1,73 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// A proxy runs ffmpeg for remux recipes, so it gets the same escape hatch a +// transcode node has: re-probe now, publish the result, and let health advertise +// it immediately instead of at the next 15-minute tick. +func TestProxyReprobeCapabilitiesRecomputesAndStoresHash(t *testing.T) { + const secret = "capability-secret" + server := newDownloadProxyServer(t, secret) + server.storeCapabilityHash("sha256:stale") + + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result reprobeCapabilitiesResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode re-probe result: %v", err) + } + if result.CapabilityHash == "" || result.CapabilityHash == "sha256:stale" { + t.Fatalf("capability_hash = %q, want a recomputed hash", result.CapabilityHash) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != result.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the re-probed %q", got, result.CapabilityHash) + } +} + +// A re-probe that cannot finish must answer 503 and keep the published hash: an +// unfinished probe is not evidence the proxy lost hardware. +func TestProxyReprobeCapabilitiesKeepsHashOnIncompleteProbe(t *testing.T) { + const secret = "capability-secret" + server := newDownloadProxyServer(t, secret) + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published before the canceled re-probe") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil).WithContext(canceled) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503; body = %s", recorder.Code, recorder.Body.String()) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished re-probe, want %q", got, published) + } +} + +// The route executes ffmpeg, so it stays inside the bearer-authed admin group. +func TestProxyReprobeCapabilitiesRequiresBearer(t *testing.T) { + server := newDownloadProxyServer(t, "capability-secret") + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil)) + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 3f08d9f75..8de2eed25 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -201,6 +201,7 @@ func (s *Server) Handler() http.Handler { r.Use(s.requireBearer) r.Get("/hw-capabilities", s.handleHWCapabilities) r.Post("/admin/force-reload", s.handleForceReload) + r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) }) return r diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index e78e9cc08..3687eec07 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -1,5 +1,6 @@ # fixture 1 POST /admin/force-reload non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true @@ -24,6 +25,7 @@ HEAD /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true GET /stream/v3/{session_id}/segment/{name} media playback viewer_egress true true # fixture 2 POST /admin/force-reload non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index e4771ded4..479f8e782 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -155,6 +155,26 @@ func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { return strings.Join([]string{binaryIdentity, backend, device, strings.Join(driverIdentities, ",")}, "\x00") } +// InvalidateProbeCache drops every cached tone-map inventory so the next probe +// re-runs its listings and single-frame conversions. +// +// A non-empty inventory is cached permanently (see probeCacheEntryCurrent), +// which is right for playback and wrong for an operator who just upgraded a +// driver or an FFmpeg build in place: the binary's identity key only changes +// when the file does, and a driver has no key at all. This is the seam an +// operator-triggered re-probe uses to force the matrix to run again. +// +// A probe already in flight is neither canceled nor discarded: it completes and +// stores its result, so a caller that invalidates concurrently with another +// probe of the same key joins that flight and can observe the pre-invalidation +// inventory once. Canceling shared work would fail the unrelated playback +// request waiting on it. +func InvalidateProbeCache() { + probeCache.Lock() + defer probeCache.Unlock() + probeCache.entries = make(map[string]probeCacheEntry) +} + // probeCacheEntryCurrent reports whether a positive result or unexpired // negative result may be reused. func probeCacheEntryCurrent(entry probeCacheEntry, now time.Time) bool { diff --git a/internal/tonemap/probe_invalidate_test.go b/internal/tonemap/probe_invalidate_test.go new file mode 100644 index 000000000..ebf55e699 --- /dev/null +++ b/internal/tonemap/probe_invalidate_test.go @@ -0,0 +1,53 @@ +package tonemap + +import ( + "context" + "testing" + "time" +) + +// A non-empty inventory never expires, which is the blind spot +// InvalidateProbeCache closes: a driver replaced underneath a running node +// changes the answer without changing the binary's identity key. The observable +// contract is that the probe commands run again. +func TestInvalidateProbeCacheForcesAnotherProbe(t *testing.T) { + resetProbeCache(t) + now := time.Unix(100, 0) + calls := 0 + runner := func(_ context.Context, _ string, args ...string) ([]byte, error) { + calls++ + if len(args) > 0 && args[len(args)-1] == "-filters" { + return []byte(" .S. zscale V->V\n .S. tonemapx V->V\n .S. sidedata V->V\n"), nil + } + if len(args) > 0 && args[len(args)-1] == "-encoders" { + return []byte("libx264"), nil + } + return nil, nil + } + clock := func() time.Time { return now } + + probe := func(stage string) int { + capabilities, err := probeCached(context.Background(), "/ffmpeg-invalidate", BackendSoftware, "", runner, clock) + if err != nil { + t.Fatalf("%s probe error = %v", stage, err) + } + if len(capabilities) != 1 { + t.Fatalf("%s probe capabilities = %#v, want one software entry", stage, capabilities) + } + return calls + } + + first := probe("first") + if first == 0 { + t.Fatal("first probe ran no commands") + } + if cached := probe("cached"); cached != first { + t.Fatalf("cached probe ran %d commands, want the cached inventory reused", cached-first) + } + + InvalidateProbeCache() + + if reprobed := probe("re-probed"); reprobed != first*2 { + t.Fatalf("probe after invalidation ran %d commands total, want %d", reprobed, first*2) + } +} diff --git a/internal/tonemap/probe_test.go b/internal/tonemap/probe_test.go index a5f7b2c9f..5fb9b85f6 100644 --- a/internal/tonemap/probe_test.go +++ b/internal/tonemap/probe_test.go @@ -268,10 +268,10 @@ func TestProbeCallerCancellationDoesNotCancelSharedProbe(t *testing.T) { } } -// resetProbeCache clears shared probe state between tests. +// resetProbeCache clears shared probe state between tests. It delegates to the +// exported invalidation so tests exercise the same seam the operator-facing +// re-probe action uses. func resetProbeCache(t *testing.T) { t.Helper() - probeCache.Lock() - probeCache.entries = make(map[string]probeCacheEntry) - probeCache.Unlock() + InvalidateProbeCache() } diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go new file mode 100644 index 000000000..d1e2cce47 --- /dev/null +++ b/internal/transcodenode/reprobe.go @@ -0,0 +1,107 @@ +package transcodenode + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// reprobeCapabilitiesResponse is what an operator-triggered re-probe reports +// back. It is deliberately tiny: the caller that wants the whole inventory +// fetches /hw-capabilities, and after this call that endpoint answers from the +// freshly warmed caches. +type reprobeCapabilitiesResponse struct { + // Resolved is the backend the node would now use. + Resolved string `json:"resolved"` + // CapabilityHash identifies the snapshot this re-probe published, so the + // caller can tell a re-probe that changed something from one that confirmed + // the previous answer. + CapabilityHash string `json:"capability_hash"` +} + +// handleReprobeCapabilities discards this node's cached probe verdicts and +// rebuilds the capability snapshot against live hardware. +// +// Both probe caches keep a *successful* verdict for the process lifetime, which +// is the right default — a GPU that encoded a frame does not stop being able to +// between two playback requests, and re-verifying per request would put ffmpeg +// execs on the playback path. The blind spot that leaves is hardware that has +// stopped working underneath a running node: a driver replaced, a device taken +// out of the container, an ffmpeg swapped in place. No cache key sees any of +// that, so a verdict that was true keeps being served until the node restarts. +// The opposite direction needs no help — a failed GPU probe carries a 15-second +// negative TTL and is retried on its own, so a repaired driver is picked up by +// the next capability snapshot. What this route adds there is the tone-map +// matrix, which caches any non-empty inventory permanently and so can stay +// software-only on a host whose GPU was broken at start. +// +// The rebuild reuses the ordinary snapshot assembly and budget, so a re-probe +// can never cost more than a cold capability fetch already may, and a rebuild +// that does not finish keeps the previously published hash: an incomplete probe +// is not evidence the hardware changed, and republishing a degraded snapshot +// would announce a hardware change that did not happen. +// +// It is refused on a node that is transcoding, for the same reason: every +// hardware probe ends in a real smoke encode that opens an encoder session, and +// a card at its concurrent session cap fails that encode with an error nothing +// can tell apart from a missing device or a broken driver. The verdict would +// then be published as verified:false, and the server would persist a hardware +// regression for a GPU that is fine and is at that moment encoding. Waiting for +// the node to drain costs an operator a retry; a false regression costs them a +// hardware investigation and, through the tone-map inventory in the same +// snapshot, degraded routing until the next clean probe. +func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { + if active := s.activeJobs.Load(); active > 0 { + slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", + "component", "transcodenode", "active_jobs", active) + http.Error(w, fmt.Sprintf( + "node is running %d transcode job(s); a re-probe smoke-encodes on the GPU and a busy encoder would report working hardware as failed. Retry when the node is idle.", + active), http.StatusConflict) + return + } + + playback.InvalidateHWProbeCache() + tonemap.InvalidateProbeCache() + + ctx, cancel := context.WithTimeout(r.Context(), s.capabilityProbeBudget()) + defer cancel() + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + slog.WarnContext(r.Context(), "transcode node capability re-probe incomplete", + "component", "transcodenode", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return + } + previous := s.storedCapabilityHash() + // A re-probed report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) + slog.InfoContext(r.Context(), "transcode node capabilities re-probed", "component", "transcodenode", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(reprobeCapabilitiesResponse{ + Resolved: info.Resolved, + CapabilityHash: info.CapabilityHash, + }); err != nil { + slog.WarnContext(r.Context(), "encode transcode node re-probe result", "component", "transcodenode", "error", err) + } +} + +// capabilityProbeBudget is the deadline one snapshot rebuild gets. It is the +// same budget buildCapabilitySnapshot applies internally, named here so the +// re-probe route is bounded whether or not its caller sent one. +func (s *Server) capabilityProbeBudget() time.Duration { + hwAccel := playback.HWAccelNone + hwDevice := "" + if cfg := s.watcher.Config(); cfg != nil { + hwAccel = cfg.Playback.HWAccel + hwDevice = cfg.Playback.HWDevice + } + return toneMapCapabilityResolveTimeout(hwAccel, hwDevice) +} diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go new file mode 100644 index 000000000..39fdd4db7 --- /dev/null +++ b/internal/transcodenode/reprobe_test.go @@ -0,0 +1,124 @@ +package transcodenode + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +func postReprobe(t *testing.T, server *Server) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+testSecret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + return recorder +} + +// A re-probe must recompute and publish, so health starts advertising the new +// hash immediately — the whole point of the action is that the API stops seeing +// a stale answer without waiting for the 15-minute snapshot tick. +func TestReprobeCapabilitiesRecomputesAndStoresHash(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:stale") + + recorder := postReprobe(t, server) + if recorder.Code == http.StatusServiceUnavailable { + t.Skip("this host's ffmpeg cannot answer a capability probe") + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + + var result reprobeCapabilitiesResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode re-probe result: %v", err) + } + if result.CapabilityHash == "" { + t.Fatal("re-probe reported no capability_hash") + } + if result.CapabilityHash == "sha256:stale" { + t.Fatal("re-probe echoed the stale hash instead of a recomputed one") + } + if result.Resolved == "" { + t.Fatal("re-probe reported no resolved backend") + } + if got := server.storedCapabilityHash(); got != result.CapabilityHash { + t.Fatalf("stored hash = %q, want the re-probed %q", got, result.CapabilityHash) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != result.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the re-probed %q", got, result.CapabilityHash) + } + + // The reported hash must describe the report the capability endpoint would + // now serve, or the API would refetch and store something else. + capabilityRequest := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + capabilityRequest.Header.Set("Authorization", "Bearer "+testSecret) + capabilityRecorder := httptest.NewRecorder() + server.Handler().ServeHTTP(capabilityRecorder, capabilityRequest) + if capabilityRecorder.Code != http.StatusOK { + t.Fatalf("capability status = %d after a re-probe", capabilityRecorder.Code) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(capabilityRecorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash != result.CapabilityHash { + t.Fatalf("served capability_hash = %q, want the re-probed %q", info.CapabilityHash, result.CapabilityHash) + } +} + +// An incomplete probe is not evidence the hardware changed, so a degraded +// re-probe must answer 503 and leave the previously published hash alone — +// publishing a partial report would announce a hardware change that did not +// happen and make the API store it. +func TestReprobeCapabilitiesKeepsHashOnProbeFailure(t *testing.T) { + server, _ := newCapabilityTestServer(t) + server.storeCapabilityHash("sha256:previous") + + if got := postReprobe(t, server).Code; got != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 for a probe that cannot complete", got) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous hash kept", got) + } +} + +// Every hardware probe ends in a real smoke encode that opens an encoder +// session. On a card at its concurrent session cap that encode fails with an +// error indistinguishable from a missing device, and the verdict would be +// published as verified:false — a hardware regression the server then persists +// and warns on, for a GPU that is fine and is at that moment encoding. So a busy +// node refuses, and keeps the report it has. +func TestReprobeCapabilitiesRefusesWhileTranscoding(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + server.activeJobs.Store(2) + t.Cleanup(func() { server.activeJobs.Store(0) }) + + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while the node is transcoding", recorder.Code) + } + if body := recorder.Body.String(); !strings.Contains(body, "idle") { + t.Fatalf("body = %q, want it to tell the operator when to retry", body) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// The route is bearer-authed like the rest of the admin group: it executes +// ffmpeg, so an unauthenticated caller could otherwise make a node do work. +func TestReprobeCapabilitiesRequiresBearer(t *testing.T) { + server := newTestServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil)) + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 86a572ab2..db7ffd538 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -605,6 +605,7 @@ func (s *Server) Handler() http.Handler { r.Get("/transcode/{session_id}/master.m3u8", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/master.m3u8", s.handleManifest)) r.Get("/transcode/{session_id}/segment/{name}", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/segment/{name}", s.handleSegment)) r.Post("/admin/force-reload", s.handleForceReload) + r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) }) return r diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt index d8f6a798c..a3b0b829e 100644 --- a/internal/transcodenode/testdata/media_routes.txt +++ b/internal/transcodenode/testdata/media_routes.txt @@ -1,5 +1,6 @@ # fixture 1 POST /admin/force-reload non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media @@ -15,6 +16,7 @@ GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false true GET /transcode/{session_id}/segment/{name} media playback internal_relay false true # fixture 2 POST /admin/force-reload non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media diff --git a/migrations/sql/20260827125333_node_capability_drift.sql b/migrations/sql/20260827125333_node_capability_drift.sql new file mode 100644 index 000000000..dbd90fe21 --- /dev/null +++ b/migrations/sql/20260827125333_node_capability_drift.sql @@ -0,0 +1,25 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capability_drift text; + +COMMENT ON COLUMN stream_nodes.capability_drift IS + 'Human-readable note describing how this node''s hardware got worse, set ' + 'when a capability refetch shows a previously verified backend now failing ' + 'its probe or a render device that has disappeared. It stays set until a ' + 'refetch produces a report whose probes all pass: a refetch that merely ' + 'loses nothing further leaves it alone, because a comparison against an ' + 'already-degraded report always finds nothing and would report a still- ' + 'broken node as repaired. NULL therefore means no standing regression. It ' + 'exists because that regression is otherwise only a log line: a driver that ' + 'stopped working turns a node from a GPU transcoder into a silent CPU one, ' + 'and the node stays healthy throughout. Written only by the node health ' + 'sweep, in the same statement as capabilities and capabilities_hash, so it ' + 'always describes the report stored beside it. Not a routing input.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capability_drift; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index c20a735b0..045e80f10 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3843,6 +3843,13 @@ export interface HostDiskStats { stale?: boolean; /** Never measured on this host: `used_gb`/`total_gb` are meaningless. */ unavailable?: boolean; + /** + * The node's transcode working directory — the one mount whose filling up + * breaks transcoding rather than browsing. Set on at most one entry per + * sample; a media root sharing that volume is deduplicated onto it. Absent on + * every other mount, and on a node predating the flag. + */ + scratch?: boolean; } /** @@ -3944,6 +3951,14 @@ export interface StreamNode { hw_accel_override?: string | null; /** Comma-separated render device paths pinned to this node; null inherits. */ hw_device_override?: string | null; + /** + * Human-readable note describing how this node's hardware got worse at the + * last capability refetch: a backend that used to pass its probe and now + * fails, or a render device that is gone. Absent means the last refetch found + * no regression — it is not a latched incident, and a repaired node loses the + * note on its next refetch. Nothing routes on it. + */ + capability_drift?: string | null; } export interface CreateNodeRequest { @@ -3975,6 +3990,27 @@ export interface CheckNodeResponse { egress_kbps: number; } +/** + * Answer to POST /admin/nodes/{id}/reprobe. The call is always 200: a node that + * refused or could not be reached is reported as `status: "error"` here rather + * than as an HTTP status, matching the per-node check route. + */ +export interface ReprobeNodeResult { + node_id: number; + node_name: string; + status: "ok" | "error"; + error?: string; + /** Backend the node picked after re-probing: nvenc, qsv, vaapi, or none. */ + resolved?: string; + /** Hash of the snapshot the node published; compare against `capabilities_hash`. */ + capability_hash?: string; + /** + * This server also refetched and stored the node's new inventory before + * answering. False means the stored row catches up on a later health sweep. + */ + capabilities_refreshed: boolean; +} + // User-facing library (simplified, no admin fields) export interface UserLibrary { id: number; diff --git a/web/src/hooks/queries/admin/nodes.ts b/web/src/hooks/queries/admin/nodes.ts index 25c861ac6..523a45e4a 100644 --- a/web/src/hooks/queries/admin/nodes.ts +++ b/web/src/hooks/queries/admin/nodes.ts @@ -5,8 +5,10 @@ import type { CreateNodeRequest, UpdateNodeRequest, CheckNodeResponse, + ReprobeNodeResult, } from "@/api/types"; import { adminKeys } from "../keys"; +import { describeReprobeOutcome } from "@/pages/adminNodesPresentation"; import { toast } from "sonner"; const ADMIN_STALE_TIME = 30_000; @@ -88,6 +90,45 @@ export function useCheckNodeHealth() { }); } +/** + * Ask one node to re-verify its hardware against live devices. + * + * The call always answers 200 — a node that refused or could not be reached is + * reported in the body — so the outcome is read from `status`, not from a + * thrown error. It can take a couple of minutes on a node with several devices, + * since the point is to pay the full cold probe cost the node otherwise caches + * away for its process lifetime; the server extends the connection's write + * deadline to cover that, so a long wait here is the action working, not a hung + * request. A node that is transcoding refuses, because the probe encodes on the + * GPU and a busy encoder would report working hardware as failed. + * + * The nodes list is invalidated either way: on success the server has already + * stored the fresh report, and on failure the row may still have moved. + */ +export function useReprobeNode() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (node: StreamNode) => + api(`/admin/nodes/${node.id}/reprobe`, { + method: "POST", + }).then((result) => ({ node, result })), + onSuccess: ({ node, result }) => { + const outcome = describeReprobeOutcome(node, result); + if (outcome.ok) { + toast.success(outcome.message); + } else { + toast.error(outcome.message); + } + }, + onError: (err) => { + toast.error(err instanceof Error ? err.message : "Re-probe failed"); + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: adminKeys.nodes() }); + }, + }); +} + export function useToggleNode() { const queryClient = useQueryClient(); return useMutation({ diff --git a/web/src/lib/hwDevices.ts b/web/src/lib/hwDevices.ts new file mode 100644 index 000000000..30a784a31 --- /dev/null +++ b/web/src/lib/hwDevices.ts @@ -0,0 +1,39 @@ +// Shared parsing and toggling for the comma-separated render-device lists the +// server stores: the cluster-wide `playback.hw_device` setting and a node's +// `hw_device_override`. Both are edited as a per-device picker, and both must +// round-trip a path the current inventory does not list, so the rules live here +// rather than once per page. Row building stays with each page: the cluster +// picker rows carry per-node presence, a node's rows carry its own inventory. + +/** Split a stored comma-separated device list into its paths. */ +export function parseHWDeviceList(value: string | null | undefined): string[] { + if (!value) return []; + return value + .split(",") + .map((part) => part.trim()) + .filter((part) => part.length > 0); +} + +/** + * Toggles one device in the stored list, preserving the order devices are + * detected in so the stored value stays stable regardless of click order. + */ +export function toggleHWDevice( + value: string | null | undefined, + device: string, + detectedOrder: readonly string[], +): string { + const selected = new Set(parseHWDeviceList(value)); + if (selected.has(device)) { + selected.delete(device); + } else { + selected.add(device); + } + const ordered = detectedOrder.filter((path) => selected.has(path)); + // Preserve selected devices the current detection pass doesn't list (e.g. + // a temporarily unplugged GPU) rather than silently dropping them. + for (const path of selected) { + if (!detectedOrder.includes(path)) ordered.push(path); + } + return ordered.join(","); +} diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 7eac5ca95..be8e1c7fb 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -7,6 +7,7 @@ import { useUpdateNode, useDeleteNode, useCheckNodeHealth, + useReprobeNode, useToggleNode, } from "@/hooks/queries/admin/nodes"; import { Button } from "@/components/ui/button"; @@ -30,18 +31,23 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { Plus, Pencil, Trash2, RefreshCw, Info, AlertTriangle } from "lucide-react"; +import { Plus, Pencil, Trash2, RefreshCw, ScanSearch, Info, AlertTriangle } from "lucide-react"; import { ConfirmDialog } from "@/components/ConfirmDialog"; import { formatDateTime } from "@/lib/datetime"; +import { toggleHWDevice } from "@/lib/hwDevices"; import { cn } from "@/lib/utils"; -import type { ResourceMetric } from "./adminNodesPresentation"; +import type { NodeHWDeviceRow, ResourceMetric } from "./adminNodesPresentation"; import { HW_ACCEL_INHERIT, HW_ACCEL_OVERRIDE_OPTIONS, + buildNodeHWDeviceRows, + describeCapabilityDrift, describeNodeAccelerationOverride, describeNodeGPU, describeNodeSystem, describeSharedGPU, + nodeHWDevicePaths, + nodeHasHWDeviceInventory, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -127,14 +133,38 @@ function NodeSharedGPUBadge({ node, allNodes }: { node: StreamNode; allNodes: St ); } +/** + * The "Drift" marker on a node whose last capability refetch found its hardware + * got worse. Tinted, unlike the shared-GPU marker: this one is a regression an + * operator has to act on, and the Health column will not show it. + */ +function NodeDriftBadge({ node }: { node: StreamNode }) { + const drift = describeCapabilityDrift(node); + if (!drift) { + return null; + } + return ( + + {drift.label} + + ); +} + function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNode[] }) { const gpu = describeNodeGPU(node); if (gpu.kind === "awaiting") { return (
- - {gpu.label} - +
+ + {gpu.label} + + +
); @@ -146,6 +176,7 @@ function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNod {gpu.backend.label} + {gpu.failures.length > 0 && ( void; onCheckHealth: (node: StreamNode) => void; checkingHealthId: number | null; + onReprobe: (node: StreamNode) => void; + reprobingId: number | null; } function NodeSection({ @@ -224,6 +257,8 @@ function NodeSection({ onToggle, onCheckHealth, checkingHealthId, + onReprobe, + reprobingId, }: NodeSectionProps) { const label = type === "proxy" ? "Proxy" : "Transcode"; const colCount = (showJobs ? 10 : 9) + (type === "proxy" ? 1 : 0); @@ -256,7 +291,7 @@ function NodeSection({ {showJobs && {type === "proxy" ? "Streams" : "Jobs"}} {type === "proxy" && Egress} Last Check - Actions + Actions @@ -278,6 +313,7 @@ function NodeSection({ ) : ( nodes.map((node) => { const isChecking = checkingHealthId === node.id; + const isReprobing = reprobingId === node.id; return ( {node.name} @@ -352,6 +388,20 @@ function NodeSection({ aria-hidden="true" /> + + )} +
+
+ ); +} + function NodeForm({ node, nodeType, @@ -403,6 +510,12 @@ function NodeForm({ node?.hw_accel_override?.trim() || HW_ACCEL_INHERIT, ); const [hwDeviceOverride, setHwDeviceOverride] = useState(node?.hw_device_override ?? ""); + // The picker is driven by the node's own reported inventory; a node that has + // never reported one keeps the free-text field, since the override still has + // to be settable on a node this server has not heard from yet. + const hasDeviceInventory = nodeHasHWDeviceInventory(node); + const deviceRows = buildNodeHWDeviceRows(node, hwDeviceOverride); + const devicePaths = nodeHWDevicePaths(node); const createMutation = useCreateNode(); const updateMutation = useUpdateNode(); const isPending = createMutation.isPending || updateMutation.isPending; @@ -551,18 +664,33 @@ function NodeForm({
- - setHwDeviceOverride(e.target.value)} - placeholder="Inherit cluster setting" - /> -

- Optional. Comma-separated render device paths this node transcodes on (e.g.{" "} - /dev/dri/renderD128,/dev/dri/renderD129). Leave - empty to inherit the cluster-wide device selection. -

+ + {hasDeviceInventory ? ( + + setHwDeviceOverride(toggleHWDevice(hwDeviceOverride, path, devicePaths)) + } + onInherit={() => setHwDeviceOverride("")} + /> + ) : ( + <> + setHwDeviceOverride(e.target.value)} + placeholder="Inherit cluster setting" + /> +

+ Optional. Comma-separated render device paths this node transcodes on (e.g.{" "} + /dev/dri/renderD128,/dev/dri/renderD129). Leave + empty to inherit the cluster-wide device selection. This node has reported no + device inventory yet, so there is nothing to pick from. +

+ + )}
)} @@ -582,6 +710,7 @@ export default function AdminNodes() { const [confirmDeleteNode, setConfirmDeleteNode] = useState(null); const deleteMutation = useDeleteNode(); const checkHealthMutation = useCheckNodeHealth(); + const reprobeMutation = useReprobeNode(); const toggleMutation = useToggleNode(); const proxyNodes = nodes.filter((n) => n.type === "proxy"); @@ -592,6 +721,9 @@ export default function AdminNodes() { ? checkHealthMutation.variables.id : null; + const reprobingId = + reprobeMutation.isPending && reprobeMutation.variables ? reprobeMutation.variables.id : null; + const resolvedNodeType: NodeType = editingNode ? (editingNode.type as NodeType) : (addingNodeType ?? "proxy"); @@ -660,6 +792,8 @@ export default function AdminNodes() { onToggle={(node) => toggleMutation.mutate(node)} onCheckHealth={(node) => checkHealthMutation.mutate(node)} checkingHealthId={checkingHealthId} + onReprobe={(node) => reprobeMutation.mutate(node)} + reprobingId={reprobingId} infoBanner={
@@ -679,6 +813,8 @@ export default function AdminNodes() { onToggle={(node) => toggleMutation.mutate(node)} onCheckHealth={(node) => checkHealthMutation.mutate(node)} checkingHealthId={checkingHealthId} + onReprobe={(node) => reprobeMutation.mutate(node)} + reprobingId={reprobingId} infoBanner={
diff --git a/web/src/pages/admin-settings/playbackSettings.utils.ts b/web/src/pages/admin-settings/playbackSettings.utils.ts index c7c72e414..75b590a87 100644 --- a/web/src/pages/admin-settings/playbackSettings.utils.ts +++ b/web/src/pages/admin-settings/playbackSettings.utils.ts @@ -5,38 +5,14 @@ import type { HWAccelInfo } from "@/hooks/queries/admin/system"; // toggles, with no selection meaning "auto" (server picks the first // available device). The setting is cluster-wide, so rows carry per-node // presence info when transcode nodes report their inventories. +// +// Parsing and toggling that list is the same problem as editing one node's +// hw_device_override, so both live in @/lib/hwDevices and are re-exported here +// for the callers (and tests) that already know them by these names. -export function parseHWDeviceList(value: string | undefined): string[] { - if (!value) return []; - return value - .split(",") - .map((part) => part.trim()) - .filter((part) => part.length > 0); -} +import { parseHWDeviceList, toggleHWDevice } from "@/lib/hwDevices"; -/** - * Toggles one device in the stored list, preserving the order devices are - * detected in so the stored value stays stable regardless of click order. - */ -export function toggleHWDevice( - value: string | undefined, - device: string, - detectedOrder: string[], -): string { - const selected = new Set(parseHWDeviceList(value)); - if (selected.has(device)) { - selected.delete(device); - } else { - selected.add(device); - } - const ordered = detectedOrder.filter((path) => selected.has(path)); - // Preserve selected devices the current detection pass doesn't list (e.g. - // a temporarily unplugged GPU) rather than silently dropping them. - for (const path of selected) { - if (!detectedOrder.includes(path)) ordered.push(path); - } - return ordered.join(","); -} +export { parseHWDeviceList, toggleHWDevice }; export interface HWDeviceRow { path: string; diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index 09f2315c6..f887dfa37 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -1,15 +1,20 @@ import { describe, expect, it } from "vitest"; -import type { HostSystemStats, StreamNode } from "@/api/types"; +import type { HostSystemStats, ReprobeNodeResult, StreamNode } from "@/api/types"; import { CAPABILITY_STALE_AFTER_MS, DISK_FILL_WARNING_PCT, + buildNodeHWDeviceRows, + describeCapabilityDrift, describeGPUBusy, describeNodeAccelerationOverride, describeNodeGPU, describeNodeSystem, + describeReprobeOutcome, describeResourceSample, describeSharedGPU, formatBitsPerSecond, + nodeHWDevicePaths, + nodeHasHWDeviceInventory, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -821,6 +826,213 @@ describe("describeNodeAccelerationOverride", () => { }); }); +describe("describeCapabilityDrift", () => { + it("renders nothing for a node whose last refetch found no regression", () => { + expect(describeCapabilityDrift(makeNode())).toBeNull(); + expect(describeCapabilityDrift(makeNode({ capability_drift: null }))).toBeNull(); + expect(describeCapabilityDrift(makeNode({ capability_drift: " " }))).toBeNull(); + }); + + // A server predating the column sends no field at all, which must read as + // "no drift" rather than as an empty badge. + it("renders nothing for a server that predates the field", () => { + const olderServerNode = makeNode(); + expect("capability_drift" in olderServerNode).toBe(false); + expect(describeCapabilityDrift(olderServerNode)).toBeNull(); + }); + + it("shows the server's note verbatim and explains how it clears", () => { + const note = "verified hardware backends lost: qsv; resolved backend qsv -> none"; + const drift = describeCapabilityDrift(makeNode({ capability_drift: note })); + + expect(drift?.label).toBe("Drift"); + expect(drift?.title.split("\n")[0]).toBe(note); + expect(drift?.title).toContain("got worse than the report it replaced"); + expect(drift?.title).toContain("re-probe the node"); + }); + + it("trims the stored note rather than rendering its whitespace", () => { + expect( + describeCapabilityDrift( + makeNode({ capability_drift: " render devices gone: /dev/dri/renderD128 " }), + )?.title.split("\n")[0], + ).toBe("render devices gone: /dev/dri/renderD128"); + }); +}); + +describe("buildNodeHWDeviceRows", () => { + const inventoryNode = makeNode({ + capabilities: { + resolved: "qsv", + render_device_details: [ + { path: "/dev/dri/renderD128", description: "Intel GPU", pci_address: "0000:00:02.0" }, + { path: "/dev/dri/renderD129", description: "Intel GPU" }, + ], + }, + }); + + it("has nothing to pick from on a node that never reported an inventory", () => { + expect(nodeHasHWDeviceInventory(makeNode())).toBe(false); + expect(nodeHasHWDeviceInventory(makeNode({ capabilities: { resolved: "qsv" } }))).toBe(false); + expect(nodeHasHWDeviceInventory(null)).toBe(false); + expect(buildNodeHWDeviceRows(makeNode(), "")).toEqual([]); + }); + + it("builds one row per reported device, in report order, none selected by default", () => { + expect(nodeHasHWDeviceInventory(inventoryNode)).toBe(true); + expect(nodeHWDevicePaths(inventoryNode)).toEqual([ + "/dev/dri/renderD128", + "/dev/dri/renderD129", + ]); + expect(buildNodeHWDeviceRows(inventoryNode, null)).toEqual([ + { + path: "/dev/dri/renderD128", + description: "Intel GPU", + reported: true, + selected: false, + title: "/dev/dri/renderD128 — Intel GPU (0000:00:02.0)", + }, + { + path: "/dev/dri/renderD129", + description: "Intel GPU", + reported: true, + selected: false, + title: "/dev/dri/renderD129 — Intel GPU", + }, + ]); + }); + + it("checks exactly the devices the override pins", () => { + const rows = buildNodeHWDeviceRows(inventoryNode, " /dev/dri/renderD129 "); + + expect(rows.map((row) => row.selected)).toEqual([false, true]); + }); + + it("falls back to bare paths from a node that reports no device details", () => { + const rows = buildNodeHWDeviceRows( + makeNode({ + capabilities: { resolved: "vaapi", render_devices: ["/dev/dri/renderD128", " "] }, + }), + "/dev/dri/renderD128", + ); + + expect(rows).toEqual([ + { + path: "/dev/dri/renderD128", + description: "GPU", + reported: true, + selected: true, + title: "/dev/dri/renderD128", + }, + ]); + }); + + // A pinned device the node stopped reporting would otherwise be stranded: + // checked in the stored value with no control to clear it. + it("keeps a pinned device the node no longer reports, and marks it unreported", () => { + const rows = buildNodeHWDeviceRows(inventoryNode, "/dev/dri/renderD129,/dev/dri/renderD200"); + + expect(rows).toHaveLength(3); + expect(rows[2]).toMatchObject({ + path: "/dev/dri/renderD200", + reported: false, + selected: true, + description: "Pinned device this node does not report", + }); + expect(rows[2]?.title).toContain("not in this node's last capability report"); + }); +}); + +describe("describeReprobeOutcome", () => { + function result(overrides: Partial = {}): ReprobeNodeResult { + return { + node_id: 1, + node_name: "transcode-1", + status: "ok", + capabilities_refreshed: true, + ...overrides, + }; + } + + it("surfaces the node's own reason when the re-probe failed", () => { + expect( + describeReprobeOutcome( + makeNode(), + result({ status: "error", error: "node could not complete its hardware probe" }), + ), + ).toEqual({ + ok: false, + message: "transcode-1: re-probe failed — node could not complete its hardware probe", + }); + }); + + it("still says which node failed when the server sent no reason", () => { + expect(describeReprobeOutcome(makeNode(), result({ status: "error" }))).toEqual({ + ok: false, + message: "transcode-1: re-probe failed — the node reported no reason", + }); + }); + + it("reports an unchanged hash as plainly as a change", () => { + const node = makeNode({ capabilities_hash: "sha256:aaa" }); + + expect( + describeReprobeOutcome(node, result({ capability_hash: "sha256:aaa", resolved: "qsv" })), + ).toEqual({ ok: true, message: "transcode-1: re-probed, no change (QSV)" }); + }); + + it("calls out a changed report and the backend it resolved to", () => { + const node = makeNode({ capabilities_hash: "sha256:aaa" }); + + expect( + describeReprobeOutcome(node, result({ capability_hash: "sha256:bbb", resolved: "vaapi" })), + ).toEqual({ ok: true, message: "transcode-1: re-probed, hardware report changed — now VAAPI" }); + }); + + it("names a software fallback rather than the wire value", () => { + expect( + describeReprobeOutcome( + makeNode({ capabilities_hash: "sha256:aaa" }), + result({ capability_hash: "sha256:bbb", resolved: "none" }), + ).message, + ).toBe("transcode-1: re-probed, hardware report changed — now software"); + }); + + // Without a hash on both sides there is nothing to compare, and claiming + // either answer would be a guess. + it("leaves the comparison unstated when either hash is missing", () => { + expect(describeReprobeOutcome(makeNode(), result({ resolved: "qsv" })).message).toBe( + "transcode-1: re-probed — QSV", + ); + expect( + describeReprobeOutcome(makeNode({ capabilities_hash: "sha256:aaa" }), result()).message, + ).toBe("transcode-1: re-probed"); + }); + + it("says when the stored row has not caught up yet", () => { + expect( + describeReprobeOutcome( + makeNode({ capabilities_hash: "sha256:aaa" }), + result({ capability_hash: "sha256:aaa", capabilities_refreshed: false }), + ), + ).toEqual({ + ok: true, + message: + "transcode-1: re-probed, no change. The stored report will catch up on the next health check", + }); + }); + + it("prefers the name the server answered with, and falls back to the row's", () => { + expect( + describeReprobeOutcome(makeNode({ name: "stale-name" }), result({ node_name: "renamed" })) + .message, + ).toContain("renamed:"); + expect( + describeReprobeOutcome(makeNode({ name: "row-name" }), result({ node_name: " " })).message, + ).toContain("row-name:"); + }); +}); + describe("parseHWDeviceOverride", () => { it("splits, trims, and drops empty entries", () => { expect(parseHWDeviceOverride(" /dev/dri/renderD128 ,, /dev/dri/renderD129,")).toEqual([ diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index eaffec305..7711cc1fc 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -4,9 +4,11 @@ import type { HostSystemStats, NodeCapabilities, NodeRenderDevice, + ReprobeNodeResult, StreamNode, SystemResources, } from "@/api/types"; +import { parseHWDeviceList } from "@/lib/hwDevices"; import { formatFileSize } from "@/lib/mediaFormat"; /** @@ -388,6 +390,185 @@ export function describeSharedGPU( }; } +// --- Capability drift ------------------------------------------------------ + +/** The "Drift" marker on a node whose hardware got worse at its last refetch. */ +export interface CapabilityDriftBadge { + label: string; + /** Hover text: the server's note, then what it means and how it clears. */ + title: string; +} + +/** + * Describe a node's capability drift, or null when the node carries no standing + * regression — which is the normal case and renders nothing, as it does on + * every server predating the field. + * + * The note is a warning, not a routing input: a node carrying one is still + * picked for work exactly as before. It is here because the regression it + * describes is invisible in the Health column — a node whose GPU driver broke + * answers health checks perfectly well while encoding in software. + */ +export function describeCapabilityDrift(node: StreamNode): CapabilityDriftBadge | null { + const note = node.capability_drift?.trim() ?? ""; + if (note === "") { + return null; + } + return { + label: "Drift", + title: [ + note, + "A capability refetch found this node's hardware got worse than the report it replaced.", + "It stays until a refetch finds every hardware probe passing — re-probe the node to check whether it is still true.", + ].join("\n"), + }; +} + +// --- Per-node device-override picker --------------------------------------- + +/** One render device in a node's own device-override picker. */ +export interface NodeHWDeviceRow { + /** Render node path, which is what the override stores. */ + path: string; + /** The node's own label for the device, or why the path is not in its report. */ + description: string; + /** Present in the node's stored capability report. */ + reported: boolean; + /** Pinned by the override currently being edited. */ + selected: boolean; + /** Hover text: the device's full identity as the node reported it. */ + title: string; +} + +interface ReportedDevice { + path: string; + description: string; + title: string; +} + +/** + * The devices a node reported, in report order. Details win when the node sent + * them; a report carrying only paths (an older node) still yields rows, because + * the paths are what the override stores and the descriptions are decoration. + */ +function reportedDevices(node: StreamNode | null | undefined): ReportedDevice[] { + const details = (node?.capabilities?.render_device_details ?? []).filter( + (device) => (device.path?.trim() ?? "") !== "", + ); + if (details.length > 0) { + return details.map((device) => ({ + path: device.path.trim(), + description: deviceLabel(device), + title: describeDeviceLine(device), + })); + } + return (node?.capabilities?.render_devices ?? []) + .map((path) => path.trim()) + .filter((path) => path !== "") + .map((path) => ({ path, description: "GPU", title: path })); +} + +/** + * Whether this node has an inventory to pick from. False sends the editor back + * to a free-text field: a node that has never reported (or a server predating + * the inventory) must still be pinnable, and a picker with no rows would make + * the override look unavailable rather than unknown. + */ +export function nodeHasHWDeviceInventory(node: StreamNode | null | undefined): boolean { + return reportedDevices(node).length > 0; +} + +/** Paths in report order — the order `toggleHWDevice` keeps the stored value in. */ +export function nodeHWDevicePaths(node: StreamNode | null | undefined): string[] { + return reportedDevices(node).map((device) => device.path); +} + +/** + * Build the picker rows: the node's inventory, then any pinned path it does not + * report. The second group is what keeps a stale override deselectable instead + * of stranded — a device removed from the node would otherwise stay pinned with + * no control to clear it. + */ +export function buildNodeHWDeviceRows( + node: StreamNode | null | undefined, + override: string | null | undefined, +): NodeHWDeviceRow[] { + const selected = parseHWDeviceOverride(override); + const rows: NodeHWDeviceRow[] = reportedDevices(node).map((device) => ({ + ...device, + reported: true, + selected: selected.includes(device.path), + })); + for (const path of selected) { + if (rows.some((row) => row.path === path)) { + continue; + } + rows.push({ + path, + description: "Pinned device this node does not report", + reported: false, + selected: true, + title: `${path} — not in this node's last capability report. A transcode pinned to a device the node cannot open falls back to software.`, + }); + } + return rows; +} + +// --- Re-probe action ------------------------------------------------------- + +/** What to tell an operator once a re-probe returns. */ +export interface ReprobeOutcome { + /** The node re-probed. False covers both a refusal and an unreachable node. */ + ok: boolean; + message: string; +} + +/** + * Describe a re-probe result against the node as it was listed before the call. + * + * Comparing the returned hash against the stored one is the only thing that + * answers the question an operator actually asked — did the driver work land? — + * so "no change" is reported as plainly as a change. An absent hash on either + * side (an older node, or a node that never stored a report) leaves the + * comparison unstated rather than guessed at. + */ +export function describeReprobeOutcome( + node: StreamNode, + result: ReprobeNodeResult, +): ReprobeOutcome { + const name = result.node_name?.trim() || node.name; + if (result.status !== "ok") { + const reason = result.error?.trim() || "the node reported no reason"; + return { ok: false, message: `${name}: re-probe failed — ${reason}` }; + } + + const backend = reprobeBackendLabel(result.resolved); + const hash = result.capability_hash?.trim() ?? ""; + const previous = node.capabilities_hash?.trim() ?? ""; + let message: string; + if (hash !== "" && previous !== "" && hash === previous) { + message = `${name}: re-probed, no change${backend ? ` (${backend})` : ""}`; + } else if (hash !== "" && previous !== "") { + message = `${name}: re-probed, hardware report changed${backend ? ` — now ${backend}` : ""}`; + } else { + message = `${name}: re-probed${backend ? ` — ${backend}` : ""}`; + } + // The node has recomputed either way; only the stored row lags, and saying so + // stops an operator re-running the action to explain an unchanged table. + if (!result.capabilities_refreshed) { + message += ". The stored report will catch up on the next health check"; + } + return { ok: true, message }; +} + +function reprobeBackendLabel(resolved: string | undefined): string { + const backend = resolved?.trim().toLowerCase() ?? ""; + if (backend === "") { + return ""; + } + return backend === "none" ? "software" : backend.toUpperCase(); +} + // --- Per-node acceleration overrides --------------------------------------- /** @@ -419,12 +600,12 @@ export interface NodeAccelerationOverride { title: string; } -/** Split a stored comma-separated device override into its paths. */ +/** + * Split a stored comma-separated device override into its paths. Same rules as + * the cluster-wide picker's list, which is why they share an implementation. + */ export function parseHWDeviceOverride(value: string | null | undefined): string[] { - return (value ?? "") - .split(",") - .map((part) => part.trim()) - .filter((part) => part !== ""); + return parseHWDeviceList(value); } /** From 189d0509774a87564ed49d05300c7f514d3c98dc Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:38:22 -0400 Subject: [PATCH 010/163] fix(web): hide acceleration fields on proxy nodes, clarify auto-detect default Proxy nodes only remux and strip bitstreams, so the edit dialog no longer offers Hardware Acceleration or GPU Devices for them (and a proxy edit no longer sends override fields at all, so values set via the API are not silently cleared). The inherit option is relabeled "Cluster default" with copy stating plainly that the default is auto and auto detects this node's own hardware, plus a muted line showing what the node currently resolves to from its stored capability report. Related issue: #780 Co-Authored-By: Claude Fable 5 --- web/src/pages/AdminNodes.tsx | 46 ++++++++------ web/src/pages/adminNodesPresentation.test.ts | 63 ++++++++++++++++++++ web/src/pages/adminNodesPresentation.ts | 45 +++++++++++++- 3 files changed, 135 insertions(+), 19 deletions(-) diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index be8e1c7fb..2c969538d 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -42,6 +42,7 @@ import { HW_ACCEL_OVERRIDE_OPTIONS, buildNodeHWDeviceRows, describeCapabilityDrift, + describeEffectiveAcceleration, describeNodeAccelerationOverride, describeNodeGPU, describeNodeSystem, @@ -475,14 +476,14 @@ function NodeDevicePicker({

{selectedCount === 0 - ? "Inheriting the cluster-wide device selection." + ? "Using the cluster default (auto-discover this node's devices)." : selectedCount === 1 ? "All transcodes on this node run on the selected device." : "Transcodes on this node balance across the selected devices (least loaded first)."}

{selectedCount > 0 && ( )}
@@ -516,6 +517,7 @@ function NodeForm({ const hasDeviceInventory = nodeHasHWDeviceInventory(node); const deviceRows = buildNodeHWDeviceRows(node, hwDeviceOverride); const devicePaths = nodeHWDevicePaths(node); + const effectiveAcceleration = node ? describeEffectiveAcceleration(node) : null; const createMutation = useCreateNode(); const updateMutation = useUpdateNode(); const isPending = createMutation.isPending || updateMutation.isPending; @@ -541,13 +543,15 @@ function NodeForm({ if (node) { // null on either override is what restores inheritance of the // cluster-wide playback setting; omitting the key would leave the stored - // value alone instead. - const overrideDevices = parseHWDeviceOverride(hwDeviceOverride); - const body: UpdateNodeRequest = { - ...fields, - hw_accel_override: hwAccelOverride === HW_ACCEL_INHERIT ? null : hwAccelOverride, - hw_device_override: overrideDevices.length > 0 ? overrideDevices.join(",") : null, - }; + // value alone instead. The override controls only render for transcode + // nodes, so a proxy edit must omit both keys rather than send null — + // sending null here would clear an existing value the form never showed. + const body: UpdateNodeRequest = { ...fields }; + if (nodeType === "transcode") { + const overrideDevices = parseHWDeviceOverride(hwDeviceOverride); + body.hw_accel_override = hwAccelOverride === HW_ACCEL_INHERIT ? null : hwAccelOverride; + body.hw_device_override = overrideDevices.length > 0 ? overrideDevices.join(",") : null; + } updateMutation.mutate({ id: node.id, body }, { onSuccess: onClose }); } else { const body: CreateNodeRequest = { type: nodeType, ...fields }; @@ -638,8 +642,12 @@ function NodeForm({ )} {/* Overrides are edit-only: the create endpoint takes no acceleration - fields, so offering them here would silently drop what was typed. */} - {node && ( + fields, so offering them here would silently drop what was typed. + They are also transcode-only: a proxy node only remuxes/strips + bitstreams, so it never encodes and these fields would be + meaningless — and their absence keeps a proxy edit from sending + override fields at all. */} + {node && nodeType === "transcode" && ( <>
@@ -657,10 +665,14 @@ function NodeForm({

Optional. Overrides the cluster-wide Hardware Acceleration setting for this node only - — use it when this node's hardware differs from the rest of the cluster. Applies to - new transcodes within a minute; restart the node to re-prime its encoder for the new - backend. + — use it when this node's hardware differs from the rest of the cluster. The cluster + default is Auto unless changed on the Playback settings page, and Auto detects this + node's own hardware, not the server's. Applies to new transcodes within a minute; + restart the node to re-prime its encoder for the new backend.

+ {effectiveAcceleration && ( +

{effectiveAcceleration}

+ )}
@@ -681,13 +693,13 @@ function NodeForm({ id="node-hw-device-override" value={hwDeviceOverride} onChange={(e) => setHwDeviceOverride(e.target.value)} - placeholder="Inherit cluster setting" + placeholder="Cluster default (auto-discover)" />

Optional. Comma-separated render device paths this node transcodes on (e.g.{" "} /dev/dri/renderD128,/dev/dri/renderD129). Leave - empty to inherit the cluster-wide device selection. This node has reported no - device inventory yet, so there is nothing to pick from. + empty to use the cluster default (auto-discover). This node has reported no device + inventory yet, so there is nothing to pick from.

)} diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index f887dfa37..83e3d0fcf 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -5,6 +5,7 @@ import { DISK_FILL_WARNING_PCT, buildNodeHWDeviceRows, describeCapabilityDrift, + describeEffectiveAcceleration, describeGPUBusy, describeNodeAccelerationOverride, describeNodeGPU, @@ -487,6 +488,68 @@ describe("describeNodeGPU", () => { }); }); +describe("describeEffectiveAcceleration", () => { + it("has nothing to say about a node with no stored capabilities", () => { + expect(describeEffectiveAcceleration(makeNode())).toBeNull(); + expect(describeEffectiveAcceleration(makeNode({ capabilities: null }))).toBeNull(); + }); + + it("names the device a verified backend resolved on", () => { + const node = makeNode({ + capabilities: { + resolved: "qsv", + detected_backends: [{ backend: "qsv", verified: true, device: "/dev/dri/renderD128" }], + }, + }); + + expect(describeEffectiveAcceleration(node)).toBe( + "Currently resolves: QSV — verified on /dev/dri/renderD128", + ); + }); + + it("omits the device for a verified backend with no render node, like NVENC", () => { + const node = makeNode({ + capabilities: { + resolved: "nvenc", + detected_backends: [{ backend: "nvenc", verified: true }], + }, + }); + + expect(describeEffectiveAcceleration(node)).toBe("Currently resolves: NVENC — verified"); + }); + + it("says a failed probe failed, without repeating the reason", () => { + const node = makeNode({ + capabilities: { + resolved: "qsv", + detected_backends: [ + { backend: "qsv", verified: false, reason: "h264_qsv smoke encode failed: device busy" }, + ], + }, + }); + + expect(describeEffectiveAcceleration(node)).toBe("Currently resolves: QSV — probe failed"); + }); + + it("calls a configured backend with no probe entry not verified", () => { + expect(describeEffectiveAcceleration(makeNode({ capabilities: { resolved: "qsv" } }))).toBe( + "Currently resolves: QSV — not verified", + ); + }); + + it("describes no resolved backend as software encoding", () => { + expect(describeEffectiveAcceleration(makeNode({ capabilities: { resolved: "none" } }))).toBe( + "Currently resolves: software encoding", + ); + }); + + it("treats a report from a server predating these fields as software encoding", () => { + expect(describeEffectiveAcceleration(makeNode({ capabilities: {} }))).toBe( + "Currently resolves: software encoding", + ); + }); +}); + describe("describeSharedGPU", () => { const alone = makeNode({ id: 1, name: "transcode-1" }); const nvidiaA = makeNode({ id: 2, name: "transcode-a", physical_gpu_keys: ["GPU-aaa"] }); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index 7711cc1fc..59f114580 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -231,7 +231,7 @@ function describeBackend(resolved: string, detected: readonly NodeDetected[]): N } const label = resolved.toUpperCase(); - const entry = detected.find((candidate) => candidate.backend?.trim().toLowerCase() === resolved); + const entry = findDetectedEntry(resolved, detected); if (!entry) { // A configured backend wins resolution even with no candidate hardware to // probe, so absence of an entry is unknown, not failure. @@ -260,6 +260,47 @@ function badge(label: string, state: NodeGPUBackendState, title: string): NodeGP type NodeDetected = NonNullable[number]; +/** The probe entry for the resolved backend, or undefined when none was run. */ +function findDetectedEntry( + resolved: string, + detected: readonly NodeDetected[], +): NodeDetected | undefined { + return detected.find((candidate) => candidate.backend?.trim().toLowerCase() === resolved); +} + +/** + * One-line summary of what a transcode node's acceleration resolves to right + * now, for display under the per-node override select. Null when the node has + * never reported capabilities — there is nothing stored to describe yet, which + * is also true of a server predating capability reporting. + * + * This mirrors `describeBackend`'s verified/failed/unverified/none states but + * renders them as plain prose rather than a badge, since the edit dialog has + * no badge to hang it on. + */ +export function describeEffectiveAcceleration(node: StreamNode): string | null { + const capabilities = node.capabilities; + if (!capabilities) { + return null; + } + + const resolved = capabilities.resolved?.trim().toLowerCase() ?? ""; + if (resolved === "" || resolved === "none") { + return "Currently resolves: software encoding"; + } + + const label = resolved.toUpperCase(); + const entry = findDetectedEntry(resolved, capabilities.detected_backends ?? []); + if (!entry) { + return `Currently resolves: ${label} — not verified`; + } + if (!entry.verified) { + return `Currently resolves: ${label} — probe failed`; + } + const device = entry.device?.trim(); + return `Currently resolves: ${label} — verified${device ? ` on ${device}` : ""}`; +} + function otherFailures(resolved: string, detected: readonly NodeDetected[]): NodeGPUFailure[] { return detected .filter((entry) => { @@ -584,7 +625,7 @@ export const HW_ACCEL_INHERIT = "inherit"; * backend the cluster-wide setting could also name. */ export const HW_ACCEL_OVERRIDE_OPTIONS: readonly { value: string; label: string }[] = [ - { value: HW_ACCEL_INHERIT, label: "Inherit cluster setting" }, + { value: HW_ACCEL_INHERIT, label: "Cluster default" }, { value: "auto", label: "Auto" }, { value: "qsv", label: "Intel Quick Sync (QSV)" }, { value: "vaapi", label: "VA-API" }, From cca2f45161fb5b81ab6fbab2b05ab2d2519a1908 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:57:47 -0400 Subject: [PATCH 011/163] fix(nodes): close review findings on detection, re-probe, and drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the review of this branch. Each is a case where a surface this PR added could publish or persist something that is not true. Incomplete detection is no longer publishable. walkHWAccelBackends bounds itself at 30s regardless of how many candidate devices a host exposes, and a walk that ran out marked the backends it never reached Verified=false — which is byte-identical to a real hardware failure. The transcode node's outer budget (~106s) stayed healthy through that, so the degraded report was hashed, advertised, refetched, and persisted as a capability_drift regression for hardware that was fine, with the node routed to software until a clean probe landed. Detection now reports completeness (ErrHardwareDetectionIncomplete) and both node types refuse to publish a cut-short walk, keeping the previous hash exactly as they already do for a failed tone-map probe. The proxy's ctx.Err() check only caught the outer context, so it gains the same guard plus one hardware-aware deadline over both of its probes, matching the transcode node. The verified render device now reaches execution. Detection walks a backend's candidates and stops at the first that passes a smoke encode, but resolution discarded that device: a transcode with no configured playback.hw_device fell back to PickRenderDevice, which returns whatever sorts first under /dev/dri. On a mixed-vendor host those are different GPUs, so "qsv verified" was paired with ffmpeg initializing a card the probe never touched. The passing device is recorded per probe generation and adopted by the allocator, which also closes the reporting gap it caused: a default-configured node counted no workload for an unnamed device and reported zero sessions beside a busy engine. Probe-cache invalidation now supersedes in-flight probes. Both caches only cleared their map, so a probe that started first completed, stored its verdict, and handed it to a caller that had since invalidated — the operator re-probe could republish exactly what it was asked to discard and report "nothing changed". An invalidation generation in the cache and singleflight keys moves the key instead; shared work is still never canceled. The re-probe's busy check is now an exclusion, not a sample. activeJobs only moves once ffmpeg is running, so a node idle at the check accepted a transcode milliseconds later and the smoke encode raced the live encoder anyway — the false regression the 409 exists to prevent. A gpuGate held for the whole rebuild is consulted by every path that spawns ffmpeg (start, reconstruct, prepared download); neither side ever waits, both refuse. Drift recovery needs evidence. hardwareProbesClean returned true for a report with no detected_backends at all, which is what a GPU that disappeared entirely produces, so the next unrelated hash change (a reboot moving boot_id) cleared a standing note and told the operator the node recovered. Recovery now requires at least one probed backend that passed. Also: library-N Prometheus labels are assigned before the unavailable-mount skip, so a mount going away no longer renumbers the ones after it under an unchanged label; the proxy's /status guards a nil tracker the way /health already did; and docs/admin-api.md no longer says nothing in node selection reads last_stats while documenting the scratch admission guard that does. Found by review of #794, including findings raised by Codex and CodeRabbit. Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 21 ++- docs/wiki/deployment/docker.md | 4 + internal/nodemetrics/collector.go | 7 +- internal/nodemetrics/collector_test.go | 56 +++++++ internal/nodepool/health.go | 26 ++- internal/nodepool/health_drift_test.go | 67 +++++++- internal/playback/gpudetect.go | 160 +++++++++++++++--- internal/playback/gpudetect_publish_test.go | 173 ++++++++++++++++++++ internal/playback/gpudetect_test.go | 10 +- internal/playback/hwdevice.go | 15 +- internal/proxy/reprobe.go | 26 +-- internal/proxy/server.go | 26 ++- internal/tonemap/probe.go | 29 +++- internal/tonemap/probe_invalidate_test.go | 60 +++++++ internal/transcodenode/gpugate.go | 79 +++++++++ internal/transcodenode/gpugate_test.go | 80 +++++++++ internal/transcodenode/reprobe.go | 34 ++-- internal/transcodenode/reprobe_test.go | 43 +++++ internal/transcodenode/server.go | 39 ++++- 19 files changed, 839 insertions(+), 116 deletions(-) create mode 100644 internal/playback/gpudetect_publish_test.go create mode 100644 internal/transcodenode/gpugate.go create mode 100644 internal/transcodenode/gpugate_test.go diff --git a/docs/admin-api.md b/docs/admin-api.md index cf9c1fefc..96d2077af 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -157,7 +157,7 @@ Each entry in `last_stats.gpu`: |---|---|---| | `device` | string | The render node path (`/dev/dri/renderD128`), or `cuda:N` for an NVIDIA GPU with no readable DRM node. | | `vendor` | string | `intel`, `nvidia` or `amd`. Omitted when sysfs names a vendor we do not recognize. | -| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. A workload started with no `playback.hw_device` configured under QSV/VAAPI has no device name until ffmpeg picks one, and is the one case not counted here. | +| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. With no `playback.hw_device` configured the workload is counted against the render device auto-detection verified the backend on; it goes uncounted only when no probe has verified one, which is the state of a node whose backend was named explicitly and never walked. | | `video_busy_pct`, `render_busy_pct` | int | Engine busy percentages over the sampling interval. | | `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. Present only with an enrichment source — absent is not zero, and must not be rendered as an idle GPU. | | `vram_used_mb`, `vram_total_mb` | int | GPU memory, on the same terms as `total_busy_pct`. | @@ -171,8 +171,9 @@ the device this interval; its percentages are zeros with no measurement behind them. A node reports these fields in its own `/health` and `/status`; the API stores -them opaquely and never routes on them. Nothing in node selection reads -`last_stats`. +them opaquely and parses only what it routes on. No GPU field is one of those — +nothing in node selection reads `last_stats.gpu`. The one part that is read is +the `scratch` disk entry, described under "Scratch admission" above. Capability reports are refreshed by the background health sweep, not by this read: a node advertises a `capabilities_hash` in its own health response, and @@ -206,11 +207,15 @@ long before anyone reads a log. Semantics worth knowing: - Setting it is a comparison; clearing it is not. The note appears when a refetch - loses something, and stays until a refetch produces a report whose probes all - pass. A refetch that finds nothing *newly* lost leaves it alone, because a - delta against an already-degraded report always finds nothing — a reboot moves - `boot_id`, a reworded FFmpeg failure moves the probe reason, and either would - otherwise erase a standing regression and report a broken node as repaired. + loses something, and stays until a refetch produces a report that probed at + least one backend and every backend it probed passed. A refetch that finds + nothing *newly* lost leaves it alone, because a delta against an + already-degraded report always finds nothing — a reboot moves `boot_id`, a + reworded FFmpeg failure moves the probe reason, and either would otherwise + erase a standing regression and report a broken node as repaired. A report + that probed nothing at all does not clear it either: a GPU that disappeared + completely leaves no candidate backend to fail, and the absence of anything to + probe is not evidence of recovery. - A backend reported as `skipped` does not hold the note open. Skipping means the node cannot open the devices, which is a statement about access rather than about hardware. diff --git a/docs/wiki/deployment/docker.md b/docs/wiki/deployment/docker.md index bba1f603a..bbb5c4f61 100644 --- a/docs/wiki/deployment/docker.md +++ b/docs/wiki/deployment/docker.md @@ -279,6 +279,10 @@ volumes: - /proc/loadavg:/host/proc/loadavg:ro ``` +No setting turns this on: the sampler uses each file under `/host/proc` when it +is present and falls back to its own `/proc` when it is not, per file, so the +mounts take effect on the next sample and removing one reverts it. + These three mounts are only useful when the Docker host is itself an LXC/lxcfs container — a bare-metal or VM Docker host has nothing extra to gain from them, since its own `/proc` is already correct or already cgroup-corrected. Without diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index 1a56e3b51..b4e3fd4cd 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -107,12 +107,17 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { const bytesPerGB = float64(1024 * 1024 * 1024) libraries := 0 for _, disk := range system.Disks { + // The positional index has to advance for every non-scratch entry, + // measurable or not. Skipping first would renumber the mounts after + // an unavailable one, so an alert rule keyed on mount="library-1" + // would silently start reporting a different volume with no gap in + // the series to show it happened. + label := diskSeriesLabel(disk, &libraries) if disk.Unavailable { // A path this node cannot measure has no value to report, and a // zero would read as an empty disk in every alert rule. continue } - label := diskSeriesLabel(disk, &libraries) gauge(descDiskUsed, disk.UsedGB*bytesPerGB, label) gauge(descDiskTotal, disk.TotalGB*bytesPerGB, label) } diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go index 09a061d1c..6f1ce34c0 100644 --- a/internal/nodemetrics/collector_test.go +++ b/internal/nodemetrics/collector_test.go @@ -179,3 +179,59 @@ func TestRegisterCollectorIsIdempotent(t *testing.T) { registerCollector(first) registerCollector(second) } + +// The positional library label is a promise about which volume a series +// describes, and Prometheus has no way to signal that the promise moved: a +// mount going unavailable used to renumber every library after it, so an alert +// rule keyed on mount="library-1" silently followed a different disk with no +// gap in the series to show it happened. +func TestCollectorKeepsLibraryLabelsPositional(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/nas/movies", "/mnt/nas/shows") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + // /mnt/nas/movies is never answered, so it stays unavailable — the media + // root that lives on another node, or the export whose server went away. + f.answer("/mnt/nas/shows", fsStats{UsedBytes: 3 << 30, TotalBytes: 30 << 30, FSID: "c:1"}) + + f.sampleAndSettle(t, 3) + f.sampleAndSettle(t, 3) + + used := diskSeriesByLabel(t, f.sampler) + if _, reported := used["library-1"]; reported { + t.Fatalf("unavailable mount emitted a series: %v", used) + } + // The measurable root keeps the index its position earns, rather than + // sliding into the missing one's label. + if got, ok := used["library-2"]; !ok || got != float64(3<<30) { + t.Fatalf("library-2 = %v (present=%t), want the second library root's %v bytes", + got, ok, float64(3<<30)) + } +} + +// diskSeriesByLabel collects streamapp_node_disk_used_bytes keyed by its mount +// label. +func diskSeriesByLabel(t *testing.T, sampler *Sampler) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + byLabel := map[string]float64{} + for _, family := range families { + if family.GetName() != "streamapp_node_disk_used_bytes" { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == gpuDeviceLabel { + continue + } + byLabel[label.GetValue()] = metric.GetGauge().GetValue() + } + } + } + return byLabel +} diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index d36c1c8b2..b035df8a5 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -539,21 +539,35 @@ func resolveDriftNote(stored *string, drift capabilityDrift, parsed bool, payloa return nil } -// hardwareProbesClean reports whether every backend the node actually probed -// passed. A skipped backend is not a failure — it means the node cannot open the -// devices, which is the normal reading for a proxy pointed at a cluster-wide -// hw_device — and a report that cannot be parsed is not evidence of anything. +// hardwareProbesClean reports whether the node probed at least one backend and +// every backend it probed passed. A skipped backend is not a failure — it means +// the node cannot open the devices, which is the normal reading for a proxy +// pointed at a cluster-wide hw_device — and a report that cannot be parsed is +// not evidence of anything. +// +// The "at least one" half matters as much as the "every" half. A GPU that +// disappears completely leaves a report with no detected_backends at all, and a +// loop over an empty list finds no failure: taking that as clean would clear a +// standing drift note on the next unrelated hash change — a reboot moving +// boot_id is enough — and tell an operator the node recovered while its card is +// still gone. Recovery has to be evidenced by hardware that was actually +// probed, not by the absence of anything to probe. func hardwareProbesClean(payload []byte) bool { var current capabilityDriftView if json.Unmarshal(payload, ¤t) != nil { return false } + probed := false for _, backend := range current.DetectedBackends { - if !backend.Verified && !backend.Skipped { + if backend.Skipped { + continue + } + if !backend.Verified { return false } + probed = true } - return true + return probed } // computeCapabilityDrift compares the report a node just served against the one diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 076170465..7d190c4d3 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -114,16 +114,65 @@ func TestRefreshNodeCapabilitiesKeepsDriftWhenNothingRecovered(t *testing.T) { } } -// A backend that was skipped is a statement about device access, not about -// hardware: it is the normal reading for a proxy pointed at a cluster-wide -// hw_device. It must not hold a drift note open forever. -func TestResolveDriftNoteClearsOnASkippedButOtherwiseCleanReport(t *testing.T) { - const skippedPayload = `{"resolved":"none","render_devices":[],` + - `"detected_backends":[{"backend":"vaapi","verified":false,"skipped":true}]}` +// Clearing the note requires evidence that hardware came back, and a report in +// which nothing was probed carries none. +// +// The two shapes that produce no passing probe are the two ways hardware goes +// away: a device the node can no longer open reports the backend `skipped`, and +// a card that is gone entirely leaves no candidate backend to report at all. +// Both used to read as clean — the first because a skipped backend is not a +// failure, the second because a loop over an empty list finds none — so a +// standing regression was erased by the next unrelated hash change (a reboot +// moving boot_id is enough), telling an operator a still-broken node had +// recovered. +// +// A proxy pointed at a cluster-wide hw_device does not get stuck behind this: +// it never verified those backends in the first place, so computeCapabilityDrift +// never gives it a note to hold open. +func TestResolveDriftNoteKeepsNoteWhenNoProbePassed(t *testing.T) { + tests := []struct { + name string + payload string + }{ + { + name: "every candidate device is inaccessible", + payload: `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"vaapi","verified":false,"skipped":true}]}`, + }, + { + name: "the gpu is gone, so nothing was a candidate", + payload: `{"resolved":"none","render_devices":[],"detected_backends":[]}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + standing := "verified hardware backends lost: vaapi" + payload := []byte(test.payload) + // Both sides degraded: the delta finds nothing newly lost, which is + // exactly the state in which clearing has to be refused. + drift, parsed := computeCapabilityDrift(payload, payload) + got := resolveDriftNote(&standing, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift was cleared by a report in which no probe passed") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } + }) + } +} + +// The complement: a report with a backend that actually passed its probe is the +// evidence recovery needs, and clears the note. +func TestResolveDriftNoteClearsOnAPassingProbe(t *testing.T) { + const recoveredPayload = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"vaapi","verified":true},` + + `{"backend":"qsv","verified":false,"skipped":true}]}` standing := "verified hardware backends lost: vaapi" - drift, parsed := computeCapabilityDrift([]byte(skippedPayload), []byte(skippedPayload)) - if got := resolveDriftNote(&standing, drift, parsed, []byte(skippedPayload)); got != nil { - t.Fatalf("capability_drift = %q, want a skipped backend to count as clean", *got) + payload := []byte(recoveredPayload) + drift, parsed := computeCapabilityDrift(payload, payload) + if got := resolveDriftNote(&standing, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want a verified backend to clear it", *got) } } diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index b8fadeff2..e882d7806 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -2,6 +2,7 @@ package playback import ( "context" + "errors" "fmt" "log/slog" "os" @@ -54,8 +55,20 @@ var hwProbeCache = struct { sync.Mutex entries map[string]hwProbeCacheEntry group singleflight.Group + // generation counts invalidations. It is part of every cache and + // singleflight key, which is what makes InvalidateHWProbeCache supersede a + // probe already in flight rather than merely clearing the map in front of + // it: the flight stores its result under the generation it started in, and + // a caller arriving afterwards asks a different key and therefore starts a + // fresh probe instead of joining the stale one. + generation uint64 + // verifiedDevices records, per generation and backend, the candidate device + // whose smoke encode passed. Execution reads it so a backend verified on + // one render node is not then run on another; see VerifiedHWDevice. + verifiedDevices map[string]string }{ - entries: make(map[string]hwProbeCacheEntry), + entries: make(map[string]hwProbeCacheEntry), + verifiedDevices: make(map[string]string), } // DetectedBackend reports one hardware backend that has candidate devices on @@ -138,16 +151,39 @@ func DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { // pass-through contract: an explicitly configured backend wins even when its // probe failed, and the report carries the failure reason. func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { + info, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccel, ffmpegPath, hwDevice) + return info +} + +// ErrHardwareDetectionIncomplete reports that a detection walk ended before it +// had probed every candidate backend, because its own budget or the caller's +// context ran out. +// +// The report it accompanies is still returned — an operator-facing surface can +// show what was learned — but it must never be hashed and published as this +// host's capabilities. A cut-short walk marks unprobed backends Verified=false +// and resolves to software, which is byte-for-byte what a real hardware failure +// looks like: the API's health sweep would then persist a capability_drift note +// for hardware that is fine, latch it until a clean report arrives, and route +// the node to software encoding in the meantime. +var ErrHardwareDetectionIncomplete = errors.New("hardware detection did not complete within its budget") + +// DetectHWAccelWithFFmpegContextResult is DetectHWAccelWithFFmpegContext with +// the walk's completeness reported. Callers that publish or hash the report — +// the node capability endpoints and their background snapshots — must use this +// form and refuse to publish on ErrHardwareDetectionIncomplete. +func DetectHWAccelWithFFmpegContextResult(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) (HWAccelInfo, error) { candidates := collectHWCandidates(hwDevice) resolved := HWAccelNone var detected []DetectedBackend + complete := true if currentGOOS == directPlayLinuxGOOS { - resolved, detected = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) + resolved, detected, complete = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) } if configured := strings.TrimSpace(hwAccel); configured != "" && configured != hwAccelAuto { resolved = configured } - return HWAccelInfo{ + info := HWAccelInfo{ Resolved: resolved, RenderDevices: candidates.renderDevices, RenderDeviceDetails: renderDeviceDetails(candidates.renderDevices), @@ -156,6 +192,10 @@ func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hw BootID: detectBootID(), Source: "local", } + if !complete { + return info, ErrHardwareDetectionIncomplete + } + return info, nil } // PickRenderDevice returns the GPU render device path to use. @@ -194,7 +234,7 @@ func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, h if currentGOOS != "linux" { return HWAccelNone } - resolved, _ := walkHWAccelBackends(ctx, ffmpegPath, collectHWCandidates(hwDevice), true) + resolved, _, _ := walkHWAccelBackends(ctx, ffmpegPath, collectHWCandidates(hwDevice), true) return resolved } @@ -311,23 +351,32 @@ func (c hwCandidates) probeDevicesFor(backend string) []string { // walkHWAccelBackends verifies each backend with candidate hardware in // preference order and reports the first one whose probe passes. Resolution // stops there; detection continues so every candidate backend is reported. -func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCandidates, stopAtFirstVerified bool) (string, []DetectedBackend) { +// +// complete reports whether every candidate backend was actually probed. It is +// false when the walk budget or the caller's context ran out partway through, +// which leaves unprobed backends indistinguishable from failed ones — see +// ErrHardwareDetectionIncomplete for why a publisher must not hash such a +// report. +func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCandidates, stopAtFirstVerified bool) (resolved string, detected []DetectedBackend, complete bool) { if ctx == nil { ctx = context.Background() } ctx, cancel := context.WithTimeout(ctx, hwAccelWalkTimeout) defer cancel() - resolved := "" - var detected []DetectedBackend + complete = true for _, backend := range hwAccelPreferenceOrder { if !candidates.presentFor(backend) { continue } // A caller that has already given up must not start further probes. if ctx.Err() != nil { + complete = false break } - entry := verifyHWAccelBackend(ctx, backend, ffmpegPath, candidates) + entry, probedFully := verifyHWAccelBackend(ctx, backend, ffmpegPath, candidates) + if !probedFully { + complete = false + } if !entry.Verified { slog.WarnContext(ctx, "hw_accel=auto: candidate hardware failed its FFmpeg probe", "backend", backend, "devices", entry.Devices, @@ -338,26 +387,35 @@ func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCa } detected = append(detected, entry) if resolved != "" && stopAtFirstVerified { - return resolved, detected + // Stopping early is the caller's instruction, not a budget failure: + // the remaining backends are lower preference and would not have + // been selected either way. + return resolved, detected, complete } } if resolved == "" { slog.InfoContext(ctx, "hw_accel=auto: no verified hardware backend, using software encoding") - return HWAccelNone, detected + return HWAccelNone, detected, complete } - return resolved, detected + return resolved, detected, complete } // verifyHWAccelBackend probes a backend's candidate devices in order and stops // at the first one that passes, so a broken GPU sorting ahead of a working one // does not disable the backend for the whole host. -func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candidates hwCandidates) DetectedBackend { +// +// complete reports whether every candidate was reached. A device left unprobed +// because the budget ran out is not a device that failed, and the difference is +// invisible in the returned entry. +func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candidates hwCandidates) (entry DetectedBackend, complete bool) { devices := candidates.probeDevicesFor(backend) - entry := DetectedBackend{Backend: backend, Devices: candidates.devicesFor(backend)} + entry = DetectedBackend{Backend: backend, Devices: candidates.devicesFor(backend)} reasons := make([]string, 0, len(devices)) probed := false + complete = true for _, device := range devices { if ctx.Err() != nil { + complete = false break } if !candidates.deviceProbeable(device) { @@ -369,7 +427,11 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi if available { entry.Verified = true entry.Device = device - return entry + // Execution has to land on this device and not on whatever sorts + // first under /dev/dri, or a report saying "qsv verified" is paired + // with a transcode initializing a GPU the probe never touched. + recordVerifiedHWDevice(backend, device) + return entry, complete } reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) } @@ -378,7 +440,45 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi } entry.Skipped = !probed && len(devices) > 0 && ctx.Err() == nil entry.Reason = strings.Join(reasons, "; ") - return entry + return entry, complete +} + +// recordVerifiedHWDevice remembers the candidate a backend's smoke encode +// passed on, for the generation that probed it. +// +// It is scoped to the probe generation so an operator-triggered re-probe cannot +// be answered with a device blessed by the verdicts it just discarded. NVENC +// records the empty device — CUDA addresses its GPU without a render-node path +// — which reads identically to "nothing verified" and is exactly right: there +// is no path for execution to adopt. +func recordVerifiedHWDevice(backend, device string) { + if device == "" { + return + } + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + hwProbeCache.verifiedDevices[verifiedHWDeviceKey(hwProbeCache.generation, backend)] = device +} + +// VerifiedHWDevice returns the render device this process most recently +// verified the given backend on, or "" when no probe has passed for it. +// +// It exists because auto-detection and execution pick devices independently: +// detection walks a backend's candidates in order and stops at the first that +// passes a smoke encode, while a transcode with no configured playback.hw_device +// falls back to PickRenderDevice, which returns whatever sorts first under +// /dev/dri. On a host whose first render node belongs to a different vendor, +// those are different GPUs, and the transcode initializes hardware that was +// never verified. Reading the verified device closes that gap without making +// every caller of resolution carry a second return value. +func VerifiedHWDevice(backend string) string { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + return hwProbeCache.verifiedDevices[verifiedHWDeviceKey(hwProbeCache.generation, backend)] +} + +func verifiedHWDeviceKey(generation uint64, backend string) string { + return strconv.FormatUint(generation, 10) + "\x00" + backend } // deviceProbeable reports whether a candidate device may be smoke-encoded on. @@ -451,13 +551,16 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi return false, "unsupported hardware backend " + backend } ffmpegPath = normalizeFFmpegPath(ffmpegPath) - cacheKey := hwProbeCacheKey(ffmpegPath, backend, device) // The flight below outlives an abandoned caller, so every test-mutable seam // it touches is snapshotted here rather than dereferenced inside it. commandTimeout := hwProbeCommandTimeout negativeTTL := hwProbeNegativeTTL now := hwProbeNow hwProbeCache.Lock() + // The generation is baked into the key, so an invalidation that lands while + // this probe runs leaves the flight writing to a key nobody will read again + // and sends the next caller to a fresh one. + cacheKey := hwProbeCacheKey(hwProbeCache.generation, ffmpegPath, backend, device) if entry, ok := hwProbeCache.entries[cacheKey]; ok && hwProbeCacheEntryCurrent(entry, now()) { hwProbeCache.Unlock() return entry.result.available, entry.result.reason @@ -512,23 +615,26 @@ func hwProbeCacheEntryCurrent(entry hwProbeCacheEntry, now time.Time) bool { // a card, or changing device access underneath a running node. Without this the // only way to re-verify is a restart. // -// A probe already in flight is neither canceled nor discarded: it completes and -// writes its result, and a caller that re-probes while another goroutine is -// mid-probe therefore joins that flight and can observe the pre-invalidation -// verdict once. Canceling shared work would instead fail an unrelated playback -// request that is waiting on it, which is the worse trade. The operator-facing -// re-probe action runs the detection walk itself after invalidating, so in the -// ordinary single-caller case the cache is repopulated from a cold start. +// A probe already in flight is neither canceled nor discarded — canceling +// shared work would fail an unrelated playback request waiting on it — but it +// is superseded: bumping the generation moves every cache and singleflight key, +// so the in-flight probe stores its verdict where nothing will read it and the +// next caller starts a genuinely cold probe rather than joining the old flight. +// That is what makes the operator-facing re-probe honest; without it a re-probe +// racing a background capability fetch would republish the verdict it was asked +// to discard, and report "nothing changed". func InvalidateHWProbeCache() { hwProbeCache.Lock() defer hwProbeCache.Unlock() + hwProbeCache.generation++ hwProbeCache.entries = make(map[string]hwProbeCacheEntry) + hwProbeCache.verifiedDevices = make(map[string]string) } -// hwProbeCacheKey separates results per backend and per candidate device on top -// of the FFmpeg binary's identity. -func hwProbeCacheKey(ffmpegPath, backend, device string) string { - return strings.Join([]string{ffmpegIdentityKey(ffmpegPath), backend, device}, "\x00") +// hwProbeCacheKey separates results per invalidation generation, per backend, +// and per candidate device on top of the FFmpeg binary's identity. +func hwProbeCacheKey(generation uint64, ffmpegPath, backend, device string) string { + return strings.Join([]string{strconv.FormatUint(generation, 10), ffmpegIdentityKey(ffmpegPath), backend, device}, "\x00") } // ffmpegIdentityKey invalidates cached capability results when an FFmpeg diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go new file mode 100644 index 000000000..233a9a194 --- /dev/null +++ b/internal/playback/gpudetect_publish_test.go @@ -0,0 +1,173 @@ +package playback + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// A detection walk that ran out of budget marks backends it never reached +// Verified=false, which is byte-identical to a real hardware failure. A node +// that hashed and published that report would tell the API its GPU regressed, +// and the API would persist a capability_drift note, latch it until a clean +// report arrives, and route the node to software in the meantime — all for +// hardware that is fine. So the incompleteness has to reach the publisher. +func TestDetectHWAccelReportsAnIncompleteWalk(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + info, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, ffmpeg.path, "") + if !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete", err) + } + // The report still comes back for an operator-facing surface to show; it is + // only publishing it as this host's capabilities that is refused. + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want an abandoned walk to resolve to software", info.Resolved) + } +} + +// The complement: a walk that reached every candidate backend publishes +// normally, or nothing would ever be inventoried. +func TestDetectHWAccelReportsACompleteWalk(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWQSV { + t.Fatalf("Resolved = %q, want qsv", info.Resolved) + } +} + +// Detection walks a backend's candidates in order and stops at the first that +// passes a smoke encode; execution with no configured playback.hw_device used to +// fall back to PickRenderDevice, which returns whatever sorts first under +// /dev/dri. On a mixed-vendor host those are different GPUs, so a report saying +// "qsv verified" was paired with a transcode initializing a card the probe had +// never touched. +func TestAcquireHWDeviceUsesTheVerifiedRenderDevice(t *testing.T) { + env := setupHWAccelTest(t) + // renderD128 sorts first and is AMD, so it is not a QSV candidate at all; + // only renderD129 can pass the probe. + env.addRenderDevice(t, "renderD128", "0x1002") + env.addRenderDevice(t, "renderD129", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := ResolveHWAccelWithFFmpeg(hwAccelAuto, ffmpeg.path, ""); got != transcodeHWQSV { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv", got) + } + verified := VerifiedHWDevice(transcodeHWQSV) + if want := env.devicePath("renderD129"); verified != want { + t.Fatalf("VerifiedHWDevice(qsv) = %q, want %q", verified, want) + } + // The unverified device is the one auto-detection would otherwise pick. + if fallback := PickRenderDevice(""); fallback == verified { + t.Skip("test setup no longer distinguishes the verified device from the first render node") + } + + device, release := AcquireHWDevice("", transcodeHWQSV) + defer release() + if device != verified { + t.Fatalf("AcquireHWDevice() = %q, want the verified device %q", device, verified) + } + // Counting it is the other half: a default-configured node reported zero + // sessions beside a busy engine because an unnamed device was never counted. + if got := hwDeviceActiveCount(verified); got != 1 { + t.Fatalf("active workloads on %s = %d, want 1", verified, got) + } + release() + if got := hwDeviceActiveCount(verified); got != 0 { + t.Fatalf("active workloads after release = %d, want 0", got) + } + + // An operator re-probe discards the verdicts, so the device they blessed + // goes with them: answering from the old generation would let execution + // keep using a device the re-probe was asked to re-verify. + InvalidateHWProbeCache() + if got := VerifiedHWDevice(transcodeHWQSV); got != "" { + t.Fatalf("VerifiedHWDevice(qsv) after invalidation = %q, want empty", got) + } +} + +// With nothing verified — a backend named explicitly and never walked — the +// device stays unresolved and ffmpeg picks one downstream, exactly as before. +func TestAcquireHWDeviceLeavesTheDeviceUnsetWithoutAVerifiedProbe(t *testing.T) { + setupHWAccelTest(t) + + device, workload, release := acquireHWDevice("", transcodeHWQSV, "") + defer release() + if device != "" || workload != "" { + t.Fatalf("acquireHWDevice() = (%q, %q), want both empty with no verified device", device, workload) + } +} + +// Invalidation has to supersede a probe already in flight, not merely clear the +// map in front of it. The operator-facing re-probe exists to force a cold +// re-verification; if a probe that started before the invalidation could hand +// its pre-invalidation verdict to the caller that invalidated, the action would +// republish exactly what it was asked to discard and report "nothing changed". +func TestInvalidateHWProbeCacheSupersedesAnInFlightProbe(t *testing.T) { + env := setupHWAccelTest(t) + // The shared setup pins a 200ms per-command budget so hung-probe tests stay + // fast; this one needs a probe that is slow but succeeds, so it has to + // outlive that. setupHWAccelTest restores the original on cleanup. + hwProbeCommandTimeout = 5 * time.Second + env.addRenderDevice(t, "renderD128", "0x8086") + probe := successfulVAAPIProbe() + probe.delay = 300 * time.Millisecond + ffmpeg := writeFakeFFmpeg(t, probe) + device := env.devicePath("renderD128") + + var wg sync.WaitGroup + wg.Go(func() { + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Errorf("in-flight probe failed: %s", reason) + } + }) + + // Let the flight start its first bounded command, then discard its verdict. + time.Sleep(50 * time.Millisecond) + InvalidateHWProbeCache() + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Fatalf("post-invalidation probe failed: %s", reason) + } + wg.Wait() + + // Two independent smoke encodes ran: the second call started its own probe + // rather than joining the flight the invalidation superseded, which is the + // whole difference between clearing the map and moving the key. + if got := smokeEncodeCount(t, ffmpeg.logPath); got < 2 { + t.Fatalf("smoke encodes = %d, want the post-invalidation probe to run its own", got) + } +} + +// smokeEncodeCount counts the synthetic single-frame encodes in a fake ffmpeg's +// command log. Every hardware probe ends in exactly one. +func smokeEncodeCount(t *testing.T, logPath string) int { + t.Helper() + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read ffmpeg probe log: %v", err) + } + return strings.Count(string(data), "testsrc2") +} + +// devicePath is the full path of a render device added to this test's /dev/dri +// stand-in. +func (e *hwAccelTestEnv) devicePath(name string) string { + return filepath.Join(e.driDir, name) +} diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index be8faf4f2..a26fabdf6 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -661,11 +661,11 @@ func TestHWProbeCacheSeparatesBackendsAndDevices(t *testing.T) { ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) keys := map[string]string{ - "nvenc": hwProbeCacheKey(ffmpeg.path, transcodeHWNVENC, ""), - "qsv-128": hwProbeCacheKey(ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD128"), - "qsv-129": hwProbeCacheKey(ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD129"), - "vaapi-128": hwProbeCacheKey(ffmpeg.path, transcodeHWVAAPI, "/dev/dri/renderD128"), - "identity-eq": hwProbeCacheKey(ffmpeg.path, transcodeHWNVENC, ""), + "nvenc": hwProbeCacheKey(0, ffmpeg.path, transcodeHWNVENC, ""), + "qsv-128": hwProbeCacheKey(0, ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD128"), + "qsv-129": hwProbeCacheKey(0, ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD129"), + "vaapi-128": hwProbeCacheKey(0, ffmpeg.path, transcodeHWVAAPI, "/dev/dri/renderD128"), + "identity-eq": hwProbeCacheKey(0, ffmpeg.path, transcodeHWNVENC, ""), } if keys["nvenc"] != keys["identity-eq"] { t.Fatal("identical backend and device produced different cache keys") diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index 138a70b3e..6a1b87ebf 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -253,11 +253,22 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, w // Nothing to balance, but there is still a workload on a known device. // Selection and accounting are separate decisions: skipping the second // with the first is what left every single-GPU QSV/VAAPI node reporting - // zero sessions beside a busy engine. An empty value has no device to - // name — ffmpeg picks one downstream — so it stays uncounted. + // zero sessions beside a busy engine. if first := set.First(); first != "" { return first, first, countHWDeviceWorkload(first) } + // No configured device. Resolving it here rather than leaving it to + // PickRenderDevice downstream fixes two things at once: the workload + // runs on the render node auto-detection actually verified this backend + // on — not on whatever sorts first under /dev/dri, which on a + // mixed-vendor host is a different GPU — and it becomes countable, so a + // default-configured node stops reporting zero sessions beside a busy + // engine. With no verified device (nothing probed yet, or a backend + // named explicitly and never walked) this falls through unchanged and + // ffmpeg picks one downstream exactly as before. + if verified := VerifiedHWDevice(resolvedHWAccel); verified != "" { + return verified, verified, countHWDeviceWorkload(verified) + } return "", "", noop } // Select and reserve in one critical section so concurrent workload starts diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go index 7fd3982cb..76ab42ad8 100644 --- a/internal/proxy/reprobe.go +++ b/internal/proxy/reprobe.go @@ -1,11 +1,9 @@ package proxy import ( - "context" "encoding/json" "log/slog" "net/http" - "time" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/tonemap" @@ -37,9 +35,9 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques playback.InvalidateHWProbeCache() tonemap.InvalidateProbeCache() - ctx, cancel := context.WithTimeout(r.Context(), s.capabilityProbeBudget()) - defer cancel() - info, err := s.buildCapabilitySnapshot(ctx) + // buildCapabilitySnapshot owns the probe deadline, so a re-probe can never + // cost more than a cold capability fetch already may. + info, err := s.buildCapabilitySnapshot(r.Context()) if err != nil { slog.WarnContext(r.Context(), "proxy capability re-probe incomplete", "component", "proxy", "error", err) http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) @@ -57,21 +55,3 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques slog.WarnContext(r.Context(), "encode proxy re-probe result", "component", "proxy", "error", err) } } - -// capabilityProbeBudget is the deadline one snapshot rebuild gets. -// -// A proxy's snapshot is the bounded hardware walk plus the transformation -// registry's own bounded commands — not the tone-map matrix — so this is an -// over-allowance rather than a measurement. It is deliberately the same number -// the transcode node uses: both node types advertise one probe budget to -// callers, and a second constant here would be a second thing to keep in step -// with the walk and registry timeouts. -func (s *Server) capabilityProbeBudget() time.Duration { - hwAccel := playback.HWAccelNone - hwDevice := "" - if cfg := s.watcher.Config(); cfg != nil { - hwAccel = cfg.Playback.HWAccel - hwDevice = cfg.Playback.HWDevice - } - return tonemap.ProbeEndpointTimeout(hwAccel, hwDevice) -} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 8de2eed25..c43657e4d 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -29,6 +29,7 @@ import ( "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" + "github.com/Silo-Server/silo-server/internal/tonemap" ) // Server is the HTTP handler for proxy mode. @@ -252,10 +253,19 @@ func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelI hwAccel = cfg.Playback.HWAccel hwDevice = cfg.Playback.HWDevice } - info := playback.DetectHWAccelWithFFmpegContext(ctx, hwAccel, ffmpegPath, hwDevice) - if err := ctx.Err(); err != nil { - // The hardware walk has no error return: it degrades to unverified - // backends when its context ends mid-probe. + // One hardware-aware deadline over both probes, matching the transcode + // node: each has its own internal bound, but only a shared budget keeps the + // whole rebuild inside the window a caller was told to allow, and only a + // deadline the builder owns bounds the background snapshot, whose context + // lives as long as the process. + ctx, cancel := context.WithTimeout(ctx, tonemap.ProbeEndpointTimeout(hwAccel, hwDevice)) + defer cancel() + // A walk that ran out of budget is refused rather than published: it marks + // unprobed backends Verified=false, which is byte-identical to a real + // hardware failure, so hashing it would announce a change that did not + // happen and cost this proxy its stored inventory. + info, err := playback.DetectHWAccelWithFFmpegContextResult(ctx, hwAccel, ffmpegPath, hwDevice) + if err != nil { return playback.HWAccelInfo{}, err } registry, err := playback.ProbeTransformationRegistryWithToneMapV3Result(ctx, ffmpegPath, nil) @@ -887,10 +897,16 @@ type statusResponse struct { } func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { + // NewServer accepts a nil tracker and handleHealth already tolerates one; + // this must too, or the same construction that answers /health panics here. + activeSessions := 0 + if s.tracker != nil { + activeSessions = s.tracker.ActiveCount() + } snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(statusResponse{ - ActiveSessions: s.tracker.ActiveCount(), + ActiveSessions: activeSessions, System: snapshot.System, GPU: snapshot.GPU, }) diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index 479f8e782..9658d69a6 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -7,6 +7,7 @@ import ( "errors" "os" "os/exec" + "strconv" "strings" "sync" "time" @@ -51,6 +52,12 @@ var probeCache = struct { sync.Mutex entries map[string]probeCacheEntry group singleflight.Group + // generation counts invalidations and is part of every cache and + // singleflight key. It is what makes InvalidateProbeCache supersede a probe + // already in flight instead of merely clearing the map in front of it: the + // flight stores its inventory under the generation it started in, and the + // next caller asks a different key and therefore runs a fresh probe. + generation uint64 }{entries: make(map[string]probeCacheEntry)} // Probe returns the cached, smoke-tested tone-map capabilities for an FFmpeg @@ -62,8 +69,8 @@ func Probe(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevice stri // probeCached coalesces identical probes without allowing one caller's // cancellation to abort the shared work needed by other playback requests. func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevice string, run CommandRunner, now func() time.Time) (Capabilities, error) { - key := probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice) probeCache.Lock() + key := probeCacheKey(probeCache.generation, ffmpegPath, hardwareBackend, hardwareDevice) if cached, ok := probeCache.entries[key]; ok && probeCacheEntryCurrent(cached, now()) { result := append(Capabilities(nil), cached.capabilities...) probeCache.Unlock() @@ -136,7 +143,7 @@ func probeWithRunner( // probeCacheKey binds reusable capabilities to the resolved FFmpeg binary and // the driver facts for every configured hardware device. -func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { +func probeCacheKey(generation uint64, ffmpegPath, hardwareBackend, hardwareDevice string) string { binaryIdentity := strings.TrimSpace(ffmpegPath) if _, cacheKey, cacheable := ffmpegBinaryCacheKey(binaryIdentity); cacheable { binaryIdentity = cacheKey @@ -152,7 +159,10 @@ func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { driverIdentities = append(driverIdentities, driverFingerprint(backend, configuredDevice)) } } - return strings.Join([]string{binaryIdentity, backend, device, strings.Join(driverIdentities, ",")}, "\x00") + return strings.Join([]string{ + strconv.FormatUint(generation, 10), + binaryIdentity, backend, device, strings.Join(driverIdentities, ","), + }, "\x00") } // InvalidateProbeCache drops every cached tone-map inventory so the next probe @@ -164,14 +174,17 @@ func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { // when the file does, and a driver has no key at all. This is the seam an // operator-triggered re-probe uses to force the matrix to run again. // -// A probe already in flight is neither canceled nor discarded: it completes and -// stores its result, so a caller that invalidates concurrently with another -// probe of the same key joins that flight and can observe the pre-invalidation -// inventory once. Canceling shared work would fail the unrelated playback -// request waiting on it. +// A probe already in flight is neither canceled nor discarded — canceling +// shared work would fail the unrelated playback request waiting on it — but it +// is superseded: bumping the generation moves every cache and singleflight key, +// so the in-flight probe stores its inventory where nothing will read it and +// the next caller runs a genuinely cold probe rather than joining the old +// flight. Without that, a re-probe racing a background capability fetch would +// republish the very inventory it was asked to discard. func InvalidateProbeCache() { probeCache.Lock() defer probeCache.Unlock() + probeCache.generation++ probeCache.entries = make(map[string]probeCacheEntry) } diff --git a/internal/tonemap/probe_invalidate_test.go b/internal/tonemap/probe_invalidate_test.go index ebf55e699..d7b3b2e90 100644 --- a/internal/tonemap/probe_invalidate_test.go +++ b/internal/tonemap/probe_invalidate_test.go @@ -2,6 +2,8 @@ package tonemap import ( "context" + "sync" + "sync/atomic" "testing" "time" ) @@ -51,3 +53,61 @@ func TestInvalidateProbeCacheForcesAnotherProbe(t *testing.T) { t.Fatalf("probe after invalidation ran %d commands total, want %d", reprobed, first*2) } } + +// Clearing the map is not enough: a probe already in flight completes and +// stores its inventory, so without a generation in the key the caller that +// invalidated would join that flight and be handed the very result it asked to +// discard — a re-probe reporting "nothing changed" about hardware it never +// re-examined. The generation moves the key instead, so the in-flight probe +// writes where nothing will read and the next caller runs a cold one. +func TestInvalidateProbeCacheSupersedesAnInFlightProbe(t *testing.T) { + resetProbeCache(t) + now := time.Unix(100, 0) + clock := func() time.Time { return now } + + // started closes once the first probe is inside its runner; blocked holds it + // there until the test has invalidated, so the race is decided rather than + // slept on. + started := make(chan struct{}) + blocked := make(chan struct{}) + var runs atomic.Int32 + runner := func(_ context.Context, _ string, args ...string) ([]byte, error) { + if runs.Add(1) == 1 { + close(started) + <-blocked + } + if len(args) > 0 && args[len(args)-1] == "-filters" { + return []byte(" .S. zscale V->V\n .S. tonemapx V->V\n .S. sidedata V->V\n"), nil + } + if len(args) > 0 && args[len(args)-1] == "-encoders" { + return []byte("libx264"), nil + } + return nil, nil + } + + var wg sync.WaitGroup + wg.Go(func() { + if _, err := probeCached(context.Background(), "/ffmpeg-inflight", BackendSoftware, "", runner, clock); err != nil { + t.Errorf("in-flight probe error = %v", err) + } + }) + + <-started + InvalidateProbeCache() + close(blocked) + + capabilities, err := probeCached(context.Background(), "/ffmpeg-inflight", BackendSoftware, "", runner, clock) + if err != nil { + t.Fatalf("post-invalidation probe error = %v", err) + } + if len(capabilities) != 1 { + t.Fatalf("post-invalidation capabilities = %#v, want one software entry", capabilities) + } + wg.Wait() + + // One command from the blocked flight is enough to prove the second probe + // did not simply wait on it: a joined caller would have run none of its own. + if got := runs.Load(); got < 2 { + t.Fatalf("runner invocations = %d, want the post-invalidation probe to run its own commands", got) + } +} diff --git a/internal/transcodenode/gpugate.go b/internal/transcodenode/gpugate.go new file mode 100644 index 000000000..969f84c4a --- /dev/null +++ b/internal/transcodenode/gpugate.go @@ -0,0 +1,79 @@ +package transcodenode + +import "sync" + +// gpuGate keeps an operator-triggered capability re-probe and the node's own +// GPU work off the encoder at the same time. +// +// Every hardware probe ends in a real smoke encode, which opens an encoder +// session. A card at its concurrent-session cap fails that encode with an error +// nothing can tell apart from a missing device or a broken driver, so a probe +// that races a transcode publishes a hardware regression for a GPU that is +// fine — and the API persists it, latches it, and routes the node to software +// until a clean report arrives. +// +// A point-in-time "are there active jobs" check cannot prevent that: a node +// idle at the check accepts a start milliseconds later, while the probe still +// has minutes to run. What is needed is one exclusion both sides consult, held +// from before the probe begins until after it ends. +// +// The gate is deliberately asymmetric. Work never waits: it is admitted or +// refused immediately, because a viewer pressing play must not queue behind a +// multi-minute probe. The re-probe never waits either: it refuses with 409 and +// tells the operator to retry when the node is idle, because blocking would +// hold an admin HTTP connection open for the length of a stream. +type gpuGate struct { + mu sync.Mutex + // workers counts GPU work that has been admitted and has not finished. It + // is separate from Server.activeJobs because that counter is incremented + // only once ffmpeg is already running: the window this gate exists to close + // is precisely the one between admitting work and it becoming visible + // there. + workers int + // reprobing is set for the whole capability rebuild, including the cache + // invalidation that precedes it. + reprobing bool +} + +// beginWork admits one unit of GPU work, or reports false while a re-probe +// holds the encoder. A caller that is admitted must call endWork exactly once. +func (g *gpuGate) beginWork() bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.reprobing { + return false + } + g.workers++ + return true +} + +// endWork releases one unit of admitted GPU work. +func (g *gpuGate) endWork() { + g.mu.Lock() + if g.workers > 0 { + g.workers-- + } + g.mu.Unlock() +} + +// beginReprobe claims the encoder exclusively, or reports the work in progress +// that stopped it. activeJobs is the node's own running-session count, passed in +// so both halves of "is this node busy" are read under one lock rather than +// sampled at two different instants. +func (g *gpuGate) beginReprobe(activeJobs int) (busy int, ok bool) { + g.mu.Lock() + defer g.mu.Unlock() + busy = g.workers + activeJobs + if g.reprobing || busy > 0 { + return busy, false + } + g.reprobing = true + return 0, true +} + +// endReprobe releases the exclusive claim. +func (g *gpuGate) endReprobe() { + g.mu.Lock() + g.reprobing = false + g.mu.Unlock() +} diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go new file mode 100644 index 000000000..9bbcdda29 --- /dev/null +++ b/internal/transcodenode/gpugate_test.go @@ -0,0 +1,80 @@ +package transcodenode + +import "testing" + +// The bug this gate exists for: a node idle at a point-in-time check accepts a +// transcode milliseconds later, and the re-probe's smoke encode — minutes long +// — then races a live encoder session and publishes working hardware as failed. +// Admitted work has to keep the re-probe out even before it is visible as an +// active job. +func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { + var gate gpuGate + + if !gate.beginWork() { + t.Fatal("beginWork on an idle gate was refused") + } + // activeJobs is still 0 here: the counter only moves once ffmpeg is running, + // which is exactly the window a point-in-time check missed. + if busy, ok := gate.beginReprobe(0); ok { + t.Fatal("re-probe admitted while a transcode was starting") + } else if busy != 1 { + t.Fatalf("busy = %d, want the admitted work counted", busy) + } + + gate.endWork() + if _, ok := gate.beginReprobe(0); !ok { + t.Fatal("re-probe refused after the work finished") + } +} + +// The node's own running-session count is read under the same lock, so "no +// admitted work" and "no active jobs" cannot be true at two different instants. +func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(2) + if ok { + t.Fatal("re-probe admitted on a node running transcodes") + } + if busy != 2 { + t.Fatalf("busy = %d, want 2", busy) + } +} + +// A re-probe holds the encoder for the whole rebuild, so work arriving mid-probe +// is refused rather than queued: a viewer pressing play must not wait minutes +// for a probe, and the API retries elsewhere. +func TestGPUGateRefusesWorkWhileReprobing(t *testing.T) { + var gate gpuGate + + if _, ok := gate.beginReprobe(0); !ok { + t.Fatal("re-probe refused on an idle gate") + } + if gate.beginWork() { + t.Fatal("GPU work admitted while a re-probe held the encoder") + } + if _, ok := gate.beginReprobe(0); ok { + t.Fatal("a second concurrent re-probe was admitted") + } + + gate.endReprobe() + if !gate.beginWork() { + t.Fatal("GPU work refused after the re-probe released the encoder") + } + gate.endWork() +} + +// endWork must not drive the counter negative, or one unbalanced release would +// let a re-probe run beside real transcodes forever. +func TestGPUGateEndWorkDoesNotUnderflow(t *testing.T) { + var gate gpuGate + + gate.endWork() + gate.endWork() + if !gate.beginWork() { + t.Fatal("beginWork refused after unbalanced releases") + } + if _, ok := gate.beginReprobe(0); ok { + t.Fatal("re-probe admitted while one unit of work was outstanding") + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index d1e2cce47..736d97a77 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -1,12 +1,10 @@ package transcodenode import ( - "context" "encoding/json" "fmt" "log/slog" "net/http" - "time" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/tonemap" @@ -57,21 +55,28 @@ type reprobeCapabilitiesResponse struct { // hardware investigation and, through the tone-map inventory in the same // snapshot, degraded routing until the next clean probe. func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { - if active := s.activeJobs.Load(); active > 0 { + // Claimed for the whole rebuild, not sampled once: a node idle at a + // point-in-time check can accept a transcode milliseconds later and still + // collide with the smoke encode minutes into the probe. The gate is the + // same exclusion the transcode-start path consults, so from here until the + // deferred release no new GPU work is admitted. + busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load())) + if !ok { slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", - "component", "transcodenode", "active_jobs", active) + "component", "transcodenode", "active_jobs", busy) http.Error(w, fmt.Sprintf( "node is running %d transcode job(s); a re-probe smoke-encodes on the GPU and a busy encoder would report working hardware as failed. Retry when the node is idle.", - active), http.StatusConflict) + busy), http.StatusConflict) return } + defer s.gpu.endReprobe() playback.InvalidateHWProbeCache() tonemap.InvalidateProbeCache() - ctx, cancel := context.WithTimeout(r.Context(), s.capabilityProbeBudget()) - defer cancel() - info, err := s.buildCapabilitySnapshot(ctx) + // buildCapabilitySnapshot owns the probe deadline, so a re-probe can never + // cost more than a cold capability fetch already may. + info, err := s.buildCapabilitySnapshot(r.Context()) if err != nil { slog.WarnContext(r.Context(), "transcode node capability re-probe incomplete", "component", "transcodenode", "error", err) @@ -92,16 +97,3 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques slog.WarnContext(r.Context(), "encode transcode node re-probe result", "component", "transcodenode", "error", err) } } - -// capabilityProbeBudget is the deadline one snapshot rebuild gets. It is the -// same budget buildCapabilitySnapshot applies internally, named here so the -// re-probe route is bounded whether or not its caller sent one. -func (s *Server) capabilityProbeBudget() time.Duration { - hwAccel := playback.HWAccelNone - hwDevice := "" - if cfg := s.watcher.Config(); cfg != nil { - hwAccel = cfg.Playback.HWAccel - hwDevice = cfg.Playback.HWDevice - } - return toneMapCapabilityResolveTimeout(hwAccel, hwDevice) -} diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go index 39fdd4db7..3e92ef58b 100644 --- a/internal/transcodenode/reprobe_test.go +++ b/internal/transcodenode/reprobe_test.go @@ -122,3 +122,46 @@ func TestReprobeCapabilitiesRequiresBearer(t *testing.T) { t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) } } + +// The active-job count only moves once ffmpeg is already running, so checking it +// alone leaves a window: a node idle at the check accepts a transcode while the +// probe still has minutes to go, and the smoke encode races the live encoder +// after all. Work that has been admitted but is not yet an active job has to +// refuse the re-probe too. +func TestReprobeCapabilitiesRefusesWhileWorkIsStarting(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + if !server.gpu.beginWork() { + t.Fatal("beginWork on an idle node was refused") + } + t.Cleanup(server.gpu.endWork) + + // activeJobs is deliberately zero: this is exactly the state the old + // point-in-time check read as idle. + if got := server.activeJobs.Load(); got != 0 { + t.Fatalf("active jobs = %d, want the pre-registration window", got) + } + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while a transcode is starting", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// The other direction: while a re-probe holds the encoder, new GPU work is +// refused rather than allowed to collide with the smoke encode. It is refused, +// not queued — a viewer pressing play must not wait out a multi-minute probe, +// and the API retries on another node. +func TestTranscodeStartRefusedWhileReprobing(t *testing.T) { + server := newTestServer(t) + if _, ok := server.gpu.beginReprobe(0); !ok { + t.Fatal("re-probe refused on an idle node") + } + t.Cleanup(server.gpu.endReprobe) + + if server.gpu.beginWork() { + t.Fatal("GPU work admitted while a re-probe held the encoder") + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index db7ffd538..bbf186c61 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -216,6 +216,10 @@ type Server struct { // metrics samples host and GPU resources in the background. Nil until // StartMetricsSampler runs, which leaves health exactly as it was before. metrics *nodemetrics.Sampler + + // gpu keeps a capability re-probe's smoke encode and real transcodes off + // the encoder at the same time; see gpuGate. + gpu gpuGate } // storedCapabilityHash returns the last published capability hash, or empty @@ -635,6 +639,13 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { if !s.requireApprovedInputPath(w, r, req.InputPath) { return } + // A prepared download encodes on the GPU like any transcode, so it takes + // the same exclusion against a running capability re-probe. + if !s.gpu.beginWork() { + http.Error(w, "node is re-probing its hardware; retry shortly", http.StatusServiceUnavailable) + return + } + defer s.gpu.endWork() opts := req.TranscodeOpts(cfg.Playback.FFmpegPath, cfg.Playback.HWAccel, cfg.Playback.HWDevice, s.ffmpegSink) artifactRoot := s.artifactRoot if err := os.MkdirAll(artifactRoot, 0o755); err != nil { @@ -993,7 +1004,15 @@ func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelI defer cancel() // One detection walk answers both questions: Resolved honors the configured // backend's pass-through contract, and DetectedBackends explains it. - info := playback.DetectHWAccelWithFFmpegContext(resolveCtx, configuredHWAccel, ffmpegPath, hwDevice) + // + // A walk that ran out of budget is refused rather than published: it marks + // unprobed backends Verified=false, which is byte-identical to a real + // hardware failure, so hashing it would make the API persist a capability + // regression for hardware that is fine. + info, err := playback.DetectHWAccelWithFFmpegContextResult(resolveCtx, configuredHWAccel, ffmpegPath, hwDevice) + if err != nil { + return playback.HWAccelInfo{}, err + } info.ProbeRequestTimeoutMillis = tonemap.ProbeRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), info.Resolved, hwDevice) if err != nil { @@ -1166,6 +1185,15 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { http.Error(w, "node not configured", http.StatusServiceUnavailable) return } + // Held from here until the handler returns, by which point activeJobs + // covers the session. Without the overlap a re-probe could start in the gap + // between admitting this transcode and it becoming visible as an active + // job, and its smoke encode would then race a live encoder session. + if !s.gpu.beginWork() { + http.Error(w, "node is re-probing its hardware; retry shortly", http.StatusServiceUnavailable) + return + } + defer s.gpu.endWork() outputDir := s.sessionOutputDir(req.SessionID) opts := playback.TranscodeOpts{ @@ -1467,6 +1495,15 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe if cfg == nil { return nil, nil } + // A reconstruct spawns ffmpeg on the GPU exactly as a fresh start does, so + // it takes the same exclusion against a running capability re-probe. Held + // until this call returns, by which point activeJobs covers the session. + if !s.gpu.beginWork() { + slog.InfoContext(r.Context(), "transcode node reconstruct deferred while re-probing hardware", + "component", "transcodenode", "session", sessionID) + return nil, nil + } + defer s.gpu.endWork() outputDir := s.sessionOutputDir(sessionID) opts := card.TranscodeOpts(outputDir, cfg.Playback.FFmpegPath, s.ffmpegSink) opts.SessionID = sessionID From 755c624f5e38c7699e676f32d268bac7facb7109 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:52:54 -0400 Subject: [PATCH 012/163] feat(admin): dashboard insight widgets, metrics sampler, and server-side layouts Builds the insights phase of the admin dashboard rebuild: - internal/dashmetrics: minute-resolution sampler for concurrent streams (by play method) and egress (node egress + per-process viewer egress from stream telemetry), replica-safe via idempotent inserts, 31-day retention. - New aggregate endpoints following the AdminStatsProvider pattern: /admin/stats/timeseries (adaptive peak-preserving bucketing, 1h-31d), /admin/stats/playback-activity (hourly/daily buckets from playback history unioned with live sessions, reliability counters), /admin/stats/top-activity (top titles and profiles from watch history). - /admin/server/status gains a health object (Postgres/Redis pings, 24h error and warning counts); /admin/logs/app accepts a comma-separated level list. - Server-side per-admin dashboard layouts (admin_dashboard_layouts table, GET/PUT/DELETE /admin/dashboard/layout) with localStorage as the instant-paint fallback and one-time migration of existing local layouts. - 13 new widgets (playback activity, streams/egress trends, reliability, top titles/profiles, nodes, scanner, scan activity, recent errors, health strip, and three stat tiles) on hand-rolled SVG chart primitives using re-stepped, accessibility-validated chart tokens. - Widgets resize on both axes via a corner drag handle (12-column x 100px-row grid, keyboard accessible), and metric widgets carry a persisted 1h/24h/7d/30d range picker. - Users widget sorts by last activity instead of account id. Time-to-first-frame and failed-start metrics are documented as future work; no data source exists for them yet. Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 31 ++ docs/admin-api.md | 279 ++++++++++ internal/api/handlers/admin.go | 12 + .../api/handlers/admin_dashboard_layout.go | 161 ++++++ .../handlers/admin_dashboard_layout_test.go | 222 ++++++++ internal/api/handlers/admin_logs.go | 38 +- internal/api/handlers/admin_server_status.go | 136 ++++- .../api/handlers/admin_server_status_test.go | 109 ++++ internal/api/handlers/admin_stats_playback.go | 370 +++++++++++++ .../api/handlers/admin_stats_playback_test.go | 325 +++++++++++ .../api/handlers/admin_stats_timeseries.go | 340 ++++++++++++ .../handlers/admin_stats_timeseries_test.go | 300 ++++++++++ internal/api/handlers/admin_stats_top.go | 294 ++++++++++ internal/api/handlers/admin_stats_top_test.go | 213 +++++++ internal/api/router.go | 25 + internal/api/testdata/media_routes.txt | 6 + internal/dashmetrics/egress.go | 47 ++ internal/dashmetrics/sampler.go | 226 ++++++++ internal/dashmetrics/sampler_test.go | 257 +++++++++ internal/opslog/repo.go | 143 +++-- internal/opslog/repo_test.go | 138 +++++ ...20260827013713_admin_dashboard_layouts.sql | 11 + ...60827015256_dashboard_activity_indexes.sql | 16 + ...0260827021132_dashboard_metric_samples.sql | 24 + web/src/api/types.ts | 122 +++++ web/src/app.css | 73 ++- .../admin/dashboard/DashboardGrid.tsx | 150 ++++- .../dashboard/WidgetRangePicker.test.tsx | 85 +++ .../admin/dashboard/WidgetRangePicker.tsx | 82 +++ .../admin/dashboard/charts/BarList.tsx | 98 ++++ .../dashboard/charts/ChartEmptyState.tsx | 67 +++ .../admin/dashboard/charts/LineChart.tsx | 283 ++++++++++ .../admin/dashboard/charts/Sparkline.tsx | 75 +++ .../dashboard/charts/StackedColumnChart.tsx | 191 +++++++ .../admin/dashboard/charts/chartChrome.tsx | 125 +++++ .../admin/dashboard/charts/chartMath.test.ts | 278 ++++++++++ .../admin/dashboard/charts/chartMath.ts | 245 +++++++++ .../admin/dashboard/charts/index.ts | 43 ++ .../admin/dashboard/charts/useMeasuredSize.ts | 48 ++ web/src/components/admin/dashboard/format.ts | 106 ++++ web/src/components/admin/dashboard/range.ts | 128 +++++ .../components/admin/dashboard/registry.tsx | 295 +++++++++- web/src/components/admin/dashboard/types.ts | 47 ++ .../dashboard/useDashboardLayout.test.ts | 518 ++++++++++++++++-- .../admin/dashboard/useDashboardLayout.ts | 307 +++++++++-- .../admin/dashboard/widgetChrome.tsx | 59 ++ .../widgets/ConcurrentStreamsWidget.tsx | 65 +++ .../admin/dashboard/widgets/EgressWidget.tsx | 65 +++ .../widgets/HealthStripWidget.test.tsx | 171 ++++++ .../dashboard/widgets/HealthStripWidget.tsx | 170 ++++++ .../dashboard/widgets/LibrariesWidget.tsx | 4 +- .../dashboard/widgets/NowPlayingWidget.tsx | 56 +- .../widgets/PlaybackActivityWidget.test.tsx | 203 +++++++ .../widgets/PlaybackActivityWidget.tsx | 74 +++ .../widgets/PlaybackReliabilityWidget.tsx | 92 ++++ .../widgets/RecentActivityWidget.tsx | 4 +- .../widgets/RecentErrorsWidget.test.tsx | 105 ++++ .../dashboard/widgets/RecentErrorsWidget.tsx | 100 ++++ .../dashboard/widgets/ScanActivityWidget.tsx | 129 +++++ .../admin/dashboard/widgets/ScannerWidget.tsx | 180 ++++++ .../dashboard/widgets/TopProfilesWidget.tsx | 74 +++ .../dashboard/widgets/TopTitlesWidget.tsx | 68 +++ .../widgets/TranscodeNodesWidget.tsx | 140 +++++ .../admin/dashboard/widgets/UsersWidget.tsx | 41 +- .../widgets/playbackActivitySeries.ts | 67 +++ .../admin/dashboard/widgets/statTiles.tsx | 85 ++- .../dashboard/widgets/timeseriesChart.tsx | 85 +++ .../dashboard/widgets/timeseriesSeries.ts | 74 +++ .../queries/admin/dashboardInsights.test.ts | 154 ++++++ .../hooks/queries/admin/dashboardInsights.ts | 93 ++++ .../queries/admin/dashboardLayout.test.ts | 163 ++++++ .../hooks/queries/admin/dashboardLayout.ts | 67 +++ web/src/hooks/queries/admin/logs.ts | 1 + web/src/hooks/queries/keys.ts | 12 + web/src/pages/AdminDashboard.tsx | 23 +- 75 files changed, 9469 insertions(+), 244 deletions(-) create mode 100644 internal/api/handlers/admin_dashboard_layout.go create mode 100644 internal/api/handlers/admin_dashboard_layout_test.go create mode 100644 internal/api/handlers/admin_stats_playback.go create mode 100644 internal/api/handlers/admin_stats_playback_test.go create mode 100644 internal/api/handlers/admin_stats_timeseries.go create mode 100644 internal/api/handlers/admin_stats_timeseries_test.go create mode 100644 internal/api/handlers/admin_stats_top.go create mode 100644 internal/api/handlers/admin_stats_top_test.go create mode 100644 internal/dashmetrics/egress.go create mode 100644 internal/dashmetrics/sampler.go create mode 100644 internal/dashmetrics/sampler_test.go create mode 100644 internal/opslog/repo_test.go create mode 100644 migrations/sql/20260827013713_admin_dashboard_layouts.sql create mode 100644 migrations/sql/20260827015256_dashboard_activity_indexes.sql create mode 100644 migrations/sql/20260827021132_dashboard_metric_samples.sql create mode 100644 web/src/components/admin/dashboard/WidgetRangePicker.test.tsx create mode 100644 web/src/components/admin/dashboard/WidgetRangePicker.tsx create mode 100644 web/src/components/admin/dashboard/charts/BarList.tsx create mode 100644 web/src/components/admin/dashboard/charts/ChartEmptyState.tsx create mode 100644 web/src/components/admin/dashboard/charts/LineChart.tsx create mode 100644 web/src/components/admin/dashboard/charts/Sparkline.tsx create mode 100644 web/src/components/admin/dashboard/charts/StackedColumnChart.tsx create mode 100644 web/src/components/admin/dashboard/charts/chartChrome.tsx create mode 100644 web/src/components/admin/dashboard/charts/chartMath.test.ts create mode 100644 web/src/components/admin/dashboard/charts/chartMath.ts create mode 100644 web/src/components/admin/dashboard/charts/index.ts create mode 100644 web/src/components/admin/dashboard/charts/useMeasuredSize.ts create mode 100644 web/src/components/admin/dashboard/range.ts create mode 100644 web/src/components/admin/dashboard/widgetChrome.tsx create mode 100644 web/src/components/admin/dashboard/widgets/ConcurrentStreamsWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/EgressWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/HealthStripWidget.test.tsx create mode 100644 web/src/components/admin/dashboard/widgets/HealthStripWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.test.tsx create mode 100644 web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/PlaybackReliabilityWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/RecentErrorsWidget.test.tsx create mode 100644 web/src/components/admin/dashboard/widgets/RecentErrorsWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/ScanActivityWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/ScannerWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/TopProfilesWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/TopTitlesWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/TranscodeNodesWidget.tsx create mode 100644 web/src/components/admin/dashboard/widgets/playbackActivitySeries.ts create mode 100644 web/src/components/admin/dashboard/widgets/timeseriesChart.tsx create mode 100644 web/src/components/admin/dashboard/widgets/timeseriesSeries.ts create mode 100644 web/src/hooks/queries/admin/dashboardInsights.test.ts create mode 100644 web/src/hooks/queries/admin/dashboardInsights.ts create mode 100644 web/src/hooks/queries/admin/dashboardLayout.test.ts create mode 100644 web/src/hooks/queries/admin/dashboardLayout.ts diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 31ec0f1aa..850647515 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -53,6 +53,7 @@ import ( "github.com/Silo-Server/silo-server/internal/chapterthumbs" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/dashmetrics" "github.com/Silo-Server/silo-server/internal/database" "github.com/Silo-Server/silo-server/internal/diagnostics" "github.com/Silo-Server/silo-server/internal/downloads" @@ -1909,6 +1910,36 @@ func main() { } defer adminStatsProvider.Close() deps.AdminStatsProvider = adminStatsProvider + + playbackActivityProvider, playbackActivityErr := handlers.NewAdminPlaybackActivityProvider(appCtx, deps.DB, deps.EventBus) + if playbackActivityErr != nil { + log.Fatalf("failed to create admin playback activity provider: %v", playbackActivityErr) + } + defer playbackActivityProvider.Close() + deps.AdminPlaybackActivityProvider = playbackActivityProvider + + topActivityProvider, topActivityErr := handlers.NewAdminTopActivityProvider(appCtx, deps.DB, deps.EventBus) + if topActivityErr != nil { + log.Fatalf("failed to create admin top activity provider: %v", topActivityErr) + } + defer topActivityProvider.Close() + deps.AdminTopActivityProvider = topActivityProvider + + timeseriesProvider, timeseriesErr := handlers.NewAdminTimeseriesProvider(appCtx, deps.DB, deps.EventBus) + if timeseriesErr != nil { + log.Fatalf("failed to create admin timeseries provider: %v", timeseriesErr) + } + defer timeseriesProvider.Close() + deps.AdminTimeseriesProvider = timeseriesProvider + + // The dashboard metrics sampler is the only writer of concurrent-stream + // and egress history. Proxy and transcode nodes serve bytes but do not + // own the catalog database, so only the API-facing modes sample. + if mode == "integrated" || mode == "api" { + dashSampler := dashmetrics.NewSampler(deps.DB, deps.StreamTelemetry, nodeIdentity) + dashSampler.Start(appCtx) + defer dashSampler.Stop() + } } // Wire recommendations engine, worker, and ratings repo if enabled. diff --git a/docs/admin-api.md b/docs/admin-api.md index f29fb8b46..9754b22b0 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -80,3 +80,282 @@ Each entry in `sources`: A single report samples three independently updated stores, so one-sided differences are normal and are not on their own evidence of a defect. Repeated agreement over time is what the legacy-retirement project is gated on. + +## `/api/v1/admin/dashboard/layout` + +The admin dashboard is a widget grid each admin arranges for themselves. The +arrangement is stored per **account** (`users.id`), not per household profile, +so the same admin sees the same dashboard in every browser they log in from. + +The server stores the document verbatim and validates only that the body is at +most 16 KiB and that `layout` is a JSON object. Widget ids, column spans and row +heights are the admin web client's vocabulary: it already sanitizes what it +loads — dropping widgets it does not know, clamping each axis to that widget's +range, and filling in the default height for an entry saved before row heights +existed — so a second copy of that schema on the server would only be another +place to update whenever a widget is added. That also means a layout written by +a newer build degrades gracefully on an older one instead of being rejected. + +Writes are last-write-wins. The layout is one admin's own blob, so a race +between two of their tabs can cost only the older arrangement; `updated_at` is +returned so a compare-and-set could be layered on later without a contract +change. + +The web client keeps a copy in `localStorage` for instant paint and offline use, +adopts the server document when it arrives, and — the first time it finds no +server document but does have a local one — uploads that local layout once. + +### `GET /api/v1/admin/dashboard/layout` + +`200 OK`. Both fields are `null` when this admin has never saved a layout; that +is the normal first-load answer, not an error. + +| Field | Type | Meaning | +|---|---|---| +| `layout` | object \| null | The stored document, exactly as it was written. | +| `updated_at` | RFC3339 string \| null | When it was last written. | + +```json +{ + "layout": { + "version": 1, + "entries": [{ "id": "libraries", "span": 7, "rows": 4 }] + }, + "updated_at": "2026-08-26T10:00:00Z" +} +``` + +### `PUT /api/v1/admin/dashboard/layout` + +Body: `{"layout": {…}}`. Responds `204 No Content` on success, and +`400 bad_request` when the body is not valid JSON, when `layout` is absent or +`null`, when `layout` is not a JSON object, or when the body exceeds 16 KiB. + +### `DELETE /api/v1/admin/dashboard/layout` + +Resets this admin to the default arrangement. `204 No Content`, and idempotent: +deleting a layout that is not there succeeds. + +## `GET /api/v1/admin/stats/timeseries` + +Sampled history for the concurrent-streams and egress charts. Cached in-process +for 30s, dropped early on playback or admin activity, and bypassed with +`?refresh=1`. + +| Parameter | Type | Meaning | +|---|---|---| +| `hours` | int | Window length. Default 24, clamped to 1..744 (31 days, the retention window). A non-numeric value is `400 bad_request`. | +| `refresh` | bool | Bypass the cache for this read. | + +Neither series can be reconstructed after the fact — live sessions leave no +per-minute trace once they end, and node egress is a rolling average that each +health check overwrites — so a sampler (`internal/dashmetrics`) writes them as +they happen, once a minute, into `dashboard_metric_samples`. Samples older than +31 days are deleted. + +Reads bucket those minutes down so a response stays under ~750 points at any +window. `resolution_seconds` reports the bucket that was used — read it rather +than assuming the sampler's minute: + +| Requested window | `resolution_seconds` | +|---|---| +| ≤ 2 hours | 60 | +| ≤ 48 hours | 300 | +| ≤ 336 hours (14 days) | 1800 | +| wider | 7200 | + +A bucket wider than a minute reports the **peak** minute of each column, never +an average: these charts are read to answer "how bad did it get", and a mean +would erase exactly that. Stream counts and egress are maxed independently, so +a bucket's columns may come from different minutes within it. + +Each minute holds up to two kinds of row. The `shared` row is the cluster-wide +snapshot: stream counts by play method, plus the egress reported by enabled, +healthy stream nodes. Every replica tries to write it and the first one to land +wins, so the values for a minute come from whichever replica got there first — +they differ only by sub-second timing. A `proc:` row per API process +carries the viewer egress that process served, measured from stream telemetry; +without it a deployment with no stream nodes would chart zero egress forever. +Relay traffic is excluded, so bytes a proxy node passes through the API node are +not counted twice. + +Stream counts in a point therefore come from the shared row, while `egress_kbps` +sums every source for a minute before the peak minute of the bucket is taken. +Precision is mixed by design: node egress is a 30-second rolling average and +process egress is an exact byte delta. + +A bucket with no sample in it is absent from `points` rather than zero — a gap +(a restart, a stopped server) and an idle bucket are different facts. Stream +telemetry being disabled means no `proc:` rows, not an error. +`oldest_sample_at` is `null` until the first sample exists, which is how a +fresh install renders "collecting data" instead of an empty chart. + +```json +{ + "resolution_seconds": 300, + "from": "2026-08-25T12:00:00Z", + "to": "2026-08-26T12:00:00Z", + "oldest_sample_at": "2026-08-24T09:31:00Z", + "points": [ + { + "t": "2026-08-26T11:55:00Z", + "streams": 3, + "direct": 1, + "remux": 0, + "transcode": 2, + "egress_kbps": 48211 + } + ] +} +``` + +## `GET /api/v1/admin/stats/playback-activity` + +Bucketed playback starts split by play method, plus reliability scalars, for the +admin dashboard. Answers are cached in-process for 60s and dropped early when +the shared event bus reports playback or admin activity; `?refresh=1` drops the +cache before reading. + +| Parameter | Type | Meaning | +|---|---|---| +| `hours` | int | Window length. Default 24, clamped to 1..744. A non-numeric value is `400 bad_request`. | +| `refresh` | bool | Bypass the cache for this read. | + +Buckets are hourly up to a 48-hour window and daily beyond it; `bucket_seconds` +is `3600` or `86400` accordingly. A bucket's `hour` field is its start instant +at either width — it keeps that name because it is the same fact, and +`bucket_seconds` already says how wide the bucket is. + +Sessions come from `playback_history_admin` (which only gains a row when a +session finalizes) unioned with the live sessions table, so the current hour is +not under-counted. A live session cannot already be in history, so nothing is +counted twice. Live sessions with no recorded start — reconstructed after a +restart — are dated by their last update instead. + +`buckets` contains only buckets that saw a session; the client zero-fills the +window on the `bucket_seconds` grid so a quiet server draws empty columns rather +than a shorter chart. Everything in `reliability` is computed over the whole +requested window. `completion_rate` is +`completed_sessions / finalized_sessions`: live sessions are excluded from both +sides, because a session that is still playing has not failed to complete. + +`profiles_active_24h` is a fixed rolling-24h figure that ignores `hours` — it +answers "who watched today" whatever window the chart beside it is showing. It +counts distinct (account, profile) pairs in +`user_watch_history` over a rolling 24 hours, excluding history that was +imported or synced from a watch provider (`import`, `trakt`, `simkl`, +`mdblist`), so it means "watched on this server". Marked-watched (`manual`) +rows are counted: they are on-server actions. + +**Not reported:** time-to-first-frame and failed-start counts. Nothing records +a playback *start* event today, so both would have to be inferred from log +parsing. They need start-event capture in playback first, and are deliberately +absent rather than approximated. + +```json +{ + "hours": 24, + "bucket_seconds": 3600, + "buckets": [{ "hour": "2026-08-26T10:00:00Z", "direct": 4, "remux": 1, "transcode": 2 }], + "reliability": { + "sessions_started": 42, + "transcode_starts": 11, + "finalized_sessions": 38, + "completed_sessions": 27, + "completion_rate": 0.7105, + "unique_profiles": 9 + }, + "profiles_active_24h": 9 +} +``` + +## `GET /api/v1/admin/stats/top-activity` + +Most-watched titles and most-active profiles over a multi-day window. Cached +for 5 minutes — a seven-day ranking barely moves within minutes — with the same +`?refresh=1` escape hatch. + +| Parameter | Type | Meaning | +|---|---|---| +| `days` | int | Window length. Default 7, clamped to 1..30. | +| `limit` | int | Rows per list. Default 10, clamped to 1..25. | +| `refresh` | bool | Bypass the cache for this read. | + +Both lists read `user_watch_history` with the same source exclusions as +`profiles_active_24h` above. Episodes are rolled up to their series, so a +season binge reads as one show and a title's `media_item_id` is a series +content id for TV. Profile display names live in the per-user stores rather +than in watch history, so they are read back from that profile's most recent +`playback_history_admin` row; a profile that has only ever marked things +watched falls back to its profile id. No poster URLs are returned — the +bar-list widgets do not need them, and it keeps the query cheap. + +Both lists are `[]` on a server with no history, never `null`. + +```json +{ + "days": 7, + "limit": 10, + "titles": [ + { + "media_item_id": "…", + "title": "…", + "media_type": "series", + "plays": 18, + "total_seconds": 54120 + } + ], + "profiles": [ + { + "user_id": 3, + "username": "quick", + "profile_id": "p1", + "profile_name": "Quick", + "plays": 12, + "total_seconds": 40100 + } + ] +} +``` + +## `GET /api/v1/admin/server/status` — `health` + +The status route carries an additive `health` object for the dashboard health +strip. Every field the route already returned is unchanged; only `health` is +new, and the example below is trimmed to the fields it discusses: + +```json +{ + "started_at": "2026-08-26T09:00:00Z", + "restart_required": false, + "health": { + "postgres": { "configured": true, "ok": true, "latency_ms": 1.42 }, + "redis": { "configured": true, "ok": true, "latency_ms": 0.31 }, + "errors_24h": 4, + "warnings_24h": 12 + } +} +``` + +Each component reports `configured` first: `false` means this deployment runs +without that service — a supported single-node shape for Redis — and `ok` and +`latency_ms` are then absent, so "not present" and "present but broken" do not +look the same on the strip. Latency is the round trip of one ping, in +milliseconds with two decimals, bounded by a 2s timeout: a wedged dependency is +reported as `ok: false` rather than holding the route open. + +`errors_24h` / `warnings_24h` count `operational_logs` rows at those levels over +a rolling 24 hours, cached for 30s. A server with operational logging disabled +reports zeros and logs a warning; this route never fails over a secondary +number. + +Version, uptime and node health are not repeated here. The client composes them +from `GET /admin/system/build`, `started_at` above, and `GET /admin/nodes`. + +## `GET /api/v1/admin/logs/app` — `level` + +`level` accepts a comma-separated list, so one request can ask for several +levels at once (`?level=error,warn`). Values are trimmed, lowercased and +de-duplicated; a single value behaves exactly as before. The same parsing +applies to the log-stream WebSocket, so a stream filtered on two levels +delivers both. diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 1a650092a..860c1cc65 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -20,6 +20,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" "golang.org/x/sync/errgroup" "github.com/Silo-Server/silo-server/internal/access" @@ -123,6 +124,10 @@ type AdminHandler struct { accountProvisioner *auth.AccountProvisioner DetailSvc *catalog.DetailService StatsSource AdminStatsSource + PlaybackActivitySource AdminPlaybackActivitySource + TopActivitySource AdminTopActivitySource + TimeseriesSource AdminTimeseriesSource + RedisClient *redis.Client // health reporting only; nil means this deployment runs without Redis Config *config.Config EventBus cache.EventBus EventsHub *evt.Hub @@ -140,6 +145,12 @@ type AdminHandler struct { OnServerSettingUpdated func(ctx context.Context, key, value string) RestartStatus *ServerRestartStatusTracker CatalogSearchStatus catalog.CatalogSearchStatusProvider + // logLevelCounts caches the 24h error/warning tallies served on + // /admin/server/status. The dashboard polls that route every 15s, and the + // counts are only ever read as a rough signal, so re-counting per request + // would be pure waste. Nil when the handler was built without the + // constructor (tests): the counts are then simply uncached. + logLevelCounts *cache.TTLCache[adminLogLevelCounts] } // NewAdminHandler creates a new AdminHandler backed by the given @@ -154,6 +165,7 @@ func NewAdminHandler( pool: pool, storeProv: storeProv, accountProvisioner: auth.NewAccountProvisioner(userRepo, storeProv), + logLevelCounts: cache.NewTTLCache[adminLogLevelCounts](), } } diff --git a/internal/api/handlers/admin_dashboard_layout.go b/internal/api/handlers/admin_dashboard_layout.go new file mode 100644 index 000000000..ee717f643 --- /dev/null +++ b/internal/api/handlers/admin_dashboard_layout.go @@ -0,0 +1,161 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "time" + + "github.com/jackc/pgx/v5" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" +) + +// maxDashboardLayoutBytes bounds the PUT body. The layout is a short list of +// widget ids and spans; 16 KiB leaves generous headroom while keeping the blob +// small enough that last-write-wins per admin account stays cheap. +const maxDashboardLayoutBytes = 16 << 10 + +// adminDashboardLayoutResponse is the GET body. Both fields are null when the +// admin has never saved a layout, which the web client reads as "keep the +// local/default layout" rather than as an error. +type adminDashboardLayoutResponse struct { + Layout json.RawMessage `json:"layout"` + UpdatedAt *time.Time `json:"updated_at"` +} + +type adminDashboardLayoutRequest struct { + Layout json.RawMessage `json:"layout"` +} + +// Sentinel validation failures. Their text is the message the client sees, so +// it stays lowercase (staticcheck ST1005) and reads as a sentence fragment. +var ( + errDashboardLayoutInvalidJSON = errors.New("request body must be valid JSON") + errDashboardLayoutMissing = errors.New("layout is required") + errDashboardLayoutNotObject = errors.New("layout must be a JSON object") +) + +// parseDashboardLayoutPayload validates a PUT body and returns the document to +// store. The server treats the layout as opaque past requiring a JSON object: +// widget ids and spans are the web client's vocabulary, and it already +// sanitizes them on load, so validating them here would only add a second +// place to update whenever a widget is added. +func parseDashboardLayoutPayload(body []byte) (json.RawMessage, error) { + var req adminDashboardLayoutRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, errDashboardLayoutInvalidJSON + } + // Unmarshal already checked the syntax of the whole document, so the first + // non-space byte is enough to tell an object from any other JSON value. + raw := json.RawMessage(bytes.TrimSpace(req.Layout)) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return nil, errDashboardLayoutMissing + } + if raw[0] != '{' { + return nil, errDashboardLayoutNotObject + } + return raw, nil +} + +// HandleGetDashboardLayout handles GET /admin/dashboard/layout. +func (h *AdminHandler) HandleGetDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + var ( + layout json.RawMessage + updatedAt time.Time + ) + err := h.pool.QueryRow(r.Context(), + `SELECT layout, updated_at FROM admin_dashboard_layouts WHERE user_id = $1`, + userID, + ).Scan(&layout, &updatedAt) + switch { + case errors.Is(err, pgx.ErrNoRows): + writeJSON(w, http.StatusOK, adminDashboardLayoutResponse{}) + return + case err != nil: + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load dashboard layout") + return + } + + writeJSON(w, http.StatusOK, adminDashboardLayoutResponse{Layout: layout, UpdatedAt: &updatedAt}) +} + +// HandlePutDashboardLayout handles PUT /admin/dashboard/layout. +func (h *AdminHandler) HandlePutDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxDashboardLayoutBytes)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeError(w, http.StatusBadRequest, "bad_request", "Dashboard layout is too large") + return + } + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + + layout, err := parseDashboardLayoutPayload(body) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + // Last write wins. The layout is a per-admin blob, so a race between two of + // the same admin's tabs can only cost the older arrangement; updated_at is + // returned by GET so a compare-and-set could be layered on later. + if _, err := h.pool.Exec(r.Context(), + `INSERT INTO admin_dashboard_layouts (user_id, layout, updated_at) + VALUES ($1, $2, now()) + ON CONFLICT (user_id) DO UPDATE SET layout = EXCLUDED.layout, updated_at = now()`, + userID, []byte(layout), + ); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save dashboard layout") + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// HandleDeleteDashboardLayout handles DELETE /admin/dashboard/layout. Deleting +// the row resets the admin to the default layout; it is idempotent. +func (h *AdminHandler) HandleDeleteDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + if _, err := h.pool.Exec(r.Context(), + `DELETE FROM admin_dashboard_layouts WHERE user_id = $1`, userID, + ); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to reset dashboard layout") + return + } + + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/handlers/admin_dashboard_layout_test.go b/internal/api/handlers/admin_dashboard_layout_test.go new file mode 100644 index 000000000..28004b9d1 --- /dev/null +++ b/internal/api/handlers/admin_dashboard_layout_test.go @@ -0,0 +1,222 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/auth" +) + +func TestParseDashboardLayoutPayload(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + want string + wantErr error + }{ + { + name: "layout object is stored verbatim", + body: `{"layout":{"version":1,"entries":[{"id":"libraries","span":7}]}}`, + want: `{"version":1,"entries":[{"id":"libraries","span":7}]}`, + }, + { + name: "empty object is a valid layout", + body: `{"layout":{}}`, + want: `{}`, + }, + { + name: "unknown widget ids are not the server's business", + body: `{"layout":{"entries":[{"id":"not-a-widget","span":99}]}}`, + want: `{"entries":[{"id":"not-a-widget","span":99}]}`, + }, + { + name: "missing layout", + body: `{}`, + wantErr: errDashboardLayoutMissing, + }, + { + name: "explicit null layout", + body: `{"layout":null}`, + wantErr: errDashboardLayoutMissing, + }, + { + name: "array layout", + body: `{"layout":[{"id":"libraries","span":7}]}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "string layout", + body: `{"layout":"libraries"}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "number layout", + body: `{"layout":7}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "malformed json", + body: `{"layout":`, + wantErr: errDashboardLayoutInvalidJSON, + }, + { + name: "empty body", + body: ``, + wantErr: errDashboardLayoutInvalidJSON, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseDashboardLayoutPayload([]byte(tt.body)) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if got != nil { + t.Fatalf("layout = %s, want nil on error", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != tt.want { + t.Fatalf("layout = %s, want %s", got, tt.want) + } + }) + } +} + +// newDashboardLayoutRequest builds an authenticated admin request. The routes +// are admin-gated by the surrounding route group, so the handler only needs a +// user id in the claims. +func newDashboardLayoutRequest(method, body string, userID int) *http.Request { + var req *http.Request + if body == "" { + req = httptest.NewRequest(method, "/admin/dashboard/layout", nil) + } else { + req = httptest.NewRequest(method, "/admin/dashboard/layout", strings.NewReader(body)) + } + if userID == 0 { + return req + } + return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: userID})) +} + +func TestDashboardLayoutHandlersRequireDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + for _, tc := range []struct { + name string + method string + body string + serve func(http.ResponseWriter, *http.Request) + }{ + {"get", http.MethodGet, "", handler.HandleGetDashboardLayout}, + {"put", http.MethodPut, `{"layout":{}}`, handler.HandlePutDashboardLayout}, + {"delete", http.MethodDelete, "", handler.HandleDeleteDashboardLayout}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + tc.serve(rec, newDashboardLayoutRequest(tc.method, tc.body, 7)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } + }) + } +} + +// newUnreachableLayoutHandler returns a handler whose pool never connects, so +// every request that reaches SQL fails. Tests using it assert on the checks +// that run *before* the database is touched. +func newUnreachableLayoutHandler(t *testing.T) *AdminHandler { + t.Helper() + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + return &AdminHandler{pool: pool} +} + +func TestDashboardLayoutHandlersRequireAuthenticatedUser(t *testing.T) { + t.Parallel() + + handler := newUnreachableLayoutHandler(t) + for _, tc := range []struct { + name string + method string + body string + serve func(http.ResponseWriter, *http.Request) + }{ + {"get", http.MethodGet, "", handler.HandleGetDashboardLayout}, + {"put", http.MethodPut, `{"layout":{}}`, handler.HandlePutDashboardLayout}, + {"delete", http.MethodDelete, "", handler.HandleDeleteDashboardLayout}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + tc.serve(rec, newDashboardLayoutRequest(tc.method, tc.body, 0)) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401, body = %s", rec.Code, rec.Body.String()) + } + }) + } +} + +func TestPutDashboardLayoutRejectsInvalidBodies(t *testing.T) { + t.Parallel() + + handler := newUnreachableLayoutHandler(t) + tests := []struct { + name string + body string + wantMessage string + }{ + {"not an object", `{"layout":[]}`, errDashboardLayoutNotObject.Error()}, + {"missing layout", `{}`, errDashboardLayoutMissing.Error()}, + {"malformed", `nope`, errDashboardLayoutInvalidJSON.Error()}, + { + name: "over the size limit", + body: `{"layout":{"pad":"` + strings.Repeat("x", maxDashboardLayoutBytes) + `"}}`, + wantMessage: "Dashboard layout is too large", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + handler.HandlePutDashboardLayout(rec, newDashboardLayoutRequest(http.MethodPut, tt.body, 7)) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + var resp errorResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode error body: %v", err) + } + if resp.Message != tt.wantMessage { + t.Fatalf("message = %q, want %q", resp.Message, tt.wantMessage) + } + }) + } +} diff --git a/internal/api/handlers/admin_logs.go b/internal/api/handlers/admin_logs.go index 50af411f9..27bdc7fbf 100644 --- a/internal/api/handlers/admin_logs.go +++ b/internal/api/handlers/admin_logs.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log/slog" "net/http" + "slices" "strconv" "strings" "time" @@ -58,7 +59,10 @@ func (h *AdminLogsHandler) HandleListAuditLogs(w http.ResponseWriter, r *http.Re func parseOperationalLogOptionsFromRequest(r *http.Request) (opslog.ListOptions, error) { opts := opslog.ListOptions{ - Level: strings.TrimSpace(r.URL.Query().Get("level")), + // `level` accepts a comma-separated list so a single request can ask + // for, say, errors and warnings together — what the dashboard's + // recent-errors widget needs. + Levels: opslog.NormalizeLevels(strings.Split(r.URL.Query().Get("level"), ",")), Component: strings.TrimSpace(r.URL.Query().Get("component")), NodeID: strings.TrimSpace(r.URL.Query().Get("node_id")), RequestID: strings.TrimSpace(r.URL.Query().Get("request_id")), @@ -152,6 +156,32 @@ func parseOptionalIntQuery(r *http.Request, key string) (*int, error) { return &value, nil } +// parseClampedIntQuery reads an optional integer query parameter and clamps it +// into [minValue, maxValue]. An absent or empty parameter yields the clamped +// fallback; a non-numeric one is a bad request rather than a silent default, +// so a client typo surfaces instead of quietly returning the wrong window. +func parseClampedIntQuery(r *http.Request, key string, fallback, minValue, maxValue int) (int, error) { + raw := strings.TrimSpace(r.URL.Query().Get(key)) + if raw == "" { + return clampQueryInt(fallback, minValue, maxValue), nil + } + value, err := strconv.Atoi(raw) + if err != nil { + return 0, invalidQueryError(key) + } + return clampQueryInt(value, minValue, maxValue), nil +} + +func clampQueryInt(value, minValue, maxValue int) int { + if value < minValue { + return minValue + } + if value > maxValue { + return maxValue + } + return value +} + func invalidQueryError(key string) error { return &requestParseError{message: "Invalid " + key} } @@ -428,7 +458,11 @@ func matchesOperationalLog(opts opslog.ListOptions, entry opslog.EntryRow) bool if opts.To != nil && entry.Timestamp.After(*opts.To) { return false } - if opts.Level != "" && entry.Level != strings.ToLower(opts.Level) { + if levels := opslog.NormalizeLevels(opts.Levels); len(levels) > 0 { + if !slices.Contains(levels, strings.ToLower(entry.Level)) { + return false + } + } else if opts.Level != "" && entry.Level != strings.ToLower(opts.Level) { return false } if opts.Component != "" && entry.Component != opts.Component { diff --git a/internal/api/handlers/admin_server_status.go b/internal/api/handlers/admin_server_status.go index eb436bd64..5529de7e1 100644 --- a/internal/api/handlers/admin_server_status.go +++ b/internal/api/handlers/admin_server_status.go @@ -1,21 +1,60 @@ package handlers import ( + "context" + "log/slog" + "math" "net/http" "time" "github.com/Silo-Server/silo-server/internal/jellycompat" ) +const ( + // adminHealthProbeTimeout bounds each dependency probe. A wedged Postgres + // or Redis must not hold the status route open: the dashboard would rather + // be told "not ok" promptly than hang on its health strip. + adminHealthProbeTimeout = 2 * time.Second + + adminLogLevelCountsCacheKey = "log-level-counts-24h" + adminLogLevelCountsCacheTTL = 30 * time.Second +) + type adminServerStatusResponse struct { - StartedAt time.Time `json:"started_at"` - RestartRequired bool `json:"restart_required"` - RestartRequiredAt *time.Time `json:"restart_required_at,omitempty"` - RestartRequiredReason string `json:"restart_required_reason,omitempty"` - RestartRequested bool `json:"restart_requested"` - RestartRequestedAt *time.Time `json:"restart_requested_at,omitempty"` + StartedAt time.Time `json:"started_at"` + RestartRequired bool `json:"restart_required"` + RestartRequiredAt *time.Time `json:"restart_required_at,omitempty"` + RestartRequiredReason string `json:"restart_required_reason,omitempty"` + RestartRequested bool `json:"restart_requested"` + RestartRequestedAt *time.Time `json:"restart_requested_at,omitempty"` + Health adminServerHealth `json:"health"` +} + +// adminServerHealth backs the dashboard health strip. Version, uptime and node +// counts are not repeated here: the client already has them from +// /admin/system/build, started_at above, and /admin/nodes. +type adminServerHealth struct { + Postgres adminHealthComponent `json:"postgres"` + Redis adminHealthComponent `json:"redis"` + Errors24h int64 `json:"errors_24h"` + Warnings24h int64 `json:"warnings_24h"` +} + +// adminHealthComponent reports one backing service. `configured` false means +// the deployment runs without it (a supported single-node shape for Redis), in +// which case `ok` is absent rather than false — "not present" and "present but +// broken" must not look the same on the strip. +type adminHealthComponent struct { + Configured bool `json:"configured"` + OK *bool `json:"ok,omitempty"` + LatencyMS *float64 `json:"latency_ms,omitempty"` } +// adminLogLevelCounts is the cached error/warning tally. It is an array rather +// than a struct so it satisfies cache.TTLCache's comparable constraint without +// a pointer indirection. +type adminLogLevelCounts [2]int64 + // HandleGetServerStatus handles GET /admin/server/status. func (h *AdminHandler) HandleGetServerStatus(w http.ResponseWriter, r *http.Request) { snapshot := h.RestartStatus.Snapshot() @@ -42,9 +81,94 @@ func (h *AdminHandler) HandleGetServerStatus(w http.ResponseWriter, r *http.Requ } } + resp.Health = h.collectHealth(r.Context()) + writeJSON(w, http.StatusOK, resp) } +// collectHealth probes the backing services and reads the recent log tallies. +// Every failure here is reported in the body, never as a status code: an +// unreachable dependency is exactly what an admin opened this page to see. +func (h *AdminHandler) collectHealth(ctx context.Context) adminServerHealth { + health := adminServerHealth{ + Postgres: h.probePostgres(ctx), + Redis: h.probeRedis(ctx), + } + counts := h.logLevelCounts24h(ctx) + health.Errors24h = counts[0] + health.Warnings24h = counts[1] + return health +} + +func (h *AdminHandler) probePostgres(ctx context.Context) adminHealthComponent { + if h == nil || h.pool == nil { + return adminHealthComponent{} + } + probeCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + start := time.Now() + err := h.pool.Ping(probeCtx) + return newHealthComponent(err == nil, time.Since(start)) +} + +func (h *AdminHandler) probeRedis(ctx context.Context) adminHealthComponent { + if h == nil || h.RedisClient == nil { + return adminHealthComponent{} + } + probeCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + start := time.Now() + err := h.RedisClient.Ping(probeCtx).Err() + return newHealthComponent(err == nil, time.Since(start)) +} + +func newHealthComponent(ok bool, latency time.Duration) adminHealthComponent { + // Sub-millisecond pings are the normal case for a local Postgres, so the + // latency keeps two decimals instead of rounding an entire healthy install + // down to zero. + ms := math.Round(float64(latency.Microseconds())/10) / 100 + return adminHealthComponent{Configured: true, OK: &ok, LatencyMS: &ms} +} + +// logLevelCounts24h returns [errors, warnings] logged in the last 24 hours. +// A server with operational logging disabled, or one whose log partitions have +// not been created yet, reports zeros and a warn log rather than failing the +// whole status route over a secondary number. +func (h *AdminHandler) logLevelCounts24h(ctx context.Context) adminLogLevelCounts { + if h == nil || h.pool == nil { + return adminLogLevelCounts{} + } + if h.logLevelCounts != nil { + if counts, ok := h.logLevelCounts.Get(adminLogLevelCountsCacheKey); ok { + return counts + } + } + + queryCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + var counts adminLogLevelCounts + err := h.pool.QueryRow(queryCtx, ` + SELECT + COUNT(*) FILTER (WHERE level = 'error')::bigint, + COUNT(*) FILTER (WHERE level = 'warn')::bigint + FROM operational_logs + WHERE timestamp >= now() - interval '24 hours' + `).Scan(&counts[0], &counts[1]) + if err != nil { + slog.WarnContext(ctx, "failed to count recent operational logs for server status", + "component", "api", "error", err) + return adminLogLevelCounts{} + } + + if h.logLevelCounts != nil { + h.logLevelCounts.Set(adminLogLevelCountsCacheKey, counts, adminLogLevelCountsCacheTTL) + } + return counts +} + func (h *AdminHandler) markServerRestartRequired(reason string) { if h == nil { return diff --git a/internal/api/handlers/admin_server_status_test.go b/internal/api/handlers/admin_server_status_test.go index 7c0883a0e..4cf37ba57 100644 --- a/internal/api/handlers/admin_server_status_test.go +++ b/internal/api/handlers/admin_server_status_test.go @@ -1,11 +1,16 @@ package handlers import ( + "context" "encoding/json" "net/http" "net/http/httptest" "testing" + "time" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" "github.com/Silo-Server/silo-server/internal/config" ) @@ -85,3 +90,107 @@ func TestAdminServerStatusDoesNotPromoteLiveJellyfinIdentitySettings(t *testing. t.Fatal("RestartRequired = true, want false for live Jellyfin identity settings") } } + +// A server with neither a pool nor a Redis client must still answer 200 with a +// well-formed health object: the dashboard health strip is the one place an +// admin can see that a dependency is missing. +func TestAdminServerStatusHealthWithoutDependencies(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{RestartStatus: NewServerRestartStatusTracker()} + rec := httptest.NewRecorder() + handler.HandleGetServerStatus(rec, httptest.NewRequest(http.MethodGet, "/admin/server/status", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + + var body struct { + Health struct { + Postgres map[string]any `json:"postgres"` + Redis map[string]any `json:"redis"` + Errors int64 `json:"errors_24h"` + Warnings int64 `json:"warnings_24h"` + } `json:"health"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + + for name, component := range map[string]map[string]any{ + "postgres": body.Health.Postgres, + "redis": body.Health.Redis, + } { + if component == nil { + t.Fatalf("%s health missing from the response", name) + } + if configured, _ := component["configured"].(bool); configured { + t.Fatalf("%s configured = true, want false", name) + } + if _, present := component["ok"]; present { + t.Fatalf("%s reports ok while unconfigured; absent and broken must differ", name) + } + if _, present := component["latency_ms"]; present { + t.Fatalf("%s reports a latency it never measured", name) + } + } + if body.Health.Errors != 0 || body.Health.Warnings != 0 { + t.Fatalf("log counts = %d/%d, want 0/0", body.Health.Errors, body.Health.Warnings) + } +} + +// An unreachable Postgres is reported as ok:false rather than failing the +// route, and the log tallies degrade to zero instead of 500ing. +func TestAdminServerStatusHealthWithUnreachablePostgres(t *testing.T) { + t.Parallel() + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + + handler := &AdminHandler{pool: pool, RestartStatus: NewServerRestartStatusTracker()} + rec := httptest.NewRecorder() + handler.HandleGetServerStatus(rec, httptest.NewRequest(http.MethodGet, "/admin/server/status", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + var body struct { + Health adminServerHealth `json:"health"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if !body.Health.Postgres.Configured { + t.Fatal("postgres configured = false, want true when a pool exists") + } + if body.Health.Postgres.OK == nil || *body.Health.Postgres.OK { + t.Fatalf("postgres ok = %v, want false", body.Health.Postgres.OK) + } + if body.Health.Errors24h != 0 || body.Health.Warnings24h != 0 { + t.Fatalf("log counts = %d/%d, want 0/0 when the query fails", body.Health.Errors24h, body.Health.Warnings24h) + } +} + +func TestLogLevelCountsAreServedFromCache(t *testing.T) { + t.Parallel() + + counts := cache.NewTTLCache[adminLogLevelCounts]() + t.Cleanup(counts.Close) + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + + handler := &AdminHandler{pool: pool, logLevelCounts: counts} + counts.Set(adminLogLevelCountsCacheKey, adminLogLevelCounts{4, 12}, time.Minute) + + // The pool cannot connect, so a cache miss would return zeros. + if got := handler.logLevelCounts24h(context.Background()); got != (adminLogLevelCounts{4, 12}) { + t.Fatalf("counts = %v, want [4 12] from cache", got) + } +} diff --git a/internal/api/handlers/admin_stats_playback.go b/internal/api/handlers/admin_stats_playback.go new file mode 100644 index 000000000..c8264e30a --- /dev/null +++ b/internal/api/handlers/admin_stats_playback.go @@ -0,0 +1,370 @@ +package handlers + +import ( + "context" + "fmt" + "math" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/playback" +) + +const ( + adminPlaybackActivityCacheTTL = 60 * time.Second + adminPlaybackActivityCachePrefix = "hours=" + + adminPlaybackActivityDefaultHours = 24 + adminPlaybackActivityMinHours = 1 + adminPlaybackActivityMaxHours = 744 + + // adminPlaybackActivityHourlyMaxHours is the widest window still bucketed by + // hour. Past two days an hourly chart is more columns than a widget has + // pixels, so the buckets become days. + adminPlaybackActivityHourlyMaxHours = 48 + + adminPlaybackActivityHourSeconds = 3600 + adminPlaybackActivityDaySeconds = 86400 +) + +// playbackActivityBucketSeconds picks the bucket width for a window: hourly up +// to two days, daily beyond it. The response reports the choice in +// bucket_seconds so the client zero-fills the same shape the server grouped by. +func playbackActivityBucketSeconds(hours int) int { + if hours <= adminPlaybackActivityHourlyMaxHours { + return adminPlaybackActivityHourSeconds + } + return adminPlaybackActivityDaySeconds +} + +// date_trunc field names for the two bucket widths. +const ( + playbackActivityTruncDay = "day" + playbackActivityTruncHour = "hour" +) + +// playbackActivityTruncField is the date_trunc field matching a bucket width. +func playbackActivityTruncField(bucketSeconds int) string { + if bucketSeconds >= adminPlaybackActivityDaySeconds { + return playbackActivityTruncDay + } + return playbackActivityTruncHour +} + +// AdminPlaybackActivityBucket is one bucket of playback starts, split by the +// play method that was resolved for the session. Only buckets with at least one +// session are present; the dashboard zero-fills the rest of the window so a +// quiet server draws an empty column rather than a shorter chart. +// +// The field stays `hour` for every bucket width: it is the bucket's start +// instant, and renaming it would break every client for no new information — +// bucket_seconds already says how wide the bucket is. +type AdminPlaybackActivityBucket struct { + Hour time.Time `json:"hour"` + Direct int64 `json:"direct"` + Remux int64 `json:"remux"` + Transcode int64 `json:"transcode"` +} + +// AdminPlaybackReliability summarizes how playback went over the window. +// +// Time-to-first-frame and failed-start counts are deliberately absent: nothing +// records a playback *start* event today (playback_history_admin only gains a +// row when a session finalizes), so both would have to be guessed from log +// parsing. They need client telemetry first — see docs/admin-api.md. +type AdminPlaybackReliability struct { + SessionsStarted int64 `json:"sessions_started"` + TranscodeStarts int64 `json:"transcode_starts"` + FinalizedSessions int64 `json:"finalized_sessions"` + CompletedSessions int64 `json:"completed_sessions"` + CompletionRate float64 `json:"completion_rate"` + UniqueProfiles int64 `json:"unique_profiles"` +} + +// AdminPlaybackActivity is the GET /admin/stats/playback-activity body. +// +// Buckets are hourly or daily depending on the window (BucketSeconds says +// which). Reliability is computed over the whole requested window, while +// ProfilesActive24h is a fixed rolling-24h tile that ignores the window — it +// answers "who watched today", which does not become "who watched this month" +// because a chart next to it got wider. +type AdminPlaybackActivity struct { + Hours int `json:"hours"` + BucketSeconds int `json:"bucket_seconds"` + Buckets []AdminPlaybackActivityBucket `json:"buckets"` + Reliability AdminPlaybackReliability `json:"reliability"` + ProfilesActive24h int64 `json:"profiles_active_24h"` +} + +// AdminPlaybackActivitySource returns cached or freshly queried playback +// activity for a window of hours. +type AdminPlaybackActivitySource interface { + Get(ctx context.Context, hours int) (*AdminPlaybackActivity, error) + Invalidate() +} + +// AdminPlaybackActivityProvider serves playback activity with a short in-process +// TTL and optional cross-node invalidation via the shared event bus, mirroring +// AdminStatsProvider. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries a slice. +type AdminPlaybackActivityProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminPlaybackActivity] + ttl time.Duration +} + +var _ AdminPlaybackActivitySource = (*AdminPlaybackActivityProvider)(nil) + +// NewAdminPlaybackActivityProvider creates a cached provider and subscribes it +// to the playback/admin invalidation channels when an event bus is configured. +func NewAdminPlaybackActivityProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminPlaybackActivityProvider, error) { + provider := &AdminPlaybackActivityProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminPlaybackActivity](), + ttl: adminPlaybackActivityCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin playback activity provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached window when available, otherwise queries Postgres. +func (p *AdminPlaybackActivityProvider) Get(ctx context.Context, hours int) (*AdminPlaybackActivity, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin playback activity provider is not configured") + } + hours = clampQueryInt(hours, adminPlaybackActivityMinHours, adminPlaybackActivityMaxHours) + key := adminPlaybackActivityCachePrefix + strconv.Itoa(hours) + if activity, ok := p.cache.Get(key); ok { + return activity, nil + } + + activity, err := queryAdminPlaybackActivity(ctx, p.pool, hours) + if err != nil { + return nil, err + } + p.cache.Set(key, activity, p.ttl) + return activity, nil +} + +// Invalidate drops every cached window. +func (p *AdminPlaybackActivityProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminPlaybackActivityCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminPlaybackActivityProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetPlaybackActivity handles GET /admin/stats/playback-activity. +func (h *AdminHandler) HandleGetPlaybackActivity(w http.ResponseWriter, r *http.Request) { + hours, err := parseClampedIntQuery( + r, "hours", + adminPlaybackActivityDefaultHours, + adminPlaybackActivityMinHours, + adminPlaybackActivityMaxHours, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var activity *AdminPlaybackActivity + switch { + case h.PlaybackActivitySource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.PlaybackActivitySource.Invalidate() + } + activity, err = h.PlaybackActivitySource.Get(r.Context(), hours) + case h.pool != nil: + activity, err = queryAdminPlaybackActivity(r.Context(), h.pool, hours) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get playback activity") + return + } + + writeJSON(w, http.StatusOK, activity) +} + +// playbackActivityRow is one (bucket, play_method) group from the union of +// finalized history and live sessions. +type playbackActivityRow struct { + Hour time.Time + PlayMethod string + Count int64 +} + +// assemblePlaybackBuckets folds the grouped rows into one bucket per instant. +// Play methods the resolver does not produce (empty or unknown strings on old +// rows) are dropped from the stacked series but still counted in +// reliability.sessions_started, which is measured by SQL over the same union. +func assemblePlaybackBuckets(rows []playbackActivityRow) []AdminPlaybackActivityBucket { + buckets := make([]AdminPlaybackActivityBucket, 0, len(rows)) + index := make(map[time.Time]int, len(rows)) + + for _, row := range rows { + hour := row.Hour.UTC() + pos, ok := index[hour] + if !ok { + buckets = append(buckets, AdminPlaybackActivityBucket{Hour: hour}) + pos = len(buckets) - 1 + index[hour] = pos + } + switch playback.PlayMethod(row.PlayMethod) { + case playback.PlayDirect: + buckets[pos].Direct += row.Count + case playback.PlayRemux: + buckets[pos].Remux += row.Count + case playback.PlayTranscode: + buckets[pos].Transcode += row.Count + } + } + + return buckets +} + +// completionRate is completed over finalized sessions. Live sessions are +// excluded from both sides by the caller: a session that is still playing has +// not failed to complete, so counting it as a miss would drag the rate down +// whenever someone is watching. +func completionRate(completed, finalized int64) float64 { + if finalized <= 0 { + return 0 + } + return math.Round(float64(completed)/float64(finalized)*10000) / 10000 +} + +// adminPlaybackSessionsCTE is the union both activity queries aggregate over. +// +// playback_history_admin only gains a row when a session finalizes, so the +// current hour would be under-counted without the live sessions; a live session +// is by definition not in history yet, so the union double-counts nothing. +// playback_sessions_sync.started_at is nullable for sessions reconstructed +// after a restart, hence the COALESCE onto updated_at. +const adminPlaybackSessionsCTE = ` + WITH sessions AS ( + SELECT started_at, play_method, completed, profile_id, FALSE AS live + FROM playback_history_admin + WHERE started_at >= now() - make_interval(hours => $1) + UNION ALL + SELECT COALESCE(started_at, updated_at) AS started_at, play_method, FALSE, profile_id, TRUE + FROM playback_sessions_sync + WHERE COALESCE(started_at, updated_at) >= now() - make_interval(hours => $1) + )` + +func queryAdminPlaybackActivity(ctx context.Context, pool *pgxpool.Pool, hours int) (*AdminPlaybackActivity, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + bucketSeconds := playbackActivityBucketSeconds(hours) + + rows, err := pool.Query(ctx, adminPlaybackSessionsCTE+` + SELECT date_trunc($2, started_at) AS bucket, + COALESCE(play_method, '') AS play_method, + COUNT(*)::bigint AS sessions + FROM sessions + WHERE started_at IS NOT NULL + GROUP BY 1, 2 + ORDER BY 1 + `, hours, playbackActivityTruncField(bucketSeconds)) + if err != nil { + return nil, fmt.Errorf("querying playback activity buckets: %w", err) + } + defer rows.Close() + + grouped := make([]playbackActivityRow, 0, 64) + for rows.Next() { + var row playbackActivityRow + if err := rows.Scan(&row.Hour, &row.PlayMethod, &row.Count); err != nil { + return nil, fmt.Errorf("scanning playback activity bucket: %w", err) + } + grouped = append(grouped, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating playback activity buckets: %w", err) + } + + activity := &AdminPlaybackActivity{ + Hours: hours, + BucketSeconds: bucketSeconds, + Buckets: assemblePlaybackBuckets(grouped), + } + + // profiles_active_24h deliberately excludes imported and provider-synced + // history so the tile means "watched on this server", not "appeared in + // someone's Trakt backlog". Manual marks stay in: they are on-server + // actions. + row := pool.QueryRow(ctx, adminPlaybackSessionsCTE+`, + reliability AS ( + SELECT + COUNT(*)::bigint AS sessions_started, + COUNT(*) FILTER (WHERE play_method = 'transcode')::bigint AS transcode_starts, + COUNT(*) FILTER (WHERE NOT live)::bigint AS finalized_sessions, + COUNT(*) FILTER (WHERE NOT live AND completed)::bigint AS completed_sessions, + COUNT(DISTINCT profile_id)::bigint AS unique_profiles + FROM sessions + ), + active_profiles AS ( + SELECT COUNT(DISTINCT (user_id, profile_id))::bigint AS profiles_active_24h + FROM user_watch_history + WHERE watched_at >= now() - interval '24 hours' + AND COALESCE(source, 'legacy') NOT IN ('import', 'trakt', 'simkl', 'mdblist') + ) + SELECT + reliability.sessions_started, + reliability.transcode_starts, + reliability.finalized_sessions, + reliability.completed_sessions, + reliability.unique_profiles, + active_profiles.profiles_active_24h + FROM reliability + CROSS JOIN active_profiles + `, hours) + if err := row.Scan( + &activity.Reliability.SessionsStarted, + &activity.Reliability.TranscodeStarts, + &activity.Reliability.FinalizedSessions, + &activity.Reliability.CompletedSessions, + &activity.Reliability.UniqueProfiles, + &activity.ProfilesActive24h, + ); err != nil { + return nil, fmt.Errorf("querying playback reliability: %w", err) + } + activity.Reliability.CompletionRate = completionRate( + activity.Reliability.CompletedSessions, + activity.Reliability.FinalizedSessions, + ) + + return activity, nil +} diff --git a/internal/api/handlers/admin_stats_playback_test.go b/internal/api/handlers/admin_stats_playback_test.go new file mode 100644 index 000000000..c9a9e15a0 --- /dev/null +++ b/internal/api/handlers/admin_stats_playback_test.go @@ -0,0 +1,325 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// stubPlaybackActivitySource records what the handler asked for so the tests +// can assert on parameter handling without a database. +type stubPlaybackActivitySource struct { + activity *AdminPlaybackActivity + err error + gotHours int + invalidated int + callCount int +} + +func (s *stubPlaybackActivitySource) Get(_ context.Context, hours int) (*AdminPlaybackActivity, error) { + s.callCount++ + s.gotHours = hours + return s.activity, s.err +} + +func (s *stubPlaybackActivitySource) Invalidate() { s.invalidated++ } + +func TestAssemblePlaybackBuckets(t *testing.T) { + t.Parallel() + + hourOne := time.Date(2026, 8, 26, 10, 0, 0, 0, time.UTC) + hourTwo := time.Date(2026, 8, 26, 11, 0, 0, 0, time.UTC) + + tests := []struct { + name string + rows []playbackActivityRow + want []AdminPlaybackActivityBucket + }{ + { + name: "no rows yields an empty slice", + rows: nil, + want: []AdminPlaybackActivityBucket{}, + }, + { + name: "methods fold into one bucket per hour", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "direct", Count: 4}, + {Hour: hourOne, PlayMethod: "transcode", Count: 2}, + {Hour: hourTwo, PlayMethod: "remux", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 4, Transcode: 2}, + {Hour: hourTwo, Remux: 1}, + }, + }, + { + name: "history and live rows for the same hour and method add up", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "direct", Count: 3}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 4}, + }, + }, + { + name: "unknown play methods do not create a phantom series", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "", Count: 9}, + {Hour: hourOne, PlayMethod: "sorcery", Count: 5}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 1}, + }, + }, + { + name: "hours arrive in the query's order and keep it", + rows: []playbackActivityRow{ + {Hour: hourTwo, PlayMethod: "direct", Count: 1}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourTwo, Direct: 1}, + {Hour: hourOne, Direct: 1}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := assemblePlaybackBuckets(tt.rows) + if len(got) != len(tt.want) { + t.Fatalf("buckets = %+v, want %+v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("bucket %d = %+v, want %+v", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestCompletionRate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + completed int64 + finalized int64 + want float64 + }{ + {name: "no finalized sessions is zero, not NaN", finalized: 0, completed: 0, want: 0}, + {name: "live-only window stays zero", finalized: 0, completed: 5, want: 0}, + {name: "everything completed", finalized: 4, completed: 4, want: 1}, + {name: "rounded to four decimals", finalized: 38, completed: 27, want: 0.7105}, + {name: "nothing completed", finalized: 9, completed: 0, want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := completionRate(tt.completed, tt.finalized); got != tt.want { + t.Fatalf("completionRate(%d, %d) = %v, want %v", tt.completed, tt.finalized, got, tt.want) + } + }) + } +} + +func TestPlaybackActivityBucketSeconds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hours int + want int + wantTrunc string + }{ + {name: "the minimum window is hourly", hours: adminPlaybackActivityMinHours, want: 3600, wantTrunc: "hour"}, + {name: "a day is hourly", hours: 24, want: 3600, wantTrunc: "hour"}, + {name: "two days is the last hourly window", hours: 48, want: 3600, wantTrunc: "hour"}, + {name: "just past two days becomes daily", hours: 49, want: 86400, wantTrunc: "day"}, + {name: "a week is daily", hours: 168, want: 86400, wantTrunc: "day"}, + {name: "the maximum window is daily", hours: adminPlaybackActivityMaxHours, want: 86400, wantTrunc: "day"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := playbackActivityBucketSeconds(tt.hours) + if got != tt.want { + t.Fatalf("playbackActivityBucketSeconds(%d) = %d, want %d", tt.hours, got, tt.want) + } + if trunc := playbackActivityTruncField(got); trunc != tt.wantTrunc { + t.Fatalf("playbackActivityTruncField(%d) = %q, want %q", got, trunc, tt.wantTrunc) + } + }) + } +} + +func TestHandleGetPlaybackActivityClampsHours(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + want int + }{ + {name: "absent uses the default window", query: "", want: adminPlaybackActivityDefaultHours}, + {name: "empty uses the default window", query: "?hours=", want: adminPlaybackActivityDefaultHours}, + {name: "explicit value passes through", query: "?hours=6", want: 6}, + {name: "a week passes through", query: "?hours=168", want: 168}, + {name: "zero clamps up", query: "?hours=0", want: adminPlaybackActivityMinHours}, + {name: "negative clamps up", query: "?hours=-12", want: adminPlaybackActivityMinHours}, + {name: "a month is the ceiling", query: "?hours=744", want: 744}, + {name: "oversized clamps down", query: "?hours=100000", want: adminPlaybackActivityMaxHours}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotHours != tt.want { + t.Fatalf("hours = %d, want %d", source.gotHours, tt.want) + } + }) + } +} + +func TestHandleGetPlaybackActivityRejectsNonNumericHours(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity?hours=soon", nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } +} + +func TestHandleGetPlaybackActivityRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + if source.invalidated != 0 { + t.Fatalf("invalidated = %d on a plain read, want 0", source.invalidated) + } + + rec = httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity?refresh=1", nil)) + if source.invalidated != 1 { + t.Fatalf("invalidated = %d after refresh=1, want 1", source.invalidated) + } +} + +func TestHandleGetPlaybackActivityWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestHandleGetPlaybackActivityEmptyWindowSerializesArrays(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{ + Hours: 24, + BucketSeconds: adminPlaybackActivityHourSeconds, + Buckets: []AdminPlaybackActivityBucket{}, + }} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + Hours int `json:"hours"` + BucketSeconds int `json:"bucket_seconds"` + Buckets []json.RawMessage `json:"buckets"` + Reliability struct { + CompletionRate float64 `json:"completion_rate"` + } `json:"reliability"` + ProfilesActive24h int64 `json:"profiles_active_24h"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Buckets == nil { + t.Fatal("buckets decoded as null; a data-free server must send []") + } + if len(body.Buckets) != 0 || body.ProfilesActive24h != 0 || body.Reliability.CompletionRate != 0 { + t.Fatalf("unexpected body: %+v", body) + } + // The client zero-fills from bucket_seconds, so it has to survive + // serialization even on a server with nothing to report. + if body.BucketSeconds != adminPlaybackActivityHourSeconds { + t.Fatalf("bucket_seconds = %d, want %d", body.BucketSeconds, adminPlaybackActivityHourSeconds) + } +} + +func TestAdminPlaybackActivityProviderInvalidateClearsEveryWindow(t *testing.T) { + t.Parallel() + + provider, err := NewAdminPlaybackActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + provider.cache.Set(adminPlaybackActivityCachePrefix+"1", &AdminPlaybackActivity{Hours: 1}, time.Minute) + provider.cache.Set(adminPlaybackActivityCachePrefix+"24", &AdminPlaybackActivity{Hours: 24}, time.Minute) + + provider.Invalidate() + + for _, key := range []string{adminPlaybackActivityCachePrefix + "1", adminPlaybackActivityCachePrefix + "24"} { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminPlaybackActivityProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminPlaybackActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 24); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/admin_stats_timeseries.go b/internal/api/handlers/admin_stats_timeseries.go new file mode 100644 index 000000000..6994b799c --- /dev/null +++ b/internal/api/handlers/admin_stats_timeseries.go @@ -0,0 +1,340 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" +) + +const ( + adminTimeseriesCacheTTL = 30 * time.Second + adminTimeseriesCachePrefix = "hours=" + + adminTimeseriesDefaultHours = 24 + adminTimeseriesMinHours = 1 + // adminTimeseriesMaxHours is the sampler's retention window + // (internal/dashmetrics), so the widest window a client can ask for is the + // widest one the table can answer. + adminTimeseriesMaxHours = 744 + + // adminTimeseriesResolutionSeconds is the sampler's minute cadence + // (internal/dashmetrics) and therefore the finest bucket a read can return. + // The response reports the bucket it actually used, so the client sizes gaps + // from the data instead of hardcoding a second copy of this number. + adminTimeseriesResolutionSeconds = 60 + + // adminTimeseriesMaxPoints is the point budget a response aims to stay + // under. A dashboard widget is a few hundred CSS pixels wide, so a month of + // minutes would spend megabytes drawing sub-pixel detail nobody can see. + adminTimeseriesMaxPoints = 750 +) + +// timeseriesBucketSeconds picks the display bucket for a window, keeping every +// response under adminTimeseriesMaxPoints points. +// +// The thresholds are fixed rather than derived from the budget so that two +// clients asking for neighboring windows land on the same bucket — a bucket +// that drifted with the window would make consecutive reads incomparable, and +// the cache key (which is the window) would no longer imply the resolution. +func timeseriesBucketSeconds(hours int) int { + switch { + case hours <= 2: + return adminTimeseriesResolutionSeconds // 120 points at most + case hours <= 48: + return 300 // 5 minutes, 576 points at most + case hours <= 336: + return 1800 // 30 minutes, 672 points at most + default: + return 7200 // 2 hours, 372 points at most for a 31-day window + } +} + +// AdminTimeseriesPoint is one display bucket of sampled dashboard metrics — +// one sampled minute for short windows, several minutes collapsed for wide +// ones. Stream counts come from the cluster-wide "shared" sample; egress sums +// every source for a minute, so node egress and the egress each API process +// served are both included. +// +// A bucket that spans several minutes reports the peak minute of each column, +// never an average: concurrency and egress are read to answer "how bad did it +// get", and a mean would hide exactly that. +type AdminTimeseriesPoint struct { + T time.Time `json:"t"` + Streams int64 `json:"streams"` + Direct int64 `json:"direct"` + Remux int64 `json:"remux"` + Transcode int64 `json:"transcode"` + EgressKbps int64 `json:"egress_kbps"` +} + +// AdminTimeseries is the GET /admin/stats/timeseries body. +// +// Buckets the sampler missed — a restart, a paused process, a server that was +// simply off — are absent from Points rather than zero-filled: a gap and an +// idle bucket are different facts and the chart draws them differently. +// OldestSampleAt is nil until the sampler has written anything, which is how +// the dashboard knows to say it is still collecting data. +// +// ResolutionSeconds is the bucket this window was aggregated into, not a +// constant: it widens with the requested window (timeseriesBucketSeconds). +type AdminTimeseries struct { + ResolutionSeconds int `json:"resolution_seconds"` + From time.Time `json:"from"` + To time.Time `json:"to"` + OldestSampleAt *time.Time `json:"oldest_sample_at"` + Points []AdminTimeseriesPoint `json:"points"` +} + +// AdminTimeseriesSource returns cached or freshly queried samples for a window +// of hours. +type AdminTimeseriesSource interface { + Get(ctx context.Context, hours int) (*AdminTimeseries, error) + Invalidate() +} + +// AdminTimeseriesProvider serves sampled dashboard metrics with a short +// in-process TTL and optional cross-node invalidation via the shared event bus, +// mirroring AdminStatsProvider. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries a slice. +type AdminTimeseriesProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminTimeseries] + ttl time.Duration +} + +var _ AdminTimeseriesSource = (*AdminTimeseriesProvider)(nil) + +// NewAdminTimeseriesProvider creates a cached provider and subscribes it to the +// playback/admin invalidation channels when an event bus is configured. +func NewAdminTimeseriesProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminTimeseriesProvider, error) { + provider := &AdminTimeseriesProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminTimeseries](), + ttl: adminTimeseriesCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin timeseries provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached window when available, otherwise queries Postgres. +// +// The window is clamped before the key is built, so two requests the server +// would answer identically share one entry. The bucket size is a function of +// the clamped window, so the key needs nothing else to stay honest. +func (p *AdminTimeseriesProvider) Get(ctx context.Context, hours int) (*AdminTimeseries, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin timeseries provider is not configured") + } + hours = clampQueryInt(hours, adminTimeseriesMinHours, adminTimeseriesMaxHours) + key := adminTimeseriesCachePrefix + strconv.Itoa(hours) + if series, ok := p.cache.Get(key); ok { + return series, nil + } + + series, err := queryAdminTimeseries(ctx, p.pool, hours) + if err != nil { + return nil, err + } + p.cache.Set(key, series, p.ttl) + return series, nil +} + +// Invalidate drops every cached window. +func (p *AdminTimeseriesProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminTimeseriesCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminTimeseriesProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetTimeseries handles GET /admin/stats/timeseries. +func (h *AdminHandler) HandleGetTimeseries(w http.ResponseWriter, r *http.Request) { + hours, err := parseClampedIntQuery( + r, "hours", + adminTimeseriesDefaultHours, + adminTimeseriesMinHours, + adminTimeseriesMaxHours, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var series *AdminTimeseries + switch { + case h.TimeseriesSource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.TimeseriesSource.Invalidate() + } + series, err = h.TimeseriesSource.Get(r.Context(), hours) + case h.pool != nil: + series, err = queryAdminTimeseries(r.Context(), h.pool, hours) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get dashboard timeseries") + return + } + + writeJSON(w, http.StatusOK, series) +} + +// timeseriesRow is one grouped display bucket as it comes back from Postgres. +// The stream counts are nullable: a bucket can hold process egress rows without +// a shared row when a replica sampled egress but every attempt at the shared +// row lost the race or failed. +type timeseriesRow struct { + Bucket time.Time + Streams *int64 + Direct *int64 + Remux *int64 + Transcode *int64 + EgressKbps int64 +} + +// assembleTimeseries turns grouped rows into response points, reading a missing +// shared sample as zero streams rather than dropping the bucket — the egress it +// carries is still real. +func assembleTimeseries(rows []timeseriesRow) []AdminTimeseriesPoint { + points := make([]AdminTimeseriesPoint, 0, len(rows)) + for _, row := range rows { + points = append(points, AdminTimeseriesPoint{ + T: row.Bucket.UTC(), + Streams: nullableCount(row.Streams), + Direct: nullableCount(row.Direct), + Remux: nullableCount(row.Remux), + Transcode: nullableCount(row.Transcode), + EgressKbps: row.EgressKbps, + }) + } + return points +} + +func nullableCount(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func queryAdminTimeseries(ctx context.Context, pool *pgxpool.Pool, hours int) (*AdminTimeseries, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + bucketSeconds := timeseriesBucketSeconds(hours) + + // Two passes, in one statement. The inner one collapses the sources of a + // single minute: MAX over the shared source rather than SUM, because + // replicas may each have written the same minute's cluster-wide counts and + // those describe one cluster, not several — while egress is the opposite, + // since every source served different bytes. + // + // The outer one collapses those minutes into the display bucket by taking + // the peak minute of each column. Averaging would erase the spikes these + // charts exist to show; a bucket therefore answers "the worst minute in + // here", and its columns may come from different minutes. + rows, err := pool.Query(ctx, ` + WITH minutes AS ( + SELECT bucket, + MAX(streams_total) FILTER (WHERE source = 'shared') AS streams_total, + MAX(streams_direct) FILTER (WHERE source = 'shared') AS streams_direct, + MAX(streams_remux) FILTER (WHERE source = 'shared') AS streams_remux, + MAX(streams_transcode) FILTER (WHERE source = 'shared') AS streams_transcode, + COALESCE(SUM(egress_kbps), 0)::bigint AS egress_kbps + FROM dashboard_metric_samples + WHERE bucket >= now() - make_interval(hours => $1) + GROUP BY bucket + ) + SELECT date_bin(make_interval(secs => $2), bucket, 'epoch') AS display_bucket, + MAX(streams_total), + MAX(streams_direct), + MAX(streams_remux), + MAX(streams_transcode), + MAX(egress_kbps)::bigint + FROM minutes + GROUP BY display_bucket + ORDER BY display_bucket + `, hours, float64(bucketSeconds)) + if err != nil { + return nil, fmt.Errorf("querying dashboard metric samples: %w", err) + } + defer rows.Close() + + grouped := make([]timeseriesRow, 0, hours*3600/bucketSeconds+1) + for rows.Next() { + var row timeseriesRow + if err := rows.Scan( + &row.Bucket, + &row.Streams, + &row.Direct, + &row.Remux, + &row.Transcode, + &row.EgressKbps, + ); err != nil { + return nil, fmt.Errorf("scanning dashboard metric sample: %w", err) + } + grouped = append(grouped, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating dashboard metric samples: %w", err) + } + + series := &AdminTimeseries{ + ResolutionSeconds: bucketSeconds, + Points: assembleTimeseries(grouped), + } + + // The window bounds come from the same clock the samples were bucketed + // with, so a client comparing them against point timestamps never sees the + // API server's clock skew. + var oldest *time.Time + if err := pool.QueryRow(ctx, ` + SELECT MIN(bucket), + now() - make_interval(hours => $1), + now() + FROM dashboard_metric_samples + `, hours).Scan(&oldest, &series.From, &series.To); err != nil { + return nil, fmt.Errorf("querying dashboard metric sample window: %w", err) + } + if oldest != nil { + utc := oldest.UTC() + series.OldestSampleAt = &utc + } + series.From = series.From.UTC() + series.To = series.To.UTC() + + return series, nil +} diff --git a/internal/api/handlers/admin_stats_timeseries_test.go b/internal/api/handlers/admin_stats_timeseries_test.go new file mode 100644 index 000000000..25c3dc656 --- /dev/null +++ b/internal/api/handlers/admin_stats_timeseries_test.go @@ -0,0 +1,300 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// stubTimeseriesSource records what the handler asked for so the tests can +// assert on parameter handling without a database. +type stubTimeseriesSource struct { + series *AdminTimeseries + err error + gotHours int + invalidated int + callCount int +} + +func (s *stubTimeseriesSource) Get(_ context.Context, hours int) (*AdminTimeseries, error) { + s.callCount++ + s.gotHours = hours + return s.series, s.err +} + +func (s *stubTimeseriesSource) Invalidate() { s.invalidated++ } + +func int64Ptr(value int64) *int64 { return &value } + +func TestAssembleTimeseries(t *testing.T) { + t.Parallel() + + minuteOne := time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC) + minuteTwo := time.Date(2026, 8, 26, 12, 1, 0, 0, time.UTC) + + tests := []struct { + name string + rows []timeseriesRow + want []AdminTimeseriesPoint + }{ + { + name: "no samples yields an empty slice", + rows: nil, + want: []AdminTimeseriesPoint{}, + }, + { + name: "a full minute maps straight through", + rows: []timeseriesRow{{ + Bucket: minuteOne, + Streams: int64Ptr(3), + Direct: int64Ptr(1), + Remux: int64Ptr(0), + Transcode: int64Ptr(2), + EgressKbps: 48_211, + }}, + want: []AdminTimeseriesPoint{ + {T: minuteOne, Streams: 3, Direct: 1, Remux: 0, Transcode: 2, EgressKbps: 48_211}, + }, + }, + { + name: "a minute with only process egress reads as zero streams", + rows: []timeseriesRow{{Bucket: minuteOne, EgressKbps: 900}}, + want: []AdminTimeseriesPoint{{T: minuteOne, EgressKbps: 900}}, + }, + { + name: "gaps are left as gaps, not zero-filled", + rows: []timeseriesRow{ + {Bucket: minuteOne, Streams: int64Ptr(1), EgressKbps: 10}, + {Bucket: minuteTwo, Streams: int64Ptr(2), EgressKbps: 20}, + }, + want: []AdminTimeseriesPoint{ + {T: minuteOne, Streams: 1, EgressKbps: 10}, + {T: minuteTwo, Streams: 2, EgressKbps: 20}, + }, + }, + { + name: "buckets are reported in UTC", + rows: []timeseriesRow{{Bucket: minuteOne.In(time.FixedZone("UTC-5", -5*60*60)), Streams: int64Ptr(1)}}, + want: []AdminTimeseriesPoint{{T: minuteOne, Streams: 1}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := assembleTimeseries(tt.rows) + if len(got) != len(tt.want) { + t.Fatalf("points = %+v, want %+v", got, tt.want) + } + for i := range got { + if !got[i].T.Equal(tt.want[i].T) { + t.Fatalf("point %d time = %s, want %s", i, got[i].T, tt.want[i].T) + } + if got[i].Streams != tt.want[i].Streams || + got[i].Direct != tt.want[i].Direct || + got[i].Remux != tt.want[i].Remux || + got[i].Transcode != tt.want[i].Transcode || + got[i].EgressKbps != tt.want[i].EgressKbps { + t.Fatalf("point %d = %+v, want %+v", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestTimeseriesBucketSeconds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hours int + want int + }{ + {name: "the minimum window keeps sampled minutes", hours: adminTimeseriesMinHours, want: 60}, + {name: "two hours is the last minute-resolution window", hours: 2, want: 60}, + {name: "just past two hours steps to five minutes", hours: 3, want: 300}, + {name: "a day is five minutes", hours: 24, want: 300}, + {name: "two days is the last five-minute window", hours: 48, want: 300}, + {name: "just past two days steps to half an hour", hours: 49, want: 1800}, + {name: "a week is half an hour", hours: 168, want: 1800}, + {name: "a fortnight is the last half-hour window", hours: 336, want: 1800}, + {name: "just past a fortnight steps to two hours", hours: 337, want: 7200}, + {name: "the maximum window is two hours", hours: adminTimeseriesMaxHours, want: 7200}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := timeseriesBucketSeconds(tt.hours) + if got != tt.want { + t.Fatalf("timeseriesBucketSeconds(%d) = %d, want %d", tt.hours, got, tt.want) + } + // The point budget is the reason the buckets exist; a threshold + // that drifts past it silently regresses every wide window. + if points := tt.hours * 3600 / got; points > adminTimeseriesMaxPoints { + t.Fatalf("a %d-hour window yields %d points at %ds buckets, over the %d budget", + tt.hours, points, got, adminTimeseriesMaxPoints) + } + }) + } +} + +func TestHandleGetTimeseriesClampsHours(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + want int + }{ + {name: "absent uses the default window", query: "", want: adminTimeseriesDefaultHours}, + {name: "empty uses the default window", query: "?hours=", want: adminTimeseriesDefaultHours}, + {name: "explicit value passes through", query: "?hours=1", want: 1}, + {name: "a week passes through", query: "?hours=168", want: 168}, + {name: "zero clamps up", query: "?hours=0", want: adminTimeseriesMinHours}, + {name: "negative clamps up", query: "?hours=-3", want: adminTimeseriesMinHours}, + {name: "the retention window is the ceiling", query: "?hours=744", want: 744}, + {name: "past retention clamps down", query: "?hours=100000", want: adminTimeseriesMaxHours}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotHours != tt.want { + t.Fatalf("hours = %d, want %d", source.gotHours, tt.want) + } + }) + } +} + +func TestHandleGetTimeseriesRejectsNonNumericHours(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries?hours=lots", nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } +} + +func TestHandleGetTimeseriesRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + if source.invalidated != 0 { + t.Fatalf("invalidated = %d on a plain read, want 0", source.invalidated) + } + + rec = httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries?refresh=1", nil)) + if source.invalidated != 1 { + t.Fatalf("invalidated = %d after refresh=1, want 1", source.invalidated) + } +} + +func TestHandleGetTimeseriesWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestHandleGetTimeseriesFreshInstallSerializesCollectingState(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{ + ResolutionSeconds: adminTimeseriesResolutionSeconds, + Points: []AdminTimeseriesPoint{}, + }} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + ResolutionSeconds int `json:"resolution_seconds"` + Points []json.RawMessage `json:"points"` + OldestSampleAt *string `json:"oldest_sample_at"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Points == nil { + t.Fatal("points decoded as null; a server with no samples must send []") + } + if len(body.Points) != 0 { + t.Fatalf("points = %v, want empty", body.Points) + } + if body.OldestSampleAt != nil { + t.Fatalf("oldest_sample_at = %v, want null before the first sample", *body.OldestSampleAt) + } + if body.ResolutionSeconds != adminTimeseriesResolutionSeconds { + t.Fatalf("resolution_seconds = %d, want %d", body.ResolutionSeconds, adminTimeseriesResolutionSeconds) + } +} + +func TestAdminTimeseriesProviderInvalidateClearsEveryWindow(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTimeseriesProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + provider.cache.Set(adminTimeseriesCachePrefix+"1", &AdminTimeseries{}, time.Minute) + provider.cache.Set(adminTimeseriesCachePrefix+"24", &AdminTimeseries{}, time.Minute) + + provider.Invalidate() + + for _, key := range []string{adminTimeseriesCachePrefix + "1", adminTimeseriesCachePrefix + "24"} { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminTimeseriesProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTimeseriesProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 24); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/admin_stats_top.go b/internal/api/handlers/admin_stats_top.go new file mode 100644 index 000000000..e5af8434d --- /dev/null +++ b/internal/api/handlers/admin_stats_top.go @@ -0,0 +1,294 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" +) + +const ( + adminTopActivityCacheTTL = 5 * time.Minute + adminTopActivityCachePrefix = "days=" + + adminTopActivityDefaultDays = 7 + adminTopActivityMinDays = 1 + adminTopActivityMaxDays = 30 + + adminTopActivityDefaultLimit = 10 + adminTopActivityMinLimit = 1 + adminTopActivityMaxLimit = 25 +) + +// adminTopActivityWatchSourceFilter excludes history that was imported or +// synced in from a watch provider, so the leaderboards describe what people +// actually played here. `manual` (marked-watched) rows stay in: they are +// on-server actions. +const adminTopActivityWatchSourceFilter = `COALESCE(h.source, 'legacy') NOT IN ('import', 'trakt', 'simkl', 'mdblist')` + +// AdminTopTitle is one row of the most-watched-titles list. Episodes are rolled +// up to their series, so media_item_id is a series content id for TV. +type AdminTopTitle struct { + MediaItemID string `json:"media_item_id"` + Title string `json:"title"` + MediaType string `json:"media_type"` + Plays int64 `json:"plays"` + TotalSeconds int64 `json:"total_seconds"` +} + +// AdminTopProfile is one row of the most-active-profiles list. +type AdminTopProfile struct { + UserID int `json:"user_id"` + Username string `json:"username"` + ProfileID string `json:"profile_id"` + ProfileName string `json:"profile_name"` + Plays int64 `json:"plays"` + TotalSeconds int64 `json:"total_seconds"` +} + +// AdminTopActivity is the GET /admin/stats/top-activity body. +type AdminTopActivity struct { + Days int `json:"days"` + Limit int `json:"limit"` + Titles []AdminTopTitle `json:"titles"` + Profiles []AdminTopProfile `json:"profiles"` +} + +// AdminTopActivitySource returns cached or freshly queried leaderboards. +type AdminTopActivitySource interface { + Get(ctx context.Context, days, limit int) (*AdminTopActivity, error) + Invalidate() +} + +// AdminTopActivityProvider serves the leaderboards with a longer TTL than the +// other dashboard aggregates: a seven-day ranking barely moves within minutes, +// and the query is the most expensive one on the page. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries slices. +type AdminTopActivityProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminTopActivity] + ttl time.Duration +} + +var _ AdminTopActivitySource = (*AdminTopActivityProvider)(nil) + +// NewAdminTopActivityProvider creates a cached provider and subscribes it to +// the playback/admin invalidation channels when an event bus is configured. +func NewAdminTopActivityProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminTopActivityProvider, error) { + provider := &AdminTopActivityProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminTopActivity](), + ttl: adminTopActivityCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin top activity provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached leaderboards when available, otherwise queries Postgres. +func (p *AdminTopActivityProvider) Get(ctx context.Context, days, limit int) (*AdminTopActivity, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin top activity provider is not configured") + } + days = clampQueryInt(days, adminTopActivityMinDays, adminTopActivityMaxDays) + limit = clampQueryInt(limit, adminTopActivityMinLimit, adminTopActivityMaxLimit) + key := adminTopActivityCachePrefix + strconv.Itoa(days) + "&limit=" + strconv.Itoa(limit) + if activity, ok := p.cache.Get(key); ok { + return activity, nil + } + + activity, err := queryAdminTopActivity(ctx, p.pool, days, limit) + if err != nil { + return nil, err + } + p.cache.Set(key, activity, p.ttl) + return activity, nil +} + +// Invalidate drops every cached window. +func (p *AdminTopActivityProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminTopActivityCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminTopActivityProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetTopActivity handles GET /admin/stats/top-activity. +func (h *AdminHandler) HandleGetTopActivity(w http.ResponseWriter, r *http.Request) { + days, err := parseClampedIntQuery( + r, "days", + adminTopActivityDefaultDays, + adminTopActivityMinDays, + adminTopActivityMaxDays, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + limit, err := parseClampedIntQuery( + r, "limit", + adminTopActivityDefaultLimit, + adminTopActivityMinLimit, + adminTopActivityMaxLimit, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var activity *AdminTopActivity + switch { + case h.TopActivitySource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.TopActivitySource.Invalidate() + } + activity, err = h.TopActivitySource.Get(r.Context(), days, limit) + case h.pool != nil: + activity, err = queryAdminTopActivity(r.Context(), h.pool, days, limit) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get top activity") + return + } + + writeJSON(w, http.StatusOK, activity) +} + +func queryAdminTopActivity(ctx context.Context, pool *pgxpool.Pool, days, limit int) (*AdminTopActivity, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + activity := &AdminTopActivity{ + Days: days, + Limit: limit, + Titles: []AdminTopTitle{}, + Profiles: []AdminTopProfile{}, + } + + // Episodes roll up to their series: a season binge should read as one show, + // not twelve one-play entries. media_items is joined twice — once for the + // item itself (movies) and once for the parent series (episodes). + titleRows, err := pool.Query(ctx, ` + SELECT COALESCE(ep.series_id, h.media_item_id) AS item_id, + COALESCE(smi.title, mi.title, '') AS title, + COALESCE(CASE WHEN ep.content_id IS NOT NULL THEN 'series' ELSE mi.type END, '') AS media_type, + COUNT(*)::bigint AS plays, + COALESCE(SUM(h.duration_seconds), 0)::bigint AS total_seconds + FROM user_watch_history h + LEFT JOIN episodes ep ON ep.content_id = h.media_item_id + LEFT JOIN media_items mi ON mi.content_id = h.media_item_id + LEFT JOIN media_items smi ON smi.content_id = ep.series_id + WHERE h.watched_at >= now() - make_interval(days => $1) + AND `+adminTopActivityWatchSourceFilter+` + GROUP BY 1, 2, 3 + ORDER BY plays DESC, total_seconds DESC + LIMIT $2 + `, days, limit) + if err != nil { + return nil, fmt.Errorf("querying top titles: %w", err) + } + defer titleRows.Close() + + for titleRows.Next() { + var title AdminTopTitle + if err := titleRows.Scan( + &title.MediaItemID, + &title.Title, + &title.MediaType, + &title.Plays, + &title.TotalSeconds, + ); err != nil { + return nil, fmt.Errorf("scanning top title: %w", err) + } + activity.Titles = append(activity.Titles, title) + } + if err := titleRows.Err(); err != nil { + return nil, fmt.Errorf("iterating top titles: %w", err) + } + titleRows.Close() + + // Profile display names live in the per-user stores, not in + // user_watch_history, so they are read back from the most recent admin + // playback-history row for that profile. A profile that has only ever been + // marked-watched has no such row and falls back to its id. + profileRows, err := pool.Query(ctx, ` + SELECT h.user_id, + COALESCE(u.username, '') AS username, + h.profile_id, + COALESCE(pn.profile_name, h.profile_id) AS profile_name, + COUNT(*)::bigint AS plays, + COALESCE(SUM(h.duration_seconds), 0)::bigint AS total_seconds + FROM user_watch_history h + LEFT JOIN users u ON u.id = h.user_id + LEFT JOIN LATERAL ( + SELECT p.profile_name + FROM playback_history_admin p + WHERE p.user_id = h.user_id + AND p.profile_id = h.profile_id + AND p.profile_name <> '' + ORDER BY p.ended_at DESC + LIMIT 1 + ) pn ON TRUE + WHERE h.watched_at >= now() - make_interval(days => $1) + AND `+adminTopActivityWatchSourceFilter+` + GROUP BY 1, 2, 3, 4 + ORDER BY plays DESC, total_seconds DESC + LIMIT $2 + `, days, limit) + if err != nil { + return nil, fmt.Errorf("querying top profiles: %w", err) + } + defer profileRows.Close() + + for profileRows.Next() { + var profile AdminTopProfile + if err := profileRows.Scan( + &profile.UserID, + &profile.Username, + &profile.ProfileID, + &profile.ProfileName, + &profile.Plays, + &profile.TotalSeconds, + ); err != nil { + return nil, fmt.Errorf("scanning top profile: %w", err) + } + activity.Profiles = append(activity.Profiles, profile) + } + if err := profileRows.Err(); err != nil { + return nil, fmt.Errorf("iterating top profiles: %w", err) + } + + return activity, nil +} diff --git a/internal/api/handlers/admin_stats_top_test.go b/internal/api/handlers/admin_stats_top_test.go new file mode 100644 index 000000000..41d3e5653 --- /dev/null +++ b/internal/api/handlers/admin_stats_top_test.go @@ -0,0 +1,213 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type stubTopActivitySource struct { + activity *AdminTopActivity + err error + gotDays int + gotLimit int + invalidated int + callCount int +} + +func (s *stubTopActivitySource) Get(_ context.Context, days, limit int) (*AdminTopActivity, error) { + s.callCount++ + s.gotDays = days + s.gotLimit = limit + return s.activity, s.err +} + +func (s *stubTopActivitySource) Invalidate() { s.invalidated++ } + +func TestHandleGetTopActivityClampsParams(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantDays int + wantLimit int + }{ + { + name: "defaults", + query: "", + wantDays: adminTopActivityDefaultDays, + wantLimit: adminTopActivityDefaultLimit, + }, + { + name: "explicit values pass through", + query: "?days=14&limit=5", + wantDays: 14, + wantLimit: 5, + }, + { + name: "zero clamps up", + query: "?days=0&limit=0", + wantDays: adminTopActivityMinDays, + wantLimit: adminTopActivityMinLimit, + }, + { + name: "oversized clamps down", + query: "?days=999&limit=999", + wantDays: adminTopActivityMaxDays, + wantLimit: adminTopActivityMaxLimit, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotDays != tt.wantDays || source.gotLimit != tt.wantLimit { + t.Fatalf("days/limit = %d/%d, want %d/%d", source.gotDays, source.gotLimit, tt.wantDays, tt.wantLimit) + } + }) + } +} + +func TestHandleGetTopActivityRejectsNonNumericParams(t *testing.T) { + t.Parallel() + + for _, query := range []string{"?days=week", "?limit=all"} { + t.Run(query, func(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity"+query, nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } + }) + } +} + +func TestHandleGetTopActivityRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity?refresh=true", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if source.invalidated != 1 { + t.Fatalf("invalidated = %d, want 1", source.invalidated) + } +} + +func TestHandleGetTopActivitySourceFailureIs500(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{err: errors.New("boom")} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +func TestHandleGetTopActivityWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +// A server with no watch history must answer with empty lists rather than +// nulls: the bar-list widgets map over these fields directly. +func TestAdminTopActivityEmptyListsSerializeAsArrays(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{ + Days: 7, + Limit: 10, + Titles: []AdminTopTitle{}, + Profiles: []AdminTopProfile{}, + }} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + var body struct { + Titles []AdminTopTitle `json:"titles"` + Profiles []AdminTopProfile `json:"profiles"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Titles == nil || body.Profiles == nil { + t.Fatalf("titles/profiles decoded as null: %s", rec.Body.String()) + } +} + +func TestAdminTopActivityProviderInvalidateClearsEveryVariant(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTopActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + keys := []string{ + adminTopActivityCachePrefix + "7&limit=10", + adminTopActivityCachePrefix + "30&limit=25", + } + for _, key := range keys { + provider.cache.Set(key, &AdminTopActivity{}, time.Minute) + } + + provider.Invalidate() + + for _, key := range keys { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminTopActivityProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTopActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 7, 10); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 243ef26e5..956713b1f 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -194,6 +194,12 @@ type Dependencies struct { // that case rather than failing. MDBListClient *mdblist.Client + // Admin dashboard aggregates, cached like AdminStatsProvider. Each is + // optional: without one, the matching route queries Postgres per request. + AdminPlaybackActivityProvider handlers.AdminPlaybackActivitySource + AdminTopActivityProvider handlers.AdminTopActivitySource + AdminTimeseriesProvider handlers.AdminTimeseriesSource + // ABSHandler is the Audiobookshelf-compatible HTTP handler. When non-nil // it is mounted at the root router level (not under /api/v1/) so that ABS // clients hitting /login, /api/*, /abs/api/*, and /abs/socket.io/* all @@ -1150,6 +1156,10 @@ func NewRouter(deps Dependencies) chi.Router { adminHandler.EventsHub = deps.EventsHub adminHandler.ImpersonationService = authService adminHandler.StatsSource = deps.AdminStatsProvider + adminHandler.PlaybackActivitySource = deps.AdminPlaybackActivityProvider + adminHandler.TopActivitySource = deps.AdminTopActivityProvider + adminHandler.TimeseriesSource = deps.AdminTimeseriesProvider + adminHandler.RedisClient = deps.RedisClient adminHandler.RealtimeHub = deps.RealtimeHub adminHandler.AccessGroups = accessGroupStore adminHandler.BootstrapSensitiveConfigured = deps.BootstrapSensitiveConfigured @@ -2905,7 +2915,22 @@ func NewRouter(deps Dependencies) chi.Router { r.Get("/playback-history", adminHandler.HandleListPlaybackHistory) r.Get("/unmatched", adminHandler.HandleListUnmatched) r.Get("/stats", adminHandler.HandleGetStats) + // Dashboard aggregates. Both are cached reads over the + // same catalog/playback tables /stats uses, split out + // because their windows and refresh rates differ. + r.Get("/stats/playback-activity", adminHandler.HandleGetPlaybackActivity) + r.Get("/stats/top-activity", adminHandler.HandleGetTopActivity) + // Minute-resolution samples written by the dashboard + // metrics sampler (internal/dashmetrics); the only + // history for concurrent streams and egress. + r.Get("/stats/timeseries", adminHandler.HandleGetTimeseries) r.Get("/server/status", adminHandler.HandleGetServerStatus) + // Per-admin-account dashboard arrangement. The server + // stores it as an opaque JSON object; the web client + // owns widget-id and span validation. + r.Get("/dashboard/layout", adminHandler.HandleGetDashboardLayout) + r.Put("/dashboard/layout", adminHandler.HandlePutDashboardLayout) + r.Delete("/dashboard/layout", adminHandler.HandleDeleteDashboardLayout) r.Get("/catalog/search/status", adminHandler.HandleGetCatalogSearchStatus) if policyHandler != nil { r.Route("/policy", func(r chi.Router) { diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt index 39c0431ac..dda84c5ca 100644 --- a/internal/api/testdata/media_routes.txt +++ b/internal/api/testdata/media_routes.txt @@ -40,6 +40,9 @@ PUT /api/v1/admin/collections/{id}/items/order non-media DELETE /api/v1/admin/collections/{id}/items/{item_id} non-media PUT /api/v1/admin/collections/{id}/items/{item_id} non-media POST /api/v1/admin/collections/{id}/sync non-media +DELETE /api/v1/admin/dashboard/layout non-media +GET /api/v1/admin/dashboard/layout non-media +PUT /api/v1/admin/dashboard/layout non-media GET /api/v1/admin/devices non-media GET /api/v1/admin/devices/{user_id}/{device_id} non-media GET /api/v1/admin/diagnostics/reports/ non-media @@ -149,6 +152,9 @@ GET /api/v1/admin/settings/sensitive-status non-media GET /api/v1/admin/settings/{key} non-media PUT /api/v1/admin/settings/{key} non-media GET /api/v1/admin/stats non-media +GET /api/v1/admin/stats/playback-activity non-media +GET /api/v1/admin/stats/timeseries non-media +GET /api/v1/admin/stats/top-activity non-media GET /api/v1/admin/stream-telemetry/parity non-media GET /api/v1/admin/subtitle-providers/ non-media PUT /api/v1/admin/subtitle-providers/{provider}/ non-media diff --git a/internal/dashmetrics/egress.go b/internal/dashmetrics/egress.go new file mode 100644 index 000000000..b4b824ef0 --- /dev/null +++ b/internal/dashmetrics/egress.go @@ -0,0 +1,47 @@ +package dashmetrics + +import "github.com/Silo-Server/silo-server/internal/streamtelemetry" + +// computeEgressDelta returns the viewer bytes this process served between two +// telemetry snapshots, together with the cumulative counters the next call must +// compare against. +// +// Only RoleViewerEgress routes and transfers count. A proxy node's viewer +// traffic also traverses the API node as RoleInternalRelay, and counting both +// would report every relayed byte twice. +// +// Counters only ever grow, but a session can be pruned and re-created under the +// same id, and a restarted registry starts from zero. A shrinking counter is +// therefore read as a fresh start and contributes nothing rather than a +// negative rate. Entries that vanished from the snapshot are dropped, which +// keeps the map bounded by the live session count. +func computeEgressDelta(prev map[string]int64, snapshot streamtelemetry.Snapshot) (int64, map[string]int64) { + next := make(map[string]int64, len(snapshot.Sessions)+len(snapshot.Transfers)) + var delta int64 + + record := func(key string, cumulative int64) { + next[key] = cumulative + if grown := cumulative - prev[key]; grown > 0 { + delta += grown + } + } + + for _, session := range snapshot.Sessions { + var bytes int64 + for _, route := range session.Routes { + if route.Role == streamtelemetry.RoleViewerEgress { + bytes += route.BytesAccepted + } + } + record("session:"+session.SessionID, bytes) + } + + for _, transfer := range snapshot.Transfers { + if transfer.Role != streamtelemetry.RoleViewerEgress { + continue + } + record("transfer:"+transfer.ID, transfer.BytesAccepted) + } + + return delta, next +} diff --git a/internal/dashmetrics/sampler.go b/internal/dashmetrics/sampler.go new file mode 100644 index 000000000..c32d06dde --- /dev/null +++ b/internal/dashmetrics/sampler.go @@ -0,0 +1,226 @@ +// Package dashmetrics records the admin dashboard time series that cannot be +// reconstructed after the fact: how many streams were running (split by play +// method) and how much egress the deployment served. Live sessions leave no +// per-minute trace once they end, and node egress is a rolling average that is +// overwritten on every health check, so both have to be sampled as they happen. +// +// One row per minute per source lands in dashboard_metric_samples, and rows +// older than the retention window below are pruned once an hour: +// +// - "shared" is the cluster-wide snapshot. Every replica writes it with +// INSERT ... ON CONFLICT DO NOTHING, so the first writer for a minute wins +// and the others collapse. Replica snapshots differ only by sub-second +// timing, which is below the resolution a dashboard chart can show, so this +// is deliberately cheaper than coordinating with an advisory lock. +// - "proc:" carries the viewer egress served by one API process, +// measured from the local stream-telemetry registry. stream_nodes only +// describes external stream nodes, so without these rows a single-server +// deployment would chart zero egress forever. +// +// Sampling is best-effort: every failure is logged and swallowed. A missed +// minute is a gap in the chart, never a failed request or a dead server. +package dashmetrics + +import ( + "context" + "log/slog" + "math" + "os" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +const ( + // component is the slog component key every line from this package carries. + component = "dashmetrics" + + // sampleInterval matches the minute resolution of the samples table. + sampleInterval = time.Minute + + // retentionDays is how much history the charts can show — a month, so the + // dashboard's widest range has samples to draw. 1440 minutes a day times 31 + // days is ~45k rows per source, and sources are (1 + replicas), so the table + // stays in the low hundreds of thousands of rows at most. Reads bucket the + // minutes down before returning them (internal/api/handlers), so a wide + // window costs the same on the wire as a narrow one. + retentionDays = 31 +) + +// Sampler writes one dashboard_metric_samples row per minute for as long as it +// runs. Its state is owned by the single goroutine Start launches; nothing else +// reads or mutates it. +type Sampler struct { + pool *pgxpool.Pool + telemetry *streamtelemetry.Registry // nil when stream telemetry is disabled + source string // "proc:" + interval time.Duration + + // lastBucket is the minute the last tick wrote, so a ticker that fires + // twice inside one minute does not spend an INSERT that ON CONFLICT would + // only discard — which would silently drop the egress bytes it carried. + lastBucket time.Time + + // prevBytes holds the cumulative viewer bytes per telemetry session and + // transfer at the previous tick; lastEgressAt is when it was taken. + prevBytes map[string]int64 + lastEgressAt time.Time + + stopOnce sync.Once + stop chan struct{} +} + +// NewSampler builds a sampler for this process. telemetry may be nil, in which +// case only the shared cluster row is written. nodeID identifies this process +// among the replicas; it falls back to the hostname when empty. +func NewSampler(pool *pgxpool.Pool, telemetry *streamtelemetry.Registry, nodeID string) *Sampler { + if nodeID == "" { + nodeID, _ = os.Hostname() + } + if nodeID == "" { + nodeID = "unknown" + } + return &Sampler{ + pool: pool, + telemetry: telemetry, + source: "proc:" + nodeID, + interval: sampleInterval, + stop: make(chan struct{}), + } +} + +// Start samples once immediately — which also establishes the egress baseline — +// and then every minute until ctx is canceled or Stop is called. +func (s *Sampler) Start(ctx context.Context) { + if s == nil || s.pool == nil { + return + } + go func() { + s.sampleOnce(ctx, time.Now()) + + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case at := <-ticker.C: + s.sampleOnce(ctx, at) + } + } + }() +} + +// Stop ends the sampling goroutine. It is safe to call more than once. +func (s *Sampler) Stop() { + if s == nil { + return + } + s.stopOnce.Do(func() { + close(s.stop) + }) +} + +// sampleOnce writes this minute's rows and, once an hour, prunes expired ones. +func (s *Sampler) sampleOnce(ctx context.Context, at time.Time) { + bucket := sampleBucket(at) + if bucket.Equal(s.lastBucket) { + return + } + s.lastBucket = bucket + + s.sampleShared(ctx) + s.sampleProcessEgress(ctx, at, bucket) + + // Retention runs in-band rather than as its own timer: the table is tiny + // and one DELETE an hour costs less than another goroutine. + if at.Minute() == 0 { + s.pruneExpired(ctx) + } +} + +// sampleShared records the cluster-wide stream counts and node egress. Counting +// and inserting happen in one statement so no replica can read one minute's +// state and write it into another's bucket. +func (s *Sampler) sampleShared(ctx context.Context) { + _, err := s.pool.Exec(ctx, ` + INSERT INTO dashboard_metric_samples + (bucket, source, streams_total, streams_direct, streams_remux, streams_transcode, egress_kbps) + SELECT date_trunc('minute', now()), 'shared', + (SELECT COUNT(*) FROM playback_sessions_sync), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'direct'), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'remux'), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'transcode'), + (SELECT COALESCE(SUM(egress_kbps), 0) FROM stream_nodes WHERE enabled AND healthy) + ON CONFLICT (bucket, source) DO NOTHING + `) + if err != nil { + slog.WarnContext(ctx, "failed to sample shared dashboard metrics", "component", component, "error", err) + } +} + +// sampleProcessEgress records the viewer egress this process served since the +// previous tick. The row is keyed on the same locally-computed bucket the +// dedup guard in sampleOnce uses — keying on the DB clock instead could map +// two guard-distinct ticks onto one DB minute under clock skew, and the +// ON CONFLICT discard would silently drop the egress delta the second tick +// carried. The proc source is written only by this process, so the DB clock +// buys nothing here (unlike the shared row, where it arbitrates replicas). +func (s *Sampler) sampleProcessEgress(ctx context.Context, at time.Time, bucket time.Time) { + if s.telemetry == nil { + return + } + + delta, next := computeEgressDelta(s.prevBytes, s.telemetry.Snapshot()) + previous, previousAt := s.prevBytes, s.lastEgressAt + s.prevBytes, s.lastEgressAt = next, at + + // The very first snapshot carries every byte served since the process + // started. Charting that as one minute of egress would draw a spike that + // never happened, so the first tick only establishes the baseline. + if previous == nil || previousAt.IsZero() { + return + } + + // Zero minutes are written too: an idle server should draw a line along the + // baseline, not a gap that reads as "no data". + _, err := s.pool.Exec(ctx, ` + INSERT INTO dashboard_metric_samples (bucket, source, egress_kbps) + VALUES ($1, $2, $3) + ON CONFLICT (bucket, source) DO NOTHING + `, bucket, s.source, egressKbps(delta, at.Sub(previousAt))) + if err != nil { + slog.WarnContext(ctx, "failed to sample process egress", "component", component, "source", s.source, "error", err) + } +} + +// pruneExpired drops samples older than the retention window. +func (s *Sampler) pruneExpired(ctx context.Context) { + _, err := s.pool.Exec(ctx, ` + DELETE FROM dashboard_metric_samples + WHERE bucket < now() - make_interval(days => $1) + `, retentionDays) + if err != nil { + slog.WarnContext(ctx, "failed to prune dashboard metric samples", "component", component, "error", err) + } +} + +// sampleBucket truncates a sample time to the minute it belongs to, in UTC. +func sampleBucket(at time.Time) time.Time { + return at.UTC().Truncate(time.Minute) +} + +// egressKbps converts a byte delta over an elapsed period into kilobits per +// second. A non-positive delta or elapsed period is reported as zero rather +// than as a negative rate. +func egressKbps(deltaBytes int64, elapsed time.Duration) int64 { + if deltaBytes <= 0 || elapsed <= 0 { + return 0 + } + return int64(math.Round(float64(deltaBytes) * 8 / 1000 / elapsed.Seconds())) +} diff --git a/internal/dashmetrics/sampler_test.go b/internal/dashmetrics/sampler_test.go new file mode 100644 index 000000000..91126263f --- /dev/null +++ b/internal/dashmetrics/sampler_test.go @@ -0,0 +1,257 @@ +package dashmetrics + +import ( + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +func viewerSession(id string, bytes int64) streamtelemetry.SessionView { + return streamtelemetry.SessionView{ + SessionID: id, + Routes: []streamtelemetry.RouteActivityView{ + {Method: "GET", Pattern: "/stream", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: bytes}, + }, + } +} + +func TestComputeEgressDelta(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prev map[string]int64 + snapshot streamtelemetry.Snapshot + wantDelta int64 + wantNext map[string]int64 + }{ + { + name: "empty snapshot yields nothing", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{}, + wantDelta: 0, + wantNext: map[string]int64{}, + }, + { + name: "a new session contributes all of its bytes", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 4_000)}, + }, + wantDelta: 4_000, + wantNext: map[string]int64{"session:s1": 4_000}, + }, + { + name: "a grown session contributes only the growth", + prev: map[string]int64{"session:s1": 4_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 6_500)}, + }, + wantDelta: 2_500, + wantNext: map[string]int64{"session:s1": 6_500}, + }, + { + name: "a pruned session leaves the map and adds nothing", + prev: map[string]int64{"session:s1": 4_000, "session:s2": 1_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s2", 1_000)}, + }, + wantDelta: 0, + wantNext: map[string]int64{"session:s2": 1_000}, + }, + { + name: "a counter regression clamps at zero instead of going negative", + prev: map[string]int64{"session:s1": 9_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 500)}, + }, + wantDelta: 0, + wantNext: map[string]int64{"session:s1": 500}, + }, + { + name: "a regressing session does not cancel out a growing one", + prev: map[string]int64{"session:s1": 9_000, "session:s2": 100}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{ + viewerSession("s1", 500), + viewerSession("s2", 900), + }, + }, + wantDelta: 800, + wantNext: map[string]int64{"session:s1": 500, "session:s2": 900}, + }, + { + name: "relay routes are excluded so relayed bytes are not counted twice", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{{ + SessionID: "s1", + Routes: []streamtelemetry.RouteActivityView{ + {Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 700}, + {Role: streamtelemetry.RoleInternalRelay, BytesAccepted: 50_000}, + {Role: streamtelemetry.RoleProducer, BytesAccepted: 900}, + }, + }}, + }, + wantDelta: 700, + wantNext: map[string]int64{"session:s1": 700}, + }, + { + name: "viewer transfers count and other transfer roles do not", + prev: map[string]int64{"transfer:t1": 200}, + snapshot: streamtelemetry.Snapshot{ + Transfers: []streamtelemetry.TransferView{ + {ID: "t1", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 1_200}, + {ID: "t2", Role: streamtelemetry.RoleInternalRelay, BytesAccepted: 8_000}, + }, + }, + wantDelta: 1_000, + wantNext: map[string]int64{"transfer:t1": 1_200}, + }, + { + name: "sessions and transfers with the same id stay separate", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("x", 10)}, + Transfers: []streamtelemetry.TransferView{{ID: "x", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 20}}, + }, + wantDelta: 30, + wantNext: map[string]int64{"session:x": 10, "transfer:x": 20}, + }, + { + name: "a nil previous map behaves like an empty one", + prev: nil, + snapshot: streamtelemetry.Snapshot{Sessions: []streamtelemetry.SessionView{viewerSession("s1", 42)}}, + wantDelta: 42, + wantNext: map[string]int64{"session:s1": 42}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + delta, next := computeEgressDelta(tt.prev, tt.snapshot) + if delta != tt.wantDelta { + t.Fatalf("delta = %d, want %d", delta, tt.wantDelta) + } + if len(next) != len(tt.wantNext) { + t.Fatalf("next = %v, want %v", next, tt.wantNext) + } + for key, want := range tt.wantNext { + if got, ok := next[key]; !ok || got != want { + t.Fatalf("next[%q] = %d (present %t), want %d", key, got, ok, want) + } + } + }) + } +} + +func TestSampleBucket(t *testing.T) { + t.Parallel() + + newYork, err := time.LoadLocation("America/New_York") + if err != nil { + t.Skipf("timezone database unavailable: %v", err) + } + + tests := []struct { + name string + at time.Time + want time.Time + }{ + { + name: "seconds and nanoseconds are dropped", + at: time.Date(2026, 8, 26, 11, 58, 43, 987_654_321, time.UTC), + want: time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC), + }, + { + name: "a minute boundary is already its own bucket", + at: time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC), + want: time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC), + }, + { + name: "local times truncate on the UTC minute", + at: time.Date(2026, 8, 26, 7, 58, 43, 0, newYork), + want: time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := sampleBucket(tt.at) + if !got.Equal(tt.want) { + t.Fatalf("sampleBucket(%s) = %s, want %s", tt.at, got, tt.want) + } + if got.Location() != time.UTC { + t.Fatalf("bucket location = %s, want UTC", got.Location()) + } + }) + } +} + +func TestSampleBucketDetectsRepeatedMinutes(t *testing.T) { + t.Parallel() + + first := time.Date(2026, 8, 26, 11, 58, 1, 0, time.UTC) + again := time.Date(2026, 8, 26, 11, 58, 59, 0, time.UTC) + later := time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC) + + if !sampleBucket(first).Equal(sampleBucket(again)) { + t.Fatal("two ticks inside one minute produced different buckets") + } + if sampleBucket(first).Equal(sampleBucket(later)) { + t.Fatal("ticks in different minutes produced the same bucket") + } +} + +func TestEgressKbps(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + deltaBytes int64 + elapsed time.Duration + want int64 + }{ + {name: "no bytes is zero", deltaBytes: 0, elapsed: time.Minute, want: 0}, + {name: "a negative delta never reports a negative rate", deltaBytes: -5, elapsed: time.Minute, want: 0}, + {name: "a zero elapsed period cannot divide", deltaBytes: 1_000, elapsed: 0, want: 0}, + {name: "a backwards clock cannot divide", deltaBytes: 1_000, elapsed: -time.Second, want: 0}, + {name: "one megabyte a second is 8000 kbps", deltaBytes: 1_000_000, elapsed: time.Second, want: 8_000}, + {name: "a minute of bytes spreads over the minute", deltaBytes: 60_000_000, elapsed: time.Minute, want: 8_000}, + {name: "sub-kilobit rates round rather than truncate to zero", deltaBytes: 100, elapsed: time.Second, want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := egressKbps(tt.deltaBytes, tt.elapsed); got != tt.want { + t.Fatalf("egressKbps(%d, %s) = %d, want %d", tt.deltaBytes, tt.elapsed, got, tt.want) + } + }) + } +} + +func TestNewSamplerSourceKey(t *testing.T) { + t.Parallel() + + if got := NewSampler(nil, nil, "api-2").source; got != "proc:api-2" { + t.Fatalf("source = %q, want %q", got, "proc:api-2") + } + if got := NewSampler(nil, nil, "").source; got == "proc:" { + t.Fatal("an empty node id must fall back to a host identity, not an empty source key") + } +} + +func TestSamplerStopIsIdempotent(t *testing.T) { + t.Parallel() + + sampler := NewSampler(nil, nil, "api-1") + sampler.Stop() + sampler.Stop() +} diff --git a/internal/opslog/repo.go b/internal/opslog/repo.go index fb70adada..b282cf551 100644 --- a/internal/opslog/repo.go +++ b/internal/opslog/repo.go @@ -27,9 +27,13 @@ type EntryRow struct { } type ListOptions struct { - From *time.Time - To *time.Time + From *time.Time + To *time.Time + // Level filters on a single level. Levels filters on any of several and + // wins when both are set; Level stays for callers that only ever ask for + // one. Level string + Levels []string Component string NodeID string RequestID string @@ -55,6 +59,87 @@ func NewRepo(pool *pgxpool.Pool) *Repo { } func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { + query, args, limit, err := buildListQuery(opts) + if err != nil { + return ListResult{}, err + } + + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return ListResult{}, fmt.Errorf("list operational logs: %w", err) + } + defer rows.Close() + + entries := make([]EntryRow, 0, limit+1) + for rows.Next() { + var entry EntryRow + var attrsJSON []byte + if err := rows.Scan( + &entry.ID, + &entry.Timestamp, + &entry.Level, + &entry.Component, + &entry.Message, + &entry.RequestID, + &entry.UserID, + &entry.SessionID, + &entry.PlaybackSessionID, + &entry.ClientIP, + &entry.NodeID, + &attrsJSON, + ); err != nil { + return ListResult{}, fmt.Errorf("scan operational log row: %w", err) + } + if len(attrsJSON) > 0 { + if err := json.Unmarshal(attrsJSON, &entry.Attrs); err != nil { + return ListResult{}, fmt.Errorf("decode operational log attrs: %w", err) + } + } + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return ListResult{}, fmt.Errorf("iterate operational logs: %w", err) + } + + result := ListResult{} + if len(entries) > limit { + last := entries[limit-1] + result.NextCursor = encodeCursor(last.Timestamp, last.ID) + entries = entries[:limit] + } + result.Entries = entries + return result, nil +} + +// NormalizeLevels lowercases, trims and de-duplicates a level filter, dropping +// empty entries. It returns nil when nothing usable is left so callers can test +// the result for "no level filter". +func NormalizeLevels(levels []string) []string { + if len(levels) == 0 { + return nil + } + normalized := make([]string, 0, len(levels)) + seen := make(map[string]struct{}, len(levels)) + for _, level := range levels { + level = strings.ToLower(strings.TrimSpace(level)) + if level == "" { + continue + } + if _, ok := seen[level]; ok { + continue + } + seen[level] = struct{}{} + normalized = append(normalized, level) + } + if len(normalized) == 0 { + return nil + } + return normalized +} + +// buildListQuery renders the filtered query and its arguments. It is separate +// from List so the predicate assembly can be tested without a database. +func buildListQuery(opts ListOptions) (string, []any, int, error) { limit := opts.Limit if limit <= 0 || limit > 200 { limit = 100 @@ -74,7 +159,11 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { args = append(args, *opts.To) argIdx++ } - if opts.Level != "" { + if levels := NormalizeLevels(opts.Levels); len(levels) > 0 { + conditions = append(conditions, fmt.Sprintf("level = ANY($%d)", argIdx)) + args = append(args, levels) + argIdx++ + } else if opts.Level != "" { conditions = append(conditions, fmt.Sprintf("level = $%d", argIdx)) args = append(args, strings.ToLower(opts.Level)) argIdx++ @@ -117,7 +206,7 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { if opts.Cursor != "" { cursorTs, cursorID, err := decodeCursor(opts.Cursor) if err != nil { - return ListResult{}, err + return "", nil, 0, err } conditions = append(conditions, fmt.Sprintf("(timestamp, id) < ($%d, $%d)", argIdx, argIdx+1)) args = append(args, cursorTs, cursorID) @@ -134,51 +223,7 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { `, strings.Join(conditions, " AND "), argIdx) args = append(args, limit+1) - rows, err := r.pool.Query(ctx, query, args...) - if err != nil { - return ListResult{}, fmt.Errorf("list operational logs: %w", err) - } - defer rows.Close() - - entries := make([]EntryRow, 0, limit+1) - for rows.Next() { - var entry EntryRow - var attrsJSON []byte - if err := rows.Scan( - &entry.ID, - &entry.Timestamp, - &entry.Level, - &entry.Component, - &entry.Message, - &entry.RequestID, - &entry.UserID, - &entry.SessionID, - &entry.PlaybackSessionID, - &entry.ClientIP, - &entry.NodeID, - &attrsJSON, - ); err != nil { - return ListResult{}, fmt.Errorf("scan operational log row: %w", err) - } - if len(attrsJSON) > 0 { - if err := json.Unmarshal(attrsJSON, &entry.Attrs); err != nil { - return ListResult{}, fmt.Errorf("decode operational log attrs: %w", err) - } - } - entries = append(entries, entry) - } - if err := rows.Err(); err != nil { - return ListResult{}, fmt.Errorf("iterate operational logs: %w", err) - } - - result := ListResult{} - if len(entries) > limit { - last := entries[limit-1] - result.NextCursor = encodeCursor(last.Timestamp, last.ID) - entries = entries[:limit] - } - result.Entries = entries - return result, nil + return query, args, limit, nil } func encodeCursor(ts time.Time, id int64) string { diff --git a/internal/opslog/repo_test.go b/internal/opslog/repo_test.go new file mode 100644 index 000000000..d1a861b7a --- /dev/null +++ b/internal/opslog/repo_test.go @@ -0,0 +1,138 @@ +package opslog + +import ( + "reflect" + "strconv" + "strings" + "testing" +) + +func TestNormalizeLevels(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input []string + want []string + }{ + {name: "nil stays nil", input: nil, want: nil}, + {name: "blank entries are dropped", input: []string{"", " "}, want: nil}, + {name: "trimmed and lowercased", input: []string{" Error ", "WARN"}, want: []string{"error", "warn"}}, + {name: "duplicates collapse", input: []string{"error", "Error", "error"}, want: []string{"error"}}, + {name: "order is preserved", input: []string{"warn", "error", "info"}, want: []string{"warn", "error", "info"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := NormalizeLevels(tt.input); !reflect.DeepEqual(got, tt.want) { + t.Fatalf("NormalizeLevels(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestBuildListQueryLevelFilters(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + opts ListOptions + wantPredicate string + wantArg any + }{ + { + name: "no level filter", + opts: ListOptions{}, + wantPredicate: "", + }, + { + name: "single level keeps the equality predicate", + opts: ListOptions{Level: "ERROR"}, + wantPredicate: "level = $1", + wantArg: "error", + }, + { + name: "several levels use ANY", + opts: ListOptions{Levels: []string{"error", "warn"}}, + wantPredicate: "level = ANY($1)", + wantArg: []string{"error", "warn"}, + }, + { + name: "levels win over level", + opts: ListOptions{Level: "info", Levels: []string{"error"}}, + wantPredicate: "level = ANY($1)", + wantArg: []string{"error"}, + }, + { + name: "an all-blank levels list falls back to level", + opts: ListOptions{Level: "info", Levels: []string{"", " "}}, + wantPredicate: "level = $1", + wantArg: "info", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + query, args, _, err := buildListQuery(tt.opts) + if err != nil { + t.Fatalf("buildListQuery: %v", err) + } + if tt.wantPredicate == "" { + if strings.Contains(query, "level =") { + t.Fatalf("query filters on level without a filter being set:\n%s", query) + } + return + } + if !strings.Contains(query, tt.wantPredicate) { + t.Fatalf("query missing %q:\n%s", tt.wantPredicate, query) + } + if len(args) == 0 { + t.Fatal("no arguments bound") + } + if !reflect.DeepEqual(args[0], tt.wantArg) { + t.Fatalf("arg[0] = %#v, want %#v", args[0], tt.wantArg) + } + }) + } +} + +// The placeholder numbering has to stay in step with the argument slice +// whichever level filter is used, or a multi-filter query binds the wrong +// values. +func TestBuildListQueryPlaceholdersMatchArguments(t *testing.T) { + t.Parallel() + + userID := 42 + opts := ListOptions{ + Levels: []string{"error", "warn"}, + Component: "playback", + UserID: &userID, + Query: "timeout", + Limit: 25, + } + + query, args, limit, err := buildListQuery(opts) + if err != nil { + t.Fatalf("buildListQuery: %v", err) + } + if limit != 25 { + t.Fatalf("limit = %d, want 25", limit) + } + for i := range args { + placeholder := "$" + strconv.Itoa(i+1) + if !strings.Contains(query, placeholder) { + t.Fatalf("query does not bind %s:\n%s", placeholder, query) + } + } + // levels, component, user_id, message, limit + if len(args) != 5 { + t.Fatalf("args = %#v, want 5 entries", args) + } + if args[len(args)-1] != opts.Limit+1 { + t.Fatalf("limit arg = %v, want %d (limit + 1 for the cursor probe)", args[len(args)-1], opts.Limit+1) + } +} diff --git a/migrations/sql/20260827013713_admin_dashboard_layouts.sql b/migrations/sql/20260827013713_admin_dashboard_layouts.sql new file mode 100644 index 000000000..18b7f35ee --- /dev/null +++ b/migrations/sql/20260827013713_admin_dashboard_layouts.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- Per-admin-account admin dashboard widget layout. The server treats the blob +-- as opaque; the web client validates widget ids and spans when it loads it. +CREATE TABLE IF NOT EXISTS admin_dashboard_layouts ( + user_id integer PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + layout jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE IF EXISTS admin_dashboard_layouts; diff --git a/migrations/sql/20260827015256_dashboard_activity_indexes.sql b/migrations/sql/20260827015256_dashboard_activity_indexes.sql new file mode 100644 index 000000000..159babcdf --- /dev/null +++ b/migrations/sql/20260827015256_dashboard_activity_indexes.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- The admin dashboard's activity aggregates scan by start/watch time over a +-- rolling window. Both tables are only indexed on their end-of-play timestamps +-- today (idx_playback_history_admin_ended, idx_user_watch_history_*), which a +-- "started in the last N hours" or "watched in the last N days" filter cannot +-- use. +CREATE INDEX IF NOT EXISTS idx_playback_history_admin_started + ON public.playback_history_admin USING btree (started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_user_watch_history_watched_at + ON public.user_watch_history USING btree (watched_at DESC); + +-- +goose Down +DROP INDEX IF EXISTS public.idx_user_watch_history_watched_at; + +DROP INDEX IF EXISTS public.idx_playback_history_admin_started; diff --git a/migrations/sql/20260827021132_dashboard_metric_samples.sql b/migrations/sql/20260827021132_dashboard_metric_samples.sql new file mode 100644 index 000000000..dcc27f734 --- /dev/null +++ b/migrations/sql/20260827021132_dashboard_metric_samples.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- Minute-resolution samples for the admin dashboard's concurrent-stream and +-- egress charts. Neither series can be reconstructed after the fact: live +-- sessions leave no per-minute trace and node egress is a rolling average, so +-- the sampler writes them as they happen. +-- +-- One row per minute per source. 'shared' is the cluster-wide snapshot (any +-- replica may write it; the primary key makes the first writer win), while +-- 'proc:' rows carry the viewer egress served by one API process, +-- which stream_nodes does not cover. +CREATE TABLE IF NOT EXISTS dashboard_metric_samples ( + bucket timestamptz NOT NULL, + source text NOT NULL, + streams_total integer NOT NULL DEFAULT 0, + streams_direct integer NOT NULL DEFAULT 0, + streams_remux integer NOT NULL DEFAULT 0, + streams_transcode integer NOT NULL DEFAULT 0, + egress_kbps bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (bucket, source) +); + +-- +goose Down +DROP TABLE IF EXISTS dashboard_metric_samples; diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 7782ac3a9..ad39cceb2 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -4372,6 +4372,47 @@ export interface AdminSettingsUpdateResponse { restart_required_keys?: string[]; } +// Admin dashboard layout (per admin account, server-persisted). +// +// The server stores the document verbatim and validates only its size and that +// it is a JSON object: widget ids, column spans and row heights are the web +// client's vocabulary. `layout` is therefore typed as `unknown` on the read +// side so callers must sanitize it before use — a layout written by a newer or +// older build can name widgets this one does not have, or omit `rows`, which +// predates two-axis resizing. +export interface AdminDashboardLayoutEntry { + id: string; + span: number; + rows: number; +} + +export interface AdminDashboardLayoutDocument { + version: number; + entries: AdminDashboardLayoutEntry[]; +} + +export interface AdminDashboardLayoutResponse { + layout: unknown; + updated_at: string | null; +} + +// One backing service on the admin health strip. `configured: false` means the +// deployment runs without it — a supported single-node shape for Redis — and +// `ok` is then absent rather than false, so "not present" and "present but +// broken" stay distinguishable. +export interface AdminHealthComponent { + configured: boolean; + ok?: boolean; + latency_ms?: number; +} + +export interface AdminServerHealth { + postgres: AdminHealthComponent; + redis: AdminHealthComponent; + errors_24h: number; + warnings_24h: number; +} + export interface AdminServerStatus { started_at: string; restart_required: boolean; @@ -4379,6 +4420,87 @@ export interface AdminServerStatus { restart_required_reason?: string; restart_requested: boolean; restart_requested_at?: string; + health: AdminServerHealth; +} + +// GET /admin/stats/playback-activity. `buckets` carries only hours that saw a +// session, so the client zero-fills the window before charting it. +export interface AdminPlaybackActivityBucket { + hour: string; + direct: number; + remux: number; + transcode: number; +} + +// Time-to-first-frame and failed-start counts are deliberately absent: nothing +// records playback start events yet. See docs/admin-api.md. +export interface AdminPlaybackReliability { + sessions_started: number; + transcode_starts: number; + finalized_sessions: number; + completed_sessions: number; + completion_rate: number; + unique_profiles: number; +} + +// `bucket_seconds` is 3600 up to a two-day window and 86400 beyond it; the +// client zero-fills the window on that grid. `hour` on a bucket is its start +// instant at either width. +export interface AdminPlaybackActivity { + hours: number; + bucket_seconds: number; + buckets: AdminPlaybackActivityBucket[]; + reliability: AdminPlaybackReliability; + profiles_active_24h: number; +} + +// GET /admin/stats/top-activity. Episodes are rolled up to their series, so a +// title's media_item_id is a series content id for TV. +export interface AdminTopTitle { + media_item_id: string; + title: string; + media_type: string; + plays: number; + total_seconds: number; +} + +export interface AdminTopProfile { + user_id: number; + username: string; + profile_id: string; + profile_name: string; + plays: number; + total_seconds: number; +} + +export interface AdminTopActivity { + days: number; + limit: number; + titles: AdminTopTitle[]; + profiles: AdminTopProfile[]; +} + +// GET /admin/stats/timeseries. One point per sampled minute; minutes the +// sampler missed are absent rather than zero, so charts draw them as gaps. +export interface AdminTimeseriesPoint { + t: string; + streams: number; + direct: number; + remux: number; + transcode: number; + egress_kbps: number; +} + +// `oldest_sample_at` is null until the sampler has written anything, which is +// what the "collecting data" chart state keys off. `resolution_seconds` is the +// bucket the server aggregated into, which widens with the requested window — +// read it rather than assuming the sampler's minute. +export interface AdminTimeseries { + resolution_seconds: number; + from: string; + to: string; + oldest_sample_at: string | null; + points: AdminTimeseriesPoint[]; } // IP visibility diff --git a/web/src/app.css b/web/src/app.css index ab8011fcb..58f90f072 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -369,12 +369,14 @@ --input: #1c1c20; --ring: #e8e8ec; - /* Charts: desaturated pastels that fit the cinema aesthetic */ - --chart-1: #8b9cf7; - --chart-2: #7ec8e3; - --chart-3: #81c995; - --chart-4: #e8a87c; - --chart-5: #c78dbd; + /* Charts: categorical set validated on this theme's card surface — + lightness band, chroma floor, adjacent-pair CVD separation, and + contrast all pass. Assigned in fixed entity order, never cycled. */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; /* Sidebar: darkest surface, creating depth */ --sidebar: #0f0f12; @@ -431,12 +433,13 @@ --input: #e8e8ec; --ring: #1a1a1e; - /* Charts: richer tones for light backgrounds */ - --chart-1: #6366f1; - --chart-2: #0ea5e9; - --chart-3: #22c55e; - --chart-4: #f59e0b; - --chart-5: #a855f7; + /* Charts: light-mode steps of the same categorical set, validated against + the #ffffff card surface (dark mode is selected, never an auto flip). */ + --chart-1: #4c60dd; + --chart-2: #0b8a75; + --chart-3: #9c6410; + --chart-4: #a63f75; + --chart-5: #2b78ab; /* Sidebar: slightly darker than background for depth */ --sidebar: #eaeaee; @@ -485,11 +488,12 @@ --border: #28384d; --input: #182231; --ring: #78aefc; - --chart-1: #78aefc; - --chart-2: #87d2ff; - --chart-3: #6ec7ba; - --chart-4: #b9c7ff; - --chart-5: #f18d8d; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #0c131d; --sidebar-foreground: #f4f8ff; --sidebar-primary: #78aefc; @@ -529,11 +533,12 @@ --border: #3b2830; --input: #251a1f; --ring: #d16a78; - --chart-1: #d16a78; - --chart-2: #f08e7a; - --chart-3: #c8a0b8; - --chart-4: #8aa0c6; - --chart-5: #e1b86e; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #120d0f; --sidebar-foreground: #f8f2f3; --sidebar-primary: #d16a78; @@ -573,11 +578,12 @@ --border: #284038; --input: #182420; --ring: #5bc39d; - --chart-1: #5bc39d; - --chart-2: #7dd8bd; - --chart-3: #7eb7a4; - --chart-4: #88a9cf; - --chart-5: #e0b66b; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #0c1210; --sidebar-foreground: #f2f8f5; --sidebar-primary: #5bc39d; @@ -1907,11 +1913,20 @@ /* Admin dashboard widget grid: single column on small screens, two columns on tablets (where .admin-widget-wide keeps the full row), and a 12-column - grid from lg up where each widget spans var(--widget-span) columns. */ + grid from lg up where each widget spans var(--widget-span) columns and + var(--widget-rows) fixed-height rows. + + Rows are only fixed from lg up. Below that the grid is one or two + content-sized columns, so a widget is already as narrow as it gets and a + row span picked for the wide layout would clip it — the same reason column + spans stop applying there. */ .admin-widget-grid { display: grid; grid-template-columns: 1fr; gap: 0.875rem; + /* Height of one widget row. Declared here rather than in the media query so + DashboardGrid can read it back for a drag; it only takes effect at lg. */ + --admin-row-h: 6.25rem; } .admin-widget { @@ -1937,10 +1952,12 @@ @media (min-width: 1024px) { .admin-widget-grid { grid-template-columns: repeat(12, 1fr); + grid-auto-rows: var(--admin-row-h); } .admin-widget { grid-column: span var(--widget-span) / span var(--widget-span); + grid-row: span var(--widget-rows) / span var(--widget-rows); } } diff --git a/web/src/components/admin/dashboard/DashboardGrid.tsx b/web/src/components/admin/dashboard/DashboardGrid.tsx index 518dc1726..6896e62fa 100644 --- a/web/src/components/admin/dashboard/DashboardGrid.tsx +++ b/web/src/components/admin/dashboard/DashboardGrid.tsx @@ -17,11 +17,41 @@ import { cn } from "@/lib/utils"; import { getDashboardWidget } from "./registry"; import type { WidgetId } from "./types"; import type { DashboardLayout } from "./useDashboardLayout"; +import { WidgetChromeProvider } from "./widgetChrome"; /** Must match the `gap` of `.admin-widget-grid` in app.css (0.875rem). */ const GRID_GAP_PX = 14; +/** Must match `--admin-row-h` on `.admin-widget-grid` in app.css (6.25rem). */ +const GRID_ROW_HEIGHT_PX = 100; const GRID_COLUMNS = 12; +/** + * Row height in CSS pixels. + * + * Read back from `--admin-row-h` so a drag follows the stylesheet instead of a + * second copy of the number; `GRID_ROW_HEIGHT_PX` is the fallback for anything + * that cannot resolve the variable (jsdom, a grid that is not mounted yet). + */ +function readRowHeightPx(grid: HTMLElement | null): number { + if (!grid) { + return GRID_ROW_HEIGHT_PX; + } + const raw = window.getComputedStyle(grid).getPropertyValue("--admin-row-h").trim(); + const value = Number.parseFloat(raw); + if (!Number.isFinite(value) || value <= 0) { + return GRID_ROW_HEIGHT_PX; + } + if (raw.endsWith("rem")) { + const root = Number.parseFloat(window.getComputedStyle(document.documentElement).fontSize); + return Number.isFinite(root) && root > 0 ? value * root : GRID_ROW_HEIGHT_PX; + } + return value; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + interface DropIndicator { id: WidgetId; edge: "before" | "after"; @@ -30,6 +60,7 @@ interface DropIndicator { interface ResizePreview { id: WidgetId; span: number; + rows: number; } export function DashboardGrid({ @@ -47,6 +78,7 @@ export function DashboardGrid({ isCustomizing, moveWidget, resizeWidget, + setWidgetRange, removeWidget, addWidget, } = layout; @@ -58,12 +90,19 @@ export function DashboardGrid({ const [liveMessage, setLiveMessage] = useState(""); const resizeSessionRef = useRef<{ id: WidgetId; + title: string; startX: number; + startY: number; startSpan: number; - unit: number; + startRows: number; + columnUnit: number; + rowUnit: number; minSpan: number; maxSpan: number; + minRows: number; + maxRows: number; latestSpan: number; + latestRows: number; } | null>(null); const findWidgetIdFromEvent = useCallback((event: DragEvent): WidgetId | null => { @@ -137,7 +176,12 @@ export function DashboardGrid({ ); const handleResizePointerDown = useCallback( - (event: ReactPointerEvent, id: WidgetId, currentSpan: number) => { + ( + event: ReactPointerEvent, + id: WidgetId, + currentSpan: number, + currentRows: number, + ) => { if (!isCustomizing) return; // Only start on a primary-button press: a right-click opens the context // menu and never delivers the matching pointerup, which would leave the @@ -148,17 +192,24 @@ export function DashboardGrid({ event.stopPropagation(); event.currentTarget.setPointerCapture(event.pointerId); const gridWidth = gridRef.current?.getBoundingClientRect().width ?? 0; - const unit = gridWidth > 0 ? (gridWidth + GRID_GAP_PX) / GRID_COLUMNS : 1; + const columnUnit = gridWidth > 0 ? (gridWidth + GRID_GAP_PX) / GRID_COLUMNS : 1; resizeSessionRef.current = { id, + title: widget.title, startX: event.clientX, + startY: event.clientY, startSpan: currentSpan, - unit, + startRows: currentRows, + columnUnit, + rowUnit: readRowHeightPx(gridRef.current) + GRID_GAP_PX, minSpan: widget.minSpan, maxSpan: widget.maxSpan, + minRows: widget.minRows, + maxRows: widget.maxRows, latestSpan: currentSpan, + latestRows: currentRows, }; - setResizePreview({ id, span: currentSpan }); + setResizePreview({ id, span: currentSpan, rows: currentRows }); }, [isCustomizing], ); @@ -166,12 +217,24 @@ export function DashboardGrid({ const handleResizePointerMove = useCallback((event: ReactPointerEvent) => { const session = resizeSessionRef.current; if (!session) return; - const raw = session.startSpan + (event.clientX - session.startX) / session.unit; - const next = Math.min(session.maxSpan, Math.max(session.minSpan, Math.round(raw))); - if (next !== session.latestSpan) { - session.latestSpan = next; - setResizePreview({ id: session.id, span: next }); + // Each axis is clamped to its own range, so a widget with a pinned width + // still grows in height and never drifts sideways under the pointer. + const nextSpan = clamp( + Math.round(session.startSpan + (event.clientX - session.startX) / session.columnUnit), + session.minSpan, + session.maxSpan, + ); + const nextRows = clamp( + Math.round(session.startRows + (event.clientY - session.startY) / session.rowUnit), + session.minRows, + session.maxRows, + ); + if (nextSpan === session.latestSpan && nextRows === session.latestRows) { + return; } + session.latestSpan = nextSpan; + session.latestRows = nextRows; + setResizePreview({ id: session.id, span: nextSpan, rows: nextRows }); }, []); const handleResizePointerEnd = useCallback( @@ -183,7 +246,10 @@ export function DashboardGrid({ if (event.currentTarget.hasPointerCapture(event.pointerId)) { event.currentTarget.releasePointerCapture(event.pointerId); } - resizeWidget(session.id, session.latestSpan); + resizeWidget(session.id, { span: session.latestSpan, rows: session.latestRows }); + setLiveMessage( + `${session.title} resized to ${session.latestSpan} of ${GRID_COLUMNS} columns × ${session.latestRows} ${session.latestRows === 1 ? "row" : "rows"}`, + ); }, [resizeWidget], ); @@ -213,19 +279,27 @@ export function DashboardGrid({ ); const handleResizeKeyDown = useCallback( - (event: ReactKeyboardEvent, id: WidgetId, currentSpan: number) => { - const shrink = event.key === "ArrowLeft" || event.key === "ArrowDown"; - const grow = event.key === "ArrowRight" || event.key === "ArrowUp"; - if (!shrink && !grow) return; + ( + event: ReactKeyboardEvent, + id: WidgetId, + currentSpan: number, + currentRows: number, + ) => { + // The corner handle owns both axes: left/right walk columns, up/down walk + // rows, matching the direction the same drag would take. + const columnStep = + event.key === "ArrowLeft" ? -1 : event.key === "ArrowRight" ? 1 : undefined; + const rowStep = event.key === "ArrowUp" ? -1 : event.key === "ArrowDown" ? 1 : undefined; + if (columnStep === undefined && rowStep === undefined) return; event.preventDefault(); const widget = getDashboardWidget(id); - const next = Math.min( - widget.maxSpan, - Math.max(widget.minSpan, currentSpan + (grow ? 1 : -1)), + const nextSpan = clamp(currentSpan + (columnStep ?? 0), widget.minSpan, widget.maxSpan); + const nextRows = clamp(currentRows + (rowStep ?? 0), widget.minRows, widget.maxRows); + if (nextSpan === currentSpan && nextRows === currentRows) return; + resizeWidget(id, { span: nextSpan, rows: nextRows }); + setLiveMessage( + `${widget.title} resized to ${nextSpan} of ${GRID_COLUMNS} columns × ${nextRows} ${nextRows === 1 ? "row" : "rows"}`, ); - if (next === currentSpan) return; - resizeWidget(id, next); - setLiveMessage(`${widget.title} resized to ${next} of ${GRID_COLUMNS} columns`); }, [resizeWidget], ); @@ -247,9 +321,10 @@ export function DashboardGrid({ > {entries.map((entry) => { const widget = getDashboardWidget(entry.id); - const span = resizePreview?.id === entry.id ? resizePreview.span : entry.span; - const canResize = widget.minSpan !== widget.maxSpan; const isWidgetResizing = resizePreview?.id === entry.id; + const span = isWidgetResizing ? resizePreview.span : entry.span; + const rows = isWidgetResizing ? resizePreview.rows : entry.rows; + const canResize = widget.minSpan !== widget.maxSpan || widget.minRows !== widget.maxRows; const WidgetComponent = widget.Component; return ( @@ -262,10 +337,19 @@ export function DashboardGrid({ isCustomizing && "rounded-2xl", draggedId === entry.id && "opacity-40", )} - style={{ "--widget-span": span } as CSSProperties} + style={{ "--widget-span": span, "--widget-rows": rows } as CSSProperties} draggable={isCustomizing && !isResizing} > - + {/* The window is resolved here rather than in the widget: the + entry may predate the widget gaining ranges, in which case the + registry's default is what it has always been showing. */} + + + {isCustomizing && ( <> @@ -308,20 +392,28 @@ export function DashboardGrid({
+ {/* One handle for both axes, straddling the bottom-right + corner. Column spans (and row heights) only apply from lg + up, so the handle is hidden below that. */} {canResize && ( + ); + })} +
+ ); +} + +/** + * The range picker as a widget wears it: options come from the registry, the + * current value and the setter from the grid. Renders nothing when the widget + * is not placed in a grid (unit tests) or offers no ranges. + */ +export function WidgetRangePicker({ className }: { className?: string }) { + const { id, range, setRange } = useWidgetRange(); + const ranges = id ? findDashboardWidget(id)?.ranges : undefined; + if (!ranges) { + return null; + } + return ( + + ); +} diff --git a/web/src/components/admin/dashboard/charts/BarList.tsx b/web/src/components/admin/dashboard/charts/BarList.tsx new file mode 100644 index 000000000..8d46988cd --- /dev/null +++ b/web/src/components/admin/dashboard/charts/BarList.tsx @@ -0,0 +1,98 @@ +import { Link } from "react-router"; + +import { cn } from "@/lib/utils"; +import { chartSeriesColor } from "./chartMath"; + +export interface BarListItem { + id: string; + label: string; + value: number; + /** Muted secondary metric rendered after the value (e.g. watch hours). */ + secondary?: string; + /** Optional admin route the label links to. */ + to?: string; +} + +export interface BarListProps { + items: readonly BarListItem[]; + /** Categorical slot for the track color; bar lists are single-hue by design. */ + seriesIndex?: number; + formatValue?: (value: number) => string; + emptyLabel?: string; + className?: string; +} + +function defaultFormatValue(value: number): string { + return value.toLocaleString(); +} + +/** + * Ranked horizontal bars for "top N" lists. + * + * One hue for every row — the bars encode magnitude, not identity, so there is + * nothing for a legend to say. Labels and values stay in text tokens over the + * track wash. + */ +export function BarList({ + items, + seriesIndex = 0, + formatValue = defaultFormatValue, + emptyLabel = "No activity yet", + className, +}: BarListProps) { + if (items.length === 0) { + return
{emptyLabel}
; + } + + const color = chartSeriesColor(seriesIndex); + const max = items.reduce( + (peak, item) => (Number.isFinite(item.value) ? Math.max(peak, item.value) : peak), + 0, + ); + + return ( +
    + {items.map((item, index) => { + const value = Number.isFinite(item.value) ? Math.max(item.value, 0) : 0; + const ratio = max > 0 ? value / max : 0; + return ( +
  1. +
    2. + ); + })} +
+ ); +} diff --git a/web/src/components/admin/dashboard/charts/ChartEmptyState.tsx b/web/src/components/admin/dashboard/charts/ChartEmptyState.tsx new file mode 100644 index 000000000..935abdca5 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/ChartEmptyState.tsx @@ -0,0 +1,67 @@ +import { Skeleton } from "@/components/ui/skeleton"; +import { formatRelativeTime } from "@/lib/date"; +import { cn } from "@/lib/utils"; + +/** + * Placeholder for a chart with nothing to draw yet. + * + * The metrics sampler only collects while the server runs, so a fresh install + * has no history — say that plainly ("Collecting data — samples since …") + * instead of drawing an empty axis that reads as broken. + */ +export function ChartEmptyState({ + height = 160, + fill = false, + message = "No data yet", + since, + detail, + className, +}: { + height?: number; + /** Fill the parent instead of reserving `height`, matching a fill-height chart. */ + fill?: boolean; + message?: string; + /** ISO timestamp of the oldest sample; switches the copy to the collecting state. */ + since?: string | null; + detail?: string; + className?: string; +}) { + const collectingSince = since ? formatRelativeTime(since) : null; + const headline = collectingSince ? "Collecting data" : message; + const sub = collectingSince ? `Samples since ${collectingSince}` : detail; + + return ( +
+
{headline}
+ {sub ?
{sub}
: null} +
+ ); +} + +/** Loading placeholder for a plotted chart. */ +export function ChartSkeleton({ height = 160, fill = false }: { height?: number; fill?: boolean }) { + return ( + + ); +} + +/** Loading placeholder for a bar list. */ +export function BarListSkeleton({ rows = 5 }: { rows?: number }) { + return ( +
+ {Array.from({ length: rows }).map((_, i) => ( + + ))} +
+ ); +} diff --git a/web/src/components/admin/dashboard/charts/LineChart.tsx b/web/src/components/admin/dashboard/charts/LineChart.tsx new file mode 100644 index 000000000..a00eaae26 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/LineChart.tsx @@ -0,0 +1,283 @@ +import { + useMemo, + useState, + type PointerEvent as ReactPointerEvent, + type KeyboardEvent, +} from "react"; + +import { formatTime } from "@/lib/datetime"; +import { cn } from "@/lib/utils"; +import { buildAreaPath, buildLinePath, chartSeriesColor, niceTicks } from "./chartMath"; +import { ChartTooltip, ChartTooltipRow } from "./chartChrome"; +import { useMeasuredSize } from "./useMeasuredSize"; + +export interface LineChartPoint { + /** Sample timestamp in epoch milliseconds. */ + t: number; + /** `null` is a missing sample: the line breaks rather than inventing a zero. */ + value: number | null; +} + +export interface LineChartProps { + points: readonly LineChartPoint[]; + /** Plot height in pixels (axis labels sit outside it); ignored when `fill`. */ + height?: number; + /** + * Take the height of the parent instead of `height`. The chart then follows + * the widget's row height, which the admin can change at any time. + */ + fill?: boolean; + /** Categorical slot for the series color; single-series charts stay on slot 0. */ + seriesIndex?: number; + /** Series name for the hover readout. */ + seriesLabel?: string; + formatValue?: (value: number) => string; + /** Axis tick label, defaults to `formatValue`; pass a compact form when units are long. */ + formatTick?: (value: number) => string; + formatTimestamp?: (t: number) => string; + /** + * Labels for the two ends of the time axis. Absolute timestamps answer "when + * exactly" — which is the tooltip's job — while the axis only has to say how + * far back the window reaches, so a chart with a chosen window passes + * something like `{ start: "7d ago", end: "now" }`. + */ + edgeLabels?: { start: string; end: string }; + /** Smallest axis tick gap — counts pass 1 to avoid fractional gridlines. */ + minTickStep?: number; + ariaLabel: string; + className?: string; +} + +// Fallback plot box, used only until the ResizeObserver reports the real one. +// It is stretched to the card (`preserveAspectRatio="none"`); strokes opt out +// of that scaling so the line stays exactly 2px at every widget span. Once the +// plot has been measured the viewBox matches its CSS pixels one to one, so the +// same attribute stretches nothing and marks keep their shape at any height. +const VIEW_WIDTH = 1000; +const VIEW_HEIGHT = 100; + +function defaultFormatValue(value: number): string { + return value.toLocaleString(); +} + +/** + * Single-series time line with an area wash, a snapping crosshair, and gaps + * where samples are missing. One value axis; no legend (the card header names + * the series). + */ +export function LineChart({ + points, + height = 160, + fill = false, + seriesIndex = 0, + seriesLabel = "Value", + formatValue = defaultFormatValue, + formatTick, + formatTimestamp = (t) => formatTime(t), + edgeLabels, + minTickStep, + ariaLabel, + className, +}: LineChartProps) { + const [activeIndex, setActiveIndex] = useState(null); + const color = chartSeriesColor(seriesIndex); + const { ref: plotRef, size: plotSize } = useMeasuredSize(); + const viewWidth = plotSize && plotSize.width > 0 ? plotSize.width : VIEW_WIDTH; + const viewHeight = plotSize && plotSize.height > 0 ? plotSize.height : VIEW_HEIGHT; + + const geometry = useMemo(() => { + const values = points + .map((point) => point.value) + .filter((value): value is number => value !== null && Number.isFinite(value)); + const ticks = niceTicks(0, values.length > 0 ? Math.max(...values) : 0, 3, { + minStep: minTickStep, + }); + const top = ticks[ticks.length - 1] || 1; + + const firstT = points[0]?.t ?? 0; + const lastT = points[points.length - 1]?.t ?? 0; + const span = lastT - firstT; + const xRatios = points.map((point, index) => { + if (points.length <= 1) { + return 0.5; + } + if (span <= 0) { + return index / (points.length - 1); + } + return (point.t - firstT) / span; + }); + + const projected = points.map((point, index) => ({ + x: (xRatios[index] ?? 0) * viewWidth, + y: + point.value === null || !Number.isFinite(point.value) + ? null + : viewHeight - (point.value / top) * viewHeight, + })); + + return { + ticks, + top, + xRatios, + linePath: buildLinePath(projected), + areaPath: buildAreaPath(projected, viewHeight), + }; + }, [points, minTickStep, viewWidth, viewHeight]); + + const plottedIndexes = useMemo( + () => + points.map((point, index) => ({ point, index })).filter(({ point }) => point.value !== null), + [points], + ); + + function nearestIndex(ratio: number): number | null { + let best: number | null = null; + let bestDistance = Number.POSITIVE_INFINITY; + for (const { index } of plottedIndexes) { + const distance = Math.abs((geometry.xRatios[index] ?? 0) - ratio); + if (distance < bestDistance) { + bestDistance = distance; + best = index; + } + } + return best; + } + + function handlePointerMove(event: ReactPointerEvent) { + const bounds = event.currentTarget.getBoundingClientRect(); + if (bounds.width <= 0) { + return; + } + setActiveIndex(nearestIndex((event.clientX - bounds.left) / bounds.width)); + } + + function handleKeyDown(event: KeyboardEvent) { + if (plottedIndexes.length === 0) { + return; + } + if (event.key !== "ArrowLeft" && event.key !== "ArrowRight") { + return; + } + event.preventDefault(); + const order = plottedIndexes.map(({ index }) => index); + const current = activeIndex === null ? -1 : order.indexOf(activeIndex); + const step = event.key === "ArrowRight" ? 1 : -1; + const next = + current === -1 ? order.length - 1 : Math.min(Math.max(current + step, 0), order.length - 1); + setActiveIndex(order[next] ?? null); + } + + const activePoint = activeIndex === null ? null : (points[activeIndex] ?? null); + const activeValue = activePoint && activePoint.value !== null ? activePoint.value : null; + const activeRatio = activeIndex === null ? 0 : (geometry.xRatios[activeIndex] ?? 0); + const firstPoint = points[0]; + const lastPoint = points[points.length - 1]; + + return ( +
+
+
+ {geometry.ticks.map((tick) => ( + + {(formatTick ?? formatValue)(tick)} + + ))} +
+
+ + {geometry.ticks.map((tick) => { + const y = (1 - tick / geometry.top) * viewHeight; + return ( + + ); + })} + {geometry.areaPath ? ( + + ) : null} + {geometry.linePath ? ( + + ) : null} + + + {activePoint && activeValue !== null ? ( + <> + + ); +} diff --git a/web/src/components/admin/dashboard/charts/Sparkline.tsx b/web/src/components/admin/dashboard/charts/Sparkline.tsx new file mode 100644 index 000000000..00f29b4f7 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/Sparkline.tsx @@ -0,0 +1,75 @@ +import { useMemo } from "react"; + +import { cn } from "@/lib/utils"; +import { buildAreaPath, buildLinePath, chartSeriesColor } from "./chartMath"; + +export interface SparklineProps { + /** Evenly spaced samples; `null` breaks the line instead of reading as zero. */ + values: readonly (number | null)[]; + height?: number; + seriesIndex?: number; + showArea?: boolean; + ariaLabel: string; + className?: string; +} + +const VIEW_WIDTH = 100; +const VIEW_HEIGHT = 100; + +/** + * Axis-less trend line for stat tiles. No hover layer: the tile's value is the + * reading, the sparkline only shows its shape over the window. + */ +export function Sparkline({ + values, + height = 32, + seriesIndex = 0, + showArea = true, + ariaLabel, + className, +}: SparklineProps) { + const color = chartSeriesColor(seriesIndex); + + const { linePath, areaPath } = useMemo(() => { + const numeric = values.filter( + (value): value is number => value !== null && Number.isFinite(value), + ); + const max = numeric.length > 0 ? Math.max(...numeric) : 0; + const top = max > 0 ? max : 1; + const projected = values.map((value, index) => ({ + x: values.length <= 1 ? VIEW_WIDTH / 2 : (index / (values.length - 1)) * VIEW_WIDTH, + y: + value === null || !Number.isFinite(value) + ? null + : VIEW_HEIGHT - (value / top) * VIEW_HEIGHT, + })); + return { + linePath: buildLinePath(projected), + areaPath: showArea ? buildAreaPath(projected, VIEW_HEIGHT) : "", + }; + }, [values, showArea]); + + return ( + + {areaPath ? : null} + {linePath ? ( + + ) : null} + + ); +} diff --git a/web/src/components/admin/dashboard/charts/StackedColumnChart.tsx b/web/src/components/admin/dashboard/charts/StackedColumnChart.tsx new file mode 100644 index 000000000..32fcf6ff9 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/StackedColumnChart.tsx @@ -0,0 +1,191 @@ +import { useMemo, useState } from "react"; + +import { formatTime } from "@/lib/datetime"; +import { cn } from "@/lib/utils"; +import { chartSeriesColor, stackSegments } from "./chartMath"; +import { ChartLegend, ChartTooltip, ChartTooltipRow } from "./chartChrome"; + +export interface StackedColumnBucket { + /** Bucket start in epoch milliseconds. */ + t: number; + /** One value per series, in the same order as `seriesLabels`. */ + segments: readonly number[]; +} + +export interface StackedColumnChartProps { + buckets: readonly StackedColumnBucket[]; + /** Series names, in fixed entity order — index 0 sits at the baseline. */ + seriesLabels: readonly string[]; + /** Plot height in pixels (legend and tick labels sit outside it); ignored when `fill`. */ + height?: number; + /** + * Take the height of the parent instead of `height`, so the columns follow + * the widget's row height. The marks are laid out in CSS rather than in a + * viewBox, so a taller box simply gives every column more room. + */ + fill?: boolean; + formatValue?: (value: number) => string; + formatBucket?: (t: number) => string; + /** Label for the tooltip's total row; omit to hide the row. */ + totalLabel?: string; + ariaLabel: string; + className?: string; +} + +/** Marks stay thin: the leftover band width is deliberate air, not padding. */ +const MAX_COLUMN_WIDTH = 24; +/** Enough height that a single play is still visible above the baseline. */ +const MIN_SEGMENT_PX = 2; +const TARGET_TICK_LABELS = 4; + +function defaultFormatValue(value: number): string { + return value.toLocaleString(); +} + +/** + * Baseline-anchored stacked columns for bucketed counts (playback by method). + * + * Segments and neighbouring columns are separated by a 2px gap in the card + * surface rather than by outlines, only the topmost segment of a column is + * rounded, and every column carries its own hover/focus readout. + */ +export function StackedColumnChart({ + buckets, + seriesLabels, + height = 160, + fill = false, + formatValue = defaultFormatValue, + formatBucket = (t) => formatTime(t), + totalLabel = "Total", + ariaLabel, + className, +}: StackedColumnChartProps) { + const [activeIndex, setActiveIndex] = useState(null); + + const columns = useMemo( + () => + buckets.map((bucket) => { + const segments = stackSegments(seriesLabels.map((_, index) => bucket.segments[index] ?? 0)); + const total = segments[segments.length - 1]?.end ?? 0; + let topIndex = -1; + for (const segment of segments) { + if (segment.value > 0) { + topIndex = segment.index; + } + } + return { t: bucket.t, segments, total, topIndex }; + }), + [buckets, seriesLabels], + ); + + const max = columns.reduce((peak, column) => Math.max(peak, column.total), 0); + const scaleMax = max > 0 ? max : 1; + const tickStride = Math.max(1, Math.ceil(columns.length / TARGET_TICK_LABELS)); + const active = activeIndex === null ? null : (columns[activeIndex] ?? null); + const columnRatio = (index: number) => + columns.length > 0 ? (index + 0.5) / columns.length : 0.5; + const activeRatio = activeIndex === null ? 0.5 : columnRatio(activeIndex); + + return ( +
+ ({ + label, + color: chartSeriesColor(index), + }))} + /> +
+
setActiveIndex(null)} + > + {columns.map((column, index) => ( +
setActiveIndex(index)} + onFocus={() => setActiveIndex(index)} + onBlur={() => setActiveIndex(null)} + > +
+ {column.segments.map((segment) => + segment.value > 0 ? ( +
+ ) : null, + )} +
+
+ ))} +
+ +
+ {columns.map((column, index) => { + if (index % tickStride !== 0) { + return null; + } + const ratio = columnRatio(index); + return ( + 0.9 + ? "translateX(-100%)" + : "translateX(-50%)", + }} + > + {formatBucket(column.t)} + + ); + })} +
+
+ ); +} diff --git a/web/src/components/admin/dashboard/charts/chartChrome.tsx b/web/src/components/admin/dashboard/charts/chartChrome.tsx new file mode 100644 index 000000000..765285663 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/chartChrome.tsx @@ -0,0 +1,125 @@ +import type { ReactNode } from "react"; + +import { cn } from "@/lib/utils"; + +/** + * Small pieces every chart in this folder shares: the legend that keeps + * identity off color alone, and the hover readout. Kept here rather than in + * each chart so the two never drift apart visually. + */ + +export interface ChartLegendEntry { + label: string; + /** A `var(--chart-N)` token, from `chartSeriesColor`. */ + color: string; +} + +/** + * Legend row. Always rendered for two or more series; a single-series chart is + * titled by its card header instead. The swatch mirrors the mark (a rect for + * columns and areas, a short stroke for lines) and the label wears text tokens, + * never the series color. + */ +export function ChartLegend({ + entries, + shape = "rect", + className, +}: { + entries: readonly ChartLegendEntry[]; + shape?: "rect" | "line"; + className?: string; +}) { + if (entries.length < 2) { + return null; + } + return ( +
+ {entries.map((entry) => ( + + + ))} +
+ ); +} + +function clampRatio(ratio: number): number { + if (!Number.isFinite(ratio)) { + return 0; + } + return Math.min(Math.max(ratio, 0), 1); +} + +/** + * Hover/focus readout anchored above the plot at `xRatio` (0 = left edge, + * 1 = right edge). It shifts to stay inside the card near the edges instead of + * overflowing the widget. + */ +export function ChartTooltip({ + xRatio, + title, + children, + className, +}: { + xRatio: number; + title: string; + children?: ReactNode; + className?: string; +}) { + const ratio = clampRatio(xRatio); + const shift = ratio < 0.15 ? "0%" : ratio > 0.85 ? "-100%" : "-50%"; + return ( +
+
+
+ {title} +
+ {children} +
+
+ ); +} + +/** + * One tooltip line: the value leads in strong ink, the series name follows in + * muted ink beside a short stroke in the series color. + */ +export function ChartTooltipRow({ + color, + label, + value, +}: { + color?: string; + label: string; + value: string; +}) { + return ( +
+ {color ? ( +
+ ); +} diff --git a/web/src/components/admin/dashboard/charts/chartMath.test.ts b/web/src/components/admin/dashboard/charts/chartMath.test.ts new file mode 100644 index 000000000..3141ae088 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/chartMath.test.ts @@ -0,0 +1,278 @@ +import { describe, expect, it } from "vitest"; + +import { + buildAreaPath, + buildLinePath, + chartSeriesColor, + niceTicks, + stackSegments, + timeBuckets, + type LinePathPoint, +} from "./chartMath"; + +describe("chartSeriesColor", () => { + const cases: { name: string; index: number; expected: string }[] = [ + { name: "first slot", index: 0, expected: "var(--chart-1)" }, + { name: "third slot", index: 2, expected: "var(--chart-3)" }, + { name: "last slot", index: 4, expected: "var(--chart-5)" }, + { name: "clamps past the last slot instead of cycling", index: 9, expected: "var(--chart-5)" }, + { name: "clamps negative indexes", index: -3, expected: "var(--chart-1)" }, + { name: "falls back for non-finite indexes", index: Number.NaN, expected: "var(--chart-1)" }, + ]; + + for (const { name, index, expected } of cases) { + it(name, () => { + expect(chartSeriesColor(index)).toBe(expected); + }); + } +}); + +describe("niceTicks", () => { + const cases: { + name: string; + args: Parameters; + expected: number[]; + }[] = [ + { name: "rounds a small count range up", args: [0, 3, 4], expected: [0, 2, 4] }, + { name: "steps a wide range by tens", args: [0, 100, 5], expected: [0, 20, 40, 60, 80, 100] }, + { name: "handles a fractional range", args: [0, 0.4, 4], expected: [0, 0.2, 0.4] }, + { name: "swaps reversed bounds", args: [10, 0, 3], expected: [0, 5, 10] }, + { name: "expands a flat series", args: [0, 0, 4], expected: [0, 0.5, 1] }, + { + name: "honors a minimum step for counted values", + args: [0, 0, 4, { minStep: 1 }], + expected: [0, 1], + }, + { + name: "keeps integer steps when the range is tiny", + args: [0, 1, 4, { minStep: 1 }], + expected: [0, 1], + }, + { + name: "falls back for non-finite bounds", + args: [Number.NaN, Number.NaN, 4], + expected: [0, 0.5, 1], + }, + ]; + + for (const { name, args, expected } of cases) { + it(name, () => { + expect(niceTicks(...args)).toEqual(expected); + }); + } + + it("always covers the requested range", () => { + const ticks = niceTicks(0, 37, 4); + expect(ticks[0]).toBeLessThanOrEqual(0); + expect(ticks[ticks.length - 1]).toBeGreaterThanOrEqual(37); + }); +}); + +describe("buildLinePath", () => { + const cases: { name: string; points: LinePathPoint[]; expected: string }[] = [ + { name: "renders nothing for no points", points: [], expected: "" }, + { + name: "renders nothing when every sample is missing", + points: [ + { x: 0, y: null }, + { x: 10, y: null }, + ], + expected: "", + }, + { + name: "renders an isolated sample as a zero-length subpath", + points: [{ x: 0, y: 5 }], + expected: "M 0 5 L 0 5", + }, + { + name: "connects contiguous samples", + points: [ + { x: 0, y: 5 }, + { x: 10, y: 3 }, + { x: 20, y: 4 }, + ], + expected: "M 0 5 L 10 3 L 20 4", + }, + { + name: "breaks the path across a gap", + points: [ + { x: 0, y: 1 }, + { x: 10, y: 2 }, + { x: 20, y: null }, + { x: 30, y: 4 }, + { x: 40, y: 5 }, + ], + expected: "M 0 1 L 10 2 M 30 4 L 40 5", + }, + { + name: "treats a non-finite value as a gap", + points: [ + { x: 0, y: 1 }, + { x: 10, y: Number.NaN }, + { x: 20, y: 3 }, + ], + expected: "M 0 1 L 0 1 M 20 3 L 20 3", + }, + { + name: "rounds coordinates to two decimals", + points: [ + { x: 1.234, y: 2.567 }, + { x: 3.891, y: 4.111 }, + ], + expected: "M 1.23 2.57 L 3.89 4.11", + }, + ]; + + for (const { name, points, expected } of cases) { + it(name, () => { + expect(buildLinePath(points)).toBe(expected); + }); + } +}); + +describe("buildAreaPath", () => { + it("closes a run to the baseline", () => { + expect( + buildAreaPath( + [ + { x: 0, y: 2 }, + { x: 10, y: 4 }, + ], + 100, + ), + ).toBe("M 0 100 L 0 2 L 10 4 L 10 100 Z"); + }); + + it("closes each run separately across a gap", () => { + expect( + buildAreaPath( + [ + { x: 0, y: 2 }, + { x: 10, y: null }, + { x: 20, y: 6 }, + ], + 50, + ), + ).toBe("M 0 50 L 0 2 L 0 50 Z M 20 50 L 20 6 L 20 50 Z"); + }); + + it("renders nothing without samples", () => { + expect(buildAreaPath([{ x: 0, y: null }], 10)).toBe(""); + }); +}); + +describe("stackSegments", () => { + const cases: { + name: string; + values: number[]; + expected: { index: number; value: number; start: number; end: number }[]; + }[] = [ + { name: "returns nothing for no series", values: [], expected: [] }, + { + name: "accumulates offsets from the baseline", + values: [2, 1, 3], + expected: [ + { index: 0, value: 2, start: 0, end: 2 }, + { index: 1, value: 1, start: 2, end: 3 }, + { index: 2, value: 3, start: 3, end: 6 }, + ], + }, + { + name: "keeps zero segments so indexes line up with series", + values: [0, 4], + expected: [ + { index: 0, value: 0, start: 0, end: 0 }, + { index: 1, value: 4, start: 0, end: 4 }, + ], + }, + { + name: "clamps negative and non-finite values to zero", + values: [-5, Number.NaN, 3], + expected: [ + { index: 0, value: 0, start: 0, end: 0 }, + { index: 1, value: 0, start: 0, end: 0 }, + { index: 2, value: 3, start: 0, end: 3 }, + ], + }, + ]; + + for (const { name, values, expected } of cases) { + it(name, () => { + expect(stackSegments(values)).toEqual(expected); + }); + } +}); + +describe("timeBuckets", () => { + const MINUTE = 60_000; + + it("zero-fills every bucket in the window", () => { + const buckets = timeBuckets(0, 3 * MINUTE, MINUTE, [{ t: MINUTE, value: 5 }], 0); + + expect(buckets).toEqual([ + { t: 0, value: 0, present: false }, + { t: MINUTE, value: 5, present: true }, + { t: 2 * MINUTE, value: 0, present: false }, + { t: 3 * MINUTE, value: 0, present: false }, + ]); + }); + + it("assigns samples that land mid-bucket", () => { + const buckets = timeBuckets(0, MINUTE, MINUTE, [{ t: MINUTE + 15_000, value: 7 }], 0); + + expect(buckets[1]).toEqual({ t: MINUTE, value: 7, present: true }); + }); + + it("ignores samples outside the window", () => { + const buckets = timeBuckets( + MINUTE, + 2 * MINUTE, + MINUTE, + [ + { t: 0, value: 1 }, + { t: 9 * MINUTE, value: 2 }, + { t: Number.NaN, value: 3 }, + ], + 0, + ); + + expect(buckets.every((bucket) => !bucket.present)).toBe(true); + }); + + it("lets the later sample win within one bucket", () => { + const buckets = timeBuckets( + 0, + 0, + MINUTE, + [ + { t: 0, value: 1 }, + { t: 30_000, value: 2 }, + ], + 0, + ); + + expect(buckets).toEqual([{ t: 0, value: 2, present: true }]); + }); + + it("supports non-numeric bucket payloads", () => { + const buckets = timeBuckets(0, MINUTE, MINUTE, [{ t: 0, value: [1, 2] }], []); + + expect(buckets).toEqual([ + { t: 0, value: [1, 2], present: true }, + { t: MINUTE, value: [], present: false }, + ]); + }); + + const degenerate: { name: string; args: Parameters> }[] = [ + { name: "an inverted window", args: [MINUTE, 0, MINUTE, [], 0] }, + { name: "a zero step", args: [0, MINUTE, 0, [], 0] }, + { name: "a negative step", args: [0, MINUTE, -MINUTE, [], 0] }, + { name: "a non-finite bound", args: [0, Number.POSITIVE_INFINITY, MINUTE, [], 0] }, + ]; + + for (const { name, args } of degenerate) { + it(`returns nothing for ${name}`, () => { + expect(timeBuckets(...args)).toEqual([]); + }); + } +}); diff --git a/web/src/components/admin/dashboard/charts/chartMath.ts b/web/src/components/admin/dashboard/charts/chartMath.ts new file mode 100644 index 000000000..aa7e863e3 --- /dev/null +++ b/web/src/components/admin/dashboard/charts/chartMath.ts @@ -0,0 +1,245 @@ +/** + * Pure geometry/scale helpers shared by the dashboard chart primitives. + * + * Nothing here touches the DOM or React so the arithmetic stays unit-testable: + * the components own layout and interaction, this file owns the numbers. + */ + +/** Number of categorical series slots the theme defines (`--chart-1` … `--chart-5`). */ +export const CHART_SERIES_SLOTS = 5; + +/** + * Theme color for a categorical series slot. + * + * Colors come only from the theme's chart tokens and are assigned in fixed + * entity order (direct → 1, remux → 2, transcode → 3), never cycled: a chart + * that needs more than five series folds the tail into an "other" bucket + * instead of inventing hues, so out-of-range indexes clamp to the last slot. + */ +export function chartSeriesColor(index: number): string { + const slot = Number.isFinite(index) + ? Math.min(Math.max(Math.trunc(index), 0), CHART_SERIES_SLOTS - 1) + : 0; + return `var(--chart-${slot + 1})`; +} + +function round(value: number, decimals = 2): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +function niceNumber(range: number, roundToNearest: boolean): number { + const exponent = Math.floor(Math.log10(range)); + const fraction = range / 10 ** exponent; + let nice: number; + if (roundToNearest) { + nice = fraction < 1.5 ? 1 : fraction < 3 ? 2 : fraction < 7 ? 5 : 10; + } else { + nice = fraction <= 1 ? 1 : fraction <= 2 ? 2 : fraction <= 5 ? 5 : 10; + } + return nice * 10 ** exponent; +} + +export interface NiceTicksOptions { + /** + * Smallest allowed gap between ticks. Count-based charts pass 1 so a nearly + * flat series gets `0, 1` instead of fractional "half a stream" gridlines. + */ + minStep?: number; +} + +/** + * Axis ticks rounded to human numbers, ascending and always covering + * `[min, max]`. `count` is the desired tick count, not a guarantee. + */ +export function niceTicks( + min: number, + max: number, + count = 4, + options?: NiceTicksOptions, +): number[] { + let lo = Number.isFinite(min) ? min : 0; + let hi = Number.isFinite(max) ? max : 0; + if (lo > hi) { + [lo, hi] = [hi, lo]; + } + if (lo === hi) { + // A flat series still needs a readable axis; grow the top rather than + // collapsing every value onto one gridline. + hi = lo + (lo === 0 ? 1 : Math.abs(lo)); + } + const target = Math.max(2, Math.trunc(count)); + const rawStep = niceNumber(niceNumber(hi - lo, false) / (target - 1), true); + const minStep = options?.minStep; + const step = + minStep && Number.isFinite(minStep) && minStep > 0 ? Math.max(rawStep, minStep) : rawStep; + const decimals = Math.max(0, Math.min(10, -Math.floor(Math.log10(step)))); + const start = Math.floor(lo / step) * step; + const end = Math.ceil(hi / step) * step; + const ticks: number[] = []; + // Guard against pathological steps producing an unbounded loop. + const maxTicks = 1000; + for (let i = 0; i <= maxTicks; i += 1) { + const value = round(start + i * step, decimals); + ticks.push(value); + if (value >= end) { + break; + } + } + return ticks; +} + +export interface LinePathPoint { + x: number; + /** `null` marks a missing sample: the path breaks instead of interpolating. */ + y: number | null; +} + +function isPlotted(point: LinePathPoint): point is { x: number; y: number } { + return point.y !== null && Number.isFinite(point.y) && Number.isFinite(point.x); +} + +function contiguousRuns(points: readonly LinePathPoint[]): { x: number; y: number }[][] { + const runs: { x: number; y: number }[][] = []; + let current: { x: number; y: number }[] = []; + for (const point of points) { + if (isPlotted(point)) { + current.push({ x: point.x, y: point.y }); + continue; + } + if (current.length > 0) { + runs.push(current); + current = []; + } + } + if (current.length > 0) { + runs.push(current); + } + return runs; +} + +/** + * `d` for a polyline through already-projected pixel coordinates. Gaps (`null` + * y) break the path into separate subpaths; an isolated sample becomes a + * zero-length subpath so a round line cap still renders it as a dot. + */ +export function buildLinePath(points: readonly LinePathPoint[]): string { + return contiguousRuns(points) + .map((run) => { + const head = run[0]; + if (!head) { + return ""; + } + const segments = run + .slice(1) + .map((point) => `L ${round(point.x)} ${round(point.y)}`) + .join(" "); + const start = `M ${round(head.x)} ${round(head.y)}`; + return segments ? `${start} ${segments}` : `${start} L ${round(head.x)} ${round(head.y)}`; + }) + .filter((segment) => segment !== "") + .join(" "); +} + +/** + * `d` for the area wash under the same points, closed to `baselineY`. Each + * contiguous run is its own closed subpath so gaps stay empty. + */ +export function buildAreaPath(points: readonly LinePathPoint[], baselineY: number): string { + return contiguousRuns(points) + .map((run) => { + const head = run[0]; + const tail = run[run.length - 1]; + if (!head || !tail) { + return ""; + } + const body = run.map((point) => `L ${round(point.x)} ${round(point.y)}`).join(" "); + return `M ${round(head.x)} ${round(baselineY)} ${body} L ${round(tail.x)} ${round(baselineY)} Z`; + }) + .filter((segment) => segment !== "") + .join(" "); +} + +export interface StackSegment { + /** Series index the segment belongs to. */ + index: number; + value: number; + /** Cumulative offset where the segment starts (0 = baseline). */ + start: number; + /** Cumulative offset where the segment ends. */ + end: number; +} + +/** + * Cumulative offsets for one stacked column. Missing/negative values are + * treated as 0 so a bad sample can never push a segment below the baseline. + * Zero-valued segments are kept (with `start === end`) so callers can rely on + * segment index matching series index. + */ +export function stackSegments(values: readonly number[]): StackSegment[] { + let offset = 0; + return values.map((raw, index) => { + const value = Number.isFinite(raw) && raw > 0 ? raw : 0; + const start = offset; + offset += value; + return { index, value, start, end: offset }; + }); +} + +export interface TimeSample { + /** Bucket timestamp in epoch milliseconds. */ + t: number; + value: T; +} + +export interface TimeBucket { + /** Bucket start in epoch milliseconds. */ + t: number; + value: T; + /** False when the bucket was zero-filled because no sample covered it. */ + present: boolean; +} + +const MAX_TIME_BUCKETS = 10_000; + +/** + * Zero-filled bucket series from `from` to `to` inclusive, stepping by + * `stepMs`. Sparse API responses (which omit empty buckets entirely) become a + * dense series, and `present` marks which buckets were real samples so line + * charts can render gaps instead of fake zeros. + * + * `from` is used verbatim as the first bucket start — callers pass an already + * truncated timestamp. Samples outside the window are ignored; when two + * samples land in the same bucket the later one in iteration order wins. + */ +export function timeBuckets( + from: number, + to: number, + stepMs: number, + samples: readonly TimeSample[], + empty: T, +): TimeBucket[] { + if (!Number.isFinite(from) || !Number.isFinite(to) || !Number.isFinite(stepMs) || stepMs <= 0) { + return []; + } + if (to < from) { + return []; + } + const count = Math.min(Math.floor((to - from) / stepMs) + 1, MAX_TIME_BUCKETS); + const buckets: TimeBucket[] = []; + for (let i = 0; i < count; i += 1) { + buckets.push({ t: from + i * stepMs, value: empty, present: false }); + } + for (const sample of samples) { + if (!Number.isFinite(sample.t) || sample.t < from) { + continue; + } + const index = Math.floor((sample.t - from) / stepMs); + const bucket = buckets[index]; + if (!bucket) { + continue; + } + buckets[index] = { t: bucket.t, value: sample.value, present: true }; + } + return buckets; +} diff --git a/web/src/components/admin/dashboard/charts/index.ts b/web/src/components/admin/dashboard/charts/index.ts new file mode 100644 index 000000000..60daac97e --- /dev/null +++ b/web/src/components/admin/dashboard/charts/index.ts @@ -0,0 +1,43 @@ +/** + * Chart primitives for the admin dashboard widgets. + * + * Hand-rolled on purpose — the web app carries no chart library, and these + * widgets need only a handful of quiet marks. House rules for everything in + * this folder: + * + * - Series colors come only from the theme's `--chart-1` … `--chart-5` tokens, + * assigned in fixed entity order (direct → 1, remux → 2, transcode → 3) and + * never cycled or repainted when a filter changes the series count. + * - One value axis per chart; text (labels, values, legends, ticks) wears text + * tokens, never the series color. + * - A legend is present for two or more series; a single-series chart is named + * by its card header instead. + * - Marks are thin, separated by 2px gaps in the card surface rather than by + * outlines, and every plotted chart ships a hover/focus readout. + */ + +export { BarList, type BarListItem, type BarListProps } from "./BarList"; +export { BarListSkeleton, ChartEmptyState, ChartSkeleton } from "./ChartEmptyState"; +export { ChartLegend, ChartTooltip, ChartTooltipRow, type ChartLegendEntry } from "./chartChrome"; +export { + CHART_SERIES_SLOTS, + buildAreaPath, + buildLinePath, + chartSeriesColor, + niceTicks, + stackSegments, + timeBuckets, + type LinePathPoint, + type NiceTicksOptions, + type StackSegment, + type TimeBucket, + type TimeSample, +} from "./chartMath"; +export { LineChart, type LineChartPoint, type LineChartProps } from "./LineChart"; +export { Sparkline, type SparklineProps } from "./Sparkline"; +export { useMeasuredSize, type MeasuredSize } from "./useMeasuredSize"; +export { + StackedColumnChart, + type StackedColumnBucket, + type StackedColumnChartProps, +} from "./StackedColumnChart"; diff --git a/web/src/components/admin/dashboard/charts/useMeasuredSize.ts b/web/src/components/admin/dashboard/charts/useMeasuredSize.ts new file mode 100644 index 000000000..4d578c45e --- /dev/null +++ b/web/src/components/admin/dashboard/charts/useMeasuredSize.ts @@ -0,0 +1,48 @@ +import { useEffect, useRef, useState } from "react"; + +export interface MeasuredSize { + width: number; + height: number; +} + +/** Sub-pixel churn is noise, not a resize: re-render only past half a pixel. */ +const SIGNIFICANT_CHANGE_PX = 0.5; + +/** + * Content-box size of an element, tracked with a ResizeObserver. + * + * A chart in a widget the admin can resize has no height it can hard-code, and + * an SVG stretched to an unknown box distorts its marks. Measuring the box in + * CSS pixels lets a chart draw in the same units the browser paints in. + * + * `null` until the first observation (and wherever ResizeObserver is missing), + * so callers keep a static fallback rather than drawing into a zero-sized box. + */ +export function useMeasuredSize() { + const ref = useRef(null); + const [size, setSize] = useState(null); + + useEffect(() => { + const node = ref.current; + if (!node || typeof ResizeObserver === "undefined") { + return; + } + const observer = new ResizeObserver((entries) => { + const rect = entries[0]?.contentRect; + if (!rect) { + return; + } + setSize((previous) => + previous !== null && + Math.abs(previous.width - rect.width) < SIGNIFICANT_CHANGE_PX && + Math.abs(previous.height - rect.height) < SIGNIFICANT_CHANGE_PX + ? previous + : { width: rect.width, height: rect.height }, + ); + }); + observer.observe(node); + return () => observer.disconnect(); + }, []); + + return { ref, size }; +} diff --git a/web/src/components/admin/dashboard/format.ts b/web/src/components/admin/dashboard/format.ts index 45d1358a9..d42318c37 100644 --- a/web/src/components/admin/dashboard/format.ts +++ b/web/src/components/admin/dashboard/format.ts @@ -8,6 +8,43 @@ export function formatFileCount(count: number | null | undefined) { return count === 1 ? "1 file" : `${count.toLocaleString()} files`; } +/** + * Compact watch time for the top-activity lists. Sub-hour totals stay in + * minutes so a short session does not collapse to "0.0h". + */ +export function formatWatchTime(totalSeconds: number | null | undefined) { + if (totalSeconds == null || !Number.isFinite(totalSeconds) || totalSeconds <= 0) { + return "0m"; + } + if (totalSeconds < 3600) { + return `${Math.max(1, Math.round(totalSeconds / 60))}m`; + } + const hours = totalSeconds / 3600; + return `${hours.toLocaleString(undefined, { maximumFractionDigits: 1 })}h`; +} + +/** + * Egress rate for charts and tiles. Small rates keep one decimal so a trickle + * of traffic does not round to a flat "0 Mbps" and read as idle. + */ +export function formatMbpsValue(mbps: number) { + if (!Number.isFinite(mbps)) { + return "—"; + } + const decimals = mbps > 0 && mbps < 10 ? 1 : 0; + return mbps.toLocaleString(undefined, { + minimumFractionDigits: decimals, + maximumFractionDigits: decimals, + }); +} + +export function formatMbps(mbps: number) { + if (!Number.isFinite(mbps)) { + return "—"; + } + return `${formatMbpsValue(mbps)} Mbps`; +} + export function formatDashboardLibraryScanProgress(scan: ScanRun, activeScanCount: number) { const status = scan.status === "running" ? "Scanning" : "Queued"; const progress = formatActiveScanProgress(scan); @@ -16,3 +53,72 @@ export function formatDashboardLibraryScanProgress(scan: ScanRun, activeScanCoun const extraScans = activeScanCount > 1 ? ` + ${activeScanCount - 1} more` : ""; return `${status}: ${detail}${extraScans}`; } + +/** + * Elapsed time for a scan row. A finished scan reports the span it took; a + * running one reports how long it has been going so far, which is the number + * an operator watching a slow scan actually wants. The scans endpoint reports + * the two timestamps and no duration, so it is derived here. + */ +export function formatScanDuration( + scan: { started_at?: string; completed_at?: string }, + now: number = Date.now(), +): string { + if (!scan.started_at) { + return "—"; + } + const started = Date.parse(scan.started_at); + if (!Number.isFinite(started)) { + return "—"; + } + const ended = scan.completed_at ? Date.parse(scan.completed_at) : now; + if (!Number.isFinite(ended) || ended < started) { + return "—"; + } + return formatDurationSeconds(Math.round((ended - started) / 1000)); +} + +function formatDurationSeconds(totalSeconds: number): string { + if (totalSeconds < 60) { + return `${totalSeconds}s`; + } + if (totalSeconds < 3600) { + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return seconds === 0 ? `${minutes}m` : `${minutes}m ${seconds}s`; + } + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; +} + +/** + * Uptime at the coarsest unit that still says something useful: an operator + * scanning a health strip wants "3d 4h", not a seconds counter. + */ +export function formatUptime(startedAt: string | null | undefined, now: number = Date.now()) { + if (!startedAt) { + return "—"; + } + const started = Date.parse(startedAt); + if (!Number.isFinite(started)) { + return "—"; + } + const seconds = Math.max(0, Math.floor((now - started) / 1000)); + if (seconds < 86_400) { + return formatDurationSeconds(seconds); + } + const days = Math.floor(seconds / 86_400); + const hours = Math.floor((seconds % 86_400) / 3600); + return hours === 0 ? `${days}d` : `${days}d ${hours}h`; +} + +/** Sub-10ms round trips keep their decimals; anything slower does not need them. */ +export function formatLatency(latencyMs: number) { + if (!Number.isFinite(latencyMs)) { + return "—"; + } + return `${latencyMs.toLocaleString(undefined, { + maximumFractionDigits: latencyMs < 10 ? 2 : 0, + })} ms`; +} diff --git a/web/src/components/admin/dashboard/range.ts b/web/src/components/admin/dashboard/range.ts new file mode 100644 index 000000000..a9785eb2b --- /dev/null +++ b/web/src/components/admin/dashboard/range.ts @@ -0,0 +1,128 @@ +import { formatDate, formatTime, preferredDateLocale } from "@/lib/datetime"; +import type { WidgetRange } from "./types"; + +/** + * How a `WidgetRange` turns into request parameters and into words. + * + * Every widget that offers a range reads its window from here, so the picker, + * the card title, the chart's accessible name, and the empty state can never + * disagree about what "week" means. + */ + +/** Widest-to-narrowest order the picker renders in. */ +export const WIDGET_RANGE_ORDER: readonly WidgetRange[] = ["hour", "day", "week", "month"]; + +/** + * Hours for the timeseries and playback-activity endpoints. A month is 744 + * hours (31 days), matching the sampler's retention window. + */ +const RANGE_HOURS: Record = { + hour: 1, + day: 24, + week: 168, + month: 744, +}; + +/** + * Days for the top-activity endpoint. "hour" collapses onto a single day: the + * leaderboards do not offer it, and a sub-day window would return almost + * nothing worth ranking. + */ +const RANGE_DAYS: Record = { + hour: 1, + day: 1, + week: 7, + month: 30, +}; + +/** Compact label for the segmented control. */ +const RANGE_LABELS: Record = { + hour: "1h", + day: "24h", + week: "7d", + month: "30d", +}; + +/** Spoken form, for aria-labels and prose ("No plays in the last 7 days"). */ +const RANGE_PHRASES: Record = { + hour: "the last hour", + day: "the last 24 hours", + week: "the last 7 days", + month: "the last 30 days", +}; + +/** Card-title suffix, e.g. "Egress · last 30 d". */ +const RANGE_TITLE_SUFFIXES: Record = { + hour: "last 1 h", + day: "last 24 h", + week: "last 7 d", + month: "last 30 d", +}; + +/** Leading edge label on a time axis; the trailing one is always "now". */ +const RANGE_EDGE_LABELS: Record = { + hour: "1h ago", + day: "24h ago", + week: "7d ago", + month: "30d ago", +}; + +export function rangeHours(range: WidgetRange): number { + return RANGE_HOURS[range]; +} + +export function rangeDays(range: WidgetRange): number { + return RANGE_DAYS[range]; +} + +export function rangeLabel(range: WidgetRange): string { + return RANGE_LABELS[range]; +} + +export function rangePhrase(range: WidgetRange): string { + return RANGE_PHRASES[range]; +} + +/** "Playback activity · last 24 h" — one format for every ranged widget. */ +export function rangeTitle(title: string, range: WidgetRange): string { + return `${title} · ${RANGE_TITLE_SUFFIXES[range]}`; +} + +/** Axis edges: how far back the window starts, and where it ends. */ +export function rangeEdgeLabels(range: WidgetRange): { start: string; end: string } { + return { start: RANGE_EDGE_LABELS[range], end: "now" }; +} + +/** + * Timestamp for a hover readout or a column tick. + * + * Short windows are read as clock times; from a week up the clock stops being + * the distinguishing part, so the day is shown instead (with the time as well + * in a tooltip, where there is room for it). + */ +export function formatRangeTimestamp( + range: WidgetRange, + t: number, + options?: { withTime?: boolean }, +): string { + if (range === "hour" || range === "day") { + return formatTime(t); + } + const day = formatDayLabel(t); + return options?.withTime ? `${day}, ${formatTime(t)}` : day; +} + +/** "Aug 26" — short enough for an axis tick, unambiguous across a month. */ +export function formatDayLabel(t: number): string { + const date = new Date(t); + if (Number.isNaN(date.getTime())) { + return ""; + } + const locale = preferredDateLocale(); + try { + return date.toLocaleDateString(locale, { month: "short", day: "numeric" }); + } catch { + // An unsupported locale tag must not blank out the axis. + return formatDate(date, "medium"); + } +} diff --git a/web/src/components/admin/dashboard/registry.tsx b/web/src/components/admin/dashboard/registry.tsx index 5d9f918c6..abef0643b 100644 --- a/web/src/components/admin/dashboard/registry.tsx +++ b/web/src/components/admin/dashboard/registry.tsx @@ -1,17 +1,54 @@ -import type { DashboardLayoutEntry, DashboardWidgetDefinition, WidgetId } from "./types"; +import type { + DashboardLayoutEntry, + DashboardWidgetDefinition, + WidgetId, + WidgetRangeOptions, +} from "./types"; import { ActiveStreamsStatWidget, + EgressNowStatWidget, MoviesStatWidget, + ProfilesActiveStatWidget, ShowsStatWidget, StorageStatWidget, + TranscodeShareStatWidget, UsersStatWidget, } from "./widgets/statTiles"; +import { ConcurrentStreamsWidget } from "./widgets/ConcurrentStreamsWidget"; +import { EgressWidget } from "./widgets/EgressWidget"; +import { HealthStripWidget } from "./widgets/HealthStripWidget"; +import { PlaybackActivityWidget } from "./widgets/PlaybackActivityWidget"; +import { PlaybackReliabilityWidget } from "./widgets/PlaybackReliabilityWidget"; +import { TopProfilesWidget } from "./widgets/TopProfilesWidget"; +import { TopTitlesWidget } from "./widgets/TopTitlesWidget"; import { TraktSyncWidget } from "./widgets/TraktSyncWidget"; import { NowPlayingWidget } from "./widgets/NowPlayingWidget"; +import { TranscodeNodesWidget } from "./widgets/TranscodeNodesWidget"; +import { ScannerWidget } from "./widgets/ScannerWidget"; import { LibrariesWidget } from "./widgets/LibrariesWidget"; import { UsersWidget } from "./widgets/UsersWidget"; +import { ScanActivityWidget } from "./widgets/ScanActivityWidget"; +import { RecentErrorsWidget } from "./widgets/RecentErrorsWidget"; import { RecentActivityWidget } from "./widgets/RecentActivityWidget"; +/** + * Sampled charts read `dashboard_metric_samples`, which the sampler keeps for a + * month, so they offer the full spread down to a single hour. + */ +const SAMPLED_RANGES: WidgetRangeOptions = { + allowed: ["hour", "day", "week", "month"], + default: "day", +}; + +/** + * Leaderboards start at a day: an hour of watch history ranks too little to be + * worth a chart, and the endpoint's window is measured in days anyway. + */ +const LEADERBOARD_RANGES: WidgetRangeOptions = { + allowed: ["day", "week", "month"], + default: "week", +}; + export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ { id: "stat-active-streams", @@ -20,8 +57,47 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 2, maxSpan: 4, defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, Component: ActiveStreamsStatWidget, }, + { + id: "stat-egress-now", + title: "Egress now", + description: "Egress the deployment is serving this minute", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, + Component: EgressNowStatWidget, + }, + { + id: "stat-transcode-share", + title: "Transcode share", + description: "Share of live streams being transcoded", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, + Component: TranscodeShareStatWidget, + }, + { + id: "stat-profiles-active", + title: "Profiles · 24h", + description: "Profiles that watched something in the last 24 hours", + minSpan: 2, + maxSpan: 4, + defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, + Component: ProfilesActiveStatWidget, + }, { id: "stat-movies", title: "Movies", @@ -29,6 +105,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 2, maxSpan: 4, defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, Component: MoviesStatWidget, }, { @@ -38,6 +117,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 2, maxSpan: 4, defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, Component: ShowsStatWidget, }, { @@ -47,6 +129,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 2, maxSpan: 4, defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, Component: UsersStatWidget, }, { @@ -56,8 +141,101 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 2, maxSpan: 4, defaultSpan: 3, + minRows: 1, + maxRows: 2, + defaultRows: 1, Component: StorageStatWidget, }, + { + id: "health-strip", + title: "Server health", + description: "Version, uptime, dependencies, nodes, and 24h error count", + minSpan: 6, + maxSpan: 12, + defaultSpan: 12, + minRows: 1, + maxRows: 2, + defaultRows: 1, + Component: HealthStripWidget, + }, + { + id: "playback-24h", + title: "Playback activity", + description: "Playback starts stacked by play method", + minSpan: 6, + maxSpan: 12, + defaultSpan: 6, + minRows: 2, + maxRows: 5, + defaultRows: 3, + ranges: SAMPLED_RANGES, + Component: PlaybackActivityWidget, + }, + { + id: "concurrent-streams-24h", + title: "Concurrent streams", + description: "Sampled concurrent playback sessions", + minSpan: 4, + maxSpan: 12, + defaultSpan: 6, + minRows: 2, + maxRows: 5, + defaultRows: 3, + ranges: SAMPLED_RANGES, + Component: ConcurrentStreamsWidget, + }, + { + id: "egress-24h", + title: "Egress", + description: "Sampled egress in Mbps", + minSpan: 4, + maxSpan: 12, + defaultSpan: 6, + minRows: 2, + maxRows: 5, + defaultRows: 3, + ranges: SAMPLED_RANGES, + Component: EgressWidget, + }, + { + id: "playback-reliability", + title: "Playback reliability", + description: "Sessions started, transcode starts, completion rate, profiles", + minSpan: 4, + maxSpan: 8, + defaultSpan: 6, + minRows: 2, + maxRows: 4, + defaultRows: 2, + ranges: SAMPLED_RANGES, + Component: PlaybackReliabilityWidget, + }, + { + id: "top-titles", + title: "Top titles", + description: "Most-played titles over the chosen window", + minSpan: 4, + maxSpan: 8, + defaultSpan: 6, + minRows: 2, + maxRows: 5, + defaultRows: 3, + ranges: LEADERBOARD_RANGES, + Component: TopTitlesWidget, + }, + { + id: "top-profiles", + title: "Most active profiles", + description: "Profiles with the most plays over the chosen window", + minSpan: 4, + maxSpan: 8, + defaultSpan: 6, + minRows: 2, + maxRows: 5, + defaultRows: 3, + ranges: LEADERBOARD_RANGES, + Component: TopProfilesWidget, + }, { id: "trakt-sync", title: "Trakt sync", @@ -65,6 +243,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 6, maxSpan: 12, defaultSpan: 9, + minRows: 1, + maxRows: 1, + defaultRows: 1, Component: TraktSyncWidget, }, { @@ -74,8 +255,35 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 6, maxSpan: 12, defaultSpan: 12, + minRows: 2, + maxRows: 8, + defaultRows: 4, Component: NowPlayingWidget, }, + { + id: "transcode-nodes", + title: "Transcode nodes", + description: "Stream node health, job load, and egress", + minSpan: 4, + maxSpan: 12, + defaultSpan: 6, + minRows: 2, + maxRows: 6, + defaultRows: 3, + Component: TranscodeNodesWidget, + }, + { + id: "scanner", + title: "Scanner", + description: "Live scan progress, queue depth, and autoscan state", + minSpan: 4, + maxSpan: 12, + defaultSpan: 6, + minRows: 2, + maxRows: 6, + defaultRows: 3, + Component: ScannerWidget, + }, { id: "libraries", title: "Libraries", @@ -83,6 +291,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 6, maxSpan: 12, defaultSpan: 7, + minRows: 2, + maxRows: 8, + defaultRows: 4, Component: LibrariesWidget, }, { @@ -92,8 +303,35 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 4, maxSpan: 8, defaultSpan: 5, + minRows: 2, + maxRows: 8, + defaultRows: 4, Component: UsersWidget, }, + { + id: "scan-activity", + title: "Scan activity", + description: "Recent scan runs with trigger, status, and duration", + minSpan: 6, + maxSpan: 12, + defaultSpan: 12, + minRows: 2, + maxRows: 6, + defaultRows: 3, + Component: ScanActivityWidget, + }, + { + id: "recent-errors", + title: "Recent errors", + description: "Latest error and warning lines from the operational log", + minSpan: 6, + maxSpan: 12, + defaultSpan: 12, + minRows: 2, + maxRows: 6, + defaultRows: 4, + Component: RecentErrorsWidget, + }, { id: "recent-activity", title: "Recent activity", @@ -101,6 +339,9 @@ export const DASHBOARD_WIDGETS: DashboardWidgetDefinition[] = [ minSpan: 6, maxSpan: 12, defaultSpan: 12, + minRows: 2, + maxRows: 8, + defaultRows: 4, Component: RecentActivityWidget, }, ]; @@ -119,15 +360,45 @@ export function findDashboardWidget(id: string): DashboardWidgetDefinition | und return WIDGETS_BY_ID.get(id as WidgetId); } -export const DEFAULT_LAYOUT: DashboardLayoutEntry[] = [ - { id: "stat-active-streams", span: 3 }, - { id: "stat-movies", span: 3 }, - { id: "stat-shows", span: 3 }, - { id: "stat-users", span: 3 }, - { id: "stat-storage", span: 3 }, - { id: "trakt-sync", span: 9 }, - { id: "now-playing", span: 12 }, - { id: "libraries", span: 7 }, - { id: "users", span: 5 }, - { id: "recent-activity", span: 12 }, +/** + * The boxes of the default arrangement: live numbers first, then the charts + * that explain them, then the operational surfaces that are only interesting + * when something is wrong. Admins who already customized their dashboard keep + * their stored layout — this list is only the starting point, and every widget + * stays available from the Add-widget sheet. + * + * Windows are not written here; DEFAULT_LAYOUT below stamps each ranged widget + * with the default from its own definition, so the two cannot drift apart. + */ +const DEFAULT_LAYOUT_BOXES: DashboardLayoutEntry[] = [ + { id: "stat-active-streams", span: 3, rows: 1 }, + { id: "stat-egress-now", span: 3, rows: 1 }, + { id: "stat-transcode-share", span: 3, rows: 1 }, + { id: "stat-profiles-active", span: 3, rows: 1 }, + { id: "stat-movies", span: 3, rows: 1 }, + { id: "stat-shows", span: 3, rows: 1 }, + { id: "stat-users", span: 3, rows: 1 }, + { id: "stat-storage", span: 3, rows: 1 }, + { id: "health-strip", span: 12, rows: 1 }, + { id: "playback-24h", span: 6, rows: 3 }, + { id: "concurrent-streams-24h", span: 6, rows: 3 }, + { id: "egress-24h", span: 6, rows: 3 }, + { id: "playback-reliability", span: 6, rows: 2 }, + { id: "now-playing", span: 12, rows: 4 }, + { id: "transcode-nodes", span: 6, rows: 3 }, + { id: "scanner", span: 6, rows: 3 }, + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, + { id: "top-titles", span: 6, rows: 3 }, + { id: "top-profiles", span: 6, rows: 3 }, + { id: "trakt-sync", span: 9, rows: 1 }, + { id: "scan-activity", span: 12, rows: 3 }, + { id: "recent-errors", span: 12, rows: 4 }, + { id: "recent-activity", span: 12, rows: 4 }, ]; + +/** The default arrangement, with each ranged widget on its default window. */ +export const DEFAULT_LAYOUT: DashboardLayoutEntry[] = DEFAULT_LAYOUT_BOXES.map((entry) => { + const ranges = getDashboardWidget(entry.id).ranges; + return ranges ? { ...entry, range: ranges.default } : entry; +}); diff --git a/web/src/components/admin/dashboard/types.ts b/web/src/components/admin/dashboard/types.ts index a7c5d11f4..9760b85fb 100644 --- a/web/src/components/admin/dashboard/types.ts +++ b/web/src/components/admin/dashboard/types.ts @@ -2,16 +2,56 @@ import type React from "react"; export type WidgetId = | "stat-active-streams" + | "stat-egress-now" + | "stat-transcode-share" + | "stat-profiles-active" | "stat-movies" | "stat-shows" | "stat-users" | "stat-storage" + | "health-strip" + | "playback-24h" + | "concurrent-streams-24h" + | "egress-24h" + | "playback-reliability" + | "top-titles" + | "top-profiles" | "trakt-sync" | "now-playing" + | "transcode-nodes" + | "scanner" | "libraries" | "users" + | "scan-activity" + | "recent-errors" | "recent-activity"; +/** + * The window a metric widget covers. + * + * Named by period rather than by number so one stored value can mean 1 hour on + * a sampled chart and 1 day on a leaderboard — the endpoints take different + * units, and the admin is choosing "how far back", not "how many hours". + */ +export type WidgetRange = "hour" | "day" | "week" | "month"; + +/** The ranges a widget offers, and the one it starts on. */ +export interface WidgetRangeOptions { + allowed: readonly WidgetRange[]; + default: WidgetRange; +} + +/** + * A widget's identity and the box it is allowed to occupy. + * + * Columns and rows are independent axes: `minSpan === maxSpan` pins the width, + * `minRows === maxRows` pins the height, and a widget that pins both cannot be + * resized at all. Rows are grid rows (`--admin-row-h` in app.css), not pixels, + * and only apply from the lg breakpoint up. + * + * `ranges` is absent for widgets whose window is not the admin's to choose — + * live tiles, lists that are not time-bounded, and the fixed-24h stats. + */ export interface DashboardWidgetDefinition { id: WidgetId; title: string; @@ -19,10 +59,17 @@ export interface DashboardWidgetDefinition { minSpan: number; maxSpan: number; defaultSpan: number; + minRows: number; + maxRows: number; + defaultRows: number; + ranges?: WidgetRangeOptions; Component: React.ComponentType; } export interface DashboardLayoutEntry { id: WidgetId; span: number; + rows: number; + /** Absent for widgets without `ranges`; otherwise one of that widget's allowed values. */ + range?: WidgetRange; } diff --git a/web/src/components/admin/dashboard/useDashboardLayout.test.ts b/web/src/components/admin/dashboard/useDashboardLayout.test.ts index 3de522eb1..ee4add63d 100644 --- a/web/src/components/admin/dashboard/useDashboardLayout.test.ts +++ b/web/src/components/admin/dashboard/useDashboardLayout.test.ts @@ -1,12 +1,40 @@ // @vitest-environment jsdom import { act, renderHook } from "@testing-library/react"; -import { beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AdminDashboardLayoutResponse } from "@/api/types"; import { DASHBOARD_WIDGETS, DEFAULT_LAYOUT } from "./registry"; -import { DASHBOARD_LAYOUT_STORAGE_KEY, useDashboardLayout } from "./useDashboardLayout"; +import { + DASHBOARD_LAYOUT_SAVE_DEBOUNCE_MS, + DASHBOARD_LAYOUT_STORAGE_KEY, + useDashboardLayout, +} from "./useDashboardLayout"; import type { DashboardLayoutEntry } from "./types"; +const mocks = vi.hoisted(() => ({ + query: { data: undefined as AdminDashboardLayoutResponse | undefined, isSuccess: false }, + save: vi.fn(), + reset: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/dashboardLayout", () => ({ + useAdminDashboardLayout: () => mocks.query, + useSaveAdminDashboardLayout: () => ({ mutate: mocks.save }), + useResetAdminDashboardLayout: () => ({ mutate: mocks.reset }), +})); + +function serverLayout(entries: unknown, updatedAt = "2026-08-26T10:00:00Z") { + mocks.query = { + data: { layout: { version: 1, entries }, updated_at: updatedAt }, + isSuccess: true, + }; +} + +function serverNoLayout() { + mocks.query = { data: { layout: null, updated_at: null }, isSuccess: true }; +} + function readStored(): { version: number; entries: DashboardLayoutEntry[] } { const raw = window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY); if (raw === null) { @@ -22,16 +50,29 @@ function writeStored(entries: unknown) { ); } +// A widget joins the registry before it joins DEFAULT_LAYOUT — new widgets +// ship hidden and are discovered through the Add-widget sheet — so expected +// "hidden" sets are derived from the registry instead of hardcoded. +function hiddenWidgetIds(...alsoRemoved: string[]): string[] { + const visible = new Set( + DEFAULT_LAYOUT.map((entry) => entry.id).filter((id) => !alsoRemoved.includes(id)), + ); + return DASHBOARD_WIDGETS.filter((widget) => !visible.has(widget.id)).map((widget) => widget.id); +} + describe("useDashboardLayout", () => { beforeEach(() => { window.localStorage.clear(); + mocks.query = { data: undefined, isSuccess: false }; + mocks.save.mockReset(); + mocks.reset.mockReset(); }); it("uses the default layout when storage is empty", () => { const { result } = renderHook(() => useDashboardLayout()); expect(result.current.entries).toEqual(DEFAULT_LAYOUT); - expect(result.current.hiddenWidgets).toEqual([]); + expect(result.current.hiddenWidgets.map((w) => w.id)).toEqual(hiddenWidgetIds()); expect(result.current.isCustomizing).toBe(false); }); @@ -64,31 +105,207 @@ describe("useDashboardLayout", () => { const { result } = renderHook(() => useDashboardLayout()); expect(result.current.entries).toEqual([ - { id: "libraries", span: 7 }, - { id: "users", span: 5 }, + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, ]); }); it("clamps spans to the widget's [minSpan, maxSpan] on load", () => { writeStored([ - { id: "stat-movies", span: 1 }, // min 2 - { id: "now-playing", span: 40 }, // max 12 - { id: "users", span: "wide" }, // non-numeric -> defaultSpan + { id: "stat-movies", span: 1, rows: 1 }, // min 2 + { id: "now-playing", span: 40, rows: 4 }, // max 12 + { id: "users", span: "wide", rows: 4 }, // non-numeric -> defaultSpan ]); const { result } = renderHook(() => useDashboardLayout()); expect(result.current.entries).toEqual([ - { id: "stat-movies", span: 2 }, - { id: "now-playing", span: 12 }, - { id: "users", span: 5 }, + { id: "stat-movies", span: 2, rows: 1 }, + { id: "now-playing", span: 12, rows: 4 }, + { id: "users", span: 5, rows: 4 }, + ]); + }); + + it("clamps rows to the widget's [minRows, maxRows] on load", () => { + writeStored([ + { id: "stat-movies", span: 3, rows: 0 }, // min 1 + { id: "now-playing", span: 12, rows: 40 }, // max 8 + { id: "users", span: 5, rows: "tall" }, // non-numeric -> defaultRows + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "stat-movies", span: 3, rows: 1 }, + { id: "now-playing", span: 12, rows: 8 }, + { id: "users", span: 5, rows: 4 }, + ]); + }); + + // Every layout saved before two-axis resizing shipped is missing `rows`; the + // widget's default height is what those admins were already looking at. + it("gives entries without rows the widget's default height", () => { + writeStored([ + { id: "libraries", span: 7 }, + { id: "trakt-sync", span: 9 }, + { id: "stat-movies", span: 3 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "libraries", span: 7, rows: 4 }, + { id: "trakt-sync", span: 9, rows: 1 }, + { id: "stat-movies", span: 3, rows: 1 }, + ]); + }); + + // Windows arrived after the first layouts were saved, so an entry without + // one is the common case rather than a corrupt one. + it("fills in the widget's default window and leaves unranged widgets alone", () => { + writeStored([ + { id: "egress-24h", span: 6, rows: 3 }, + { id: "top-titles", span: 6, rows: 3 }, + { id: "users", span: 5, rows: 4 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "egress-24h", span: 6, rows: 3, range: "day" }, + { id: "top-titles", span: 6, rows: 3, range: "week" }, + { id: "users", span: 5, rows: 4 }, + ]); + }); + + it("keeps a stored window the widget allows", () => { + writeStored([ + { id: "egress-24h", span: 6, rows: 3, range: "month" }, + { id: "top-titles", span: 6, rows: 3, range: "day" }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "egress-24h", span: 6, rows: 3, range: "month" }, + { id: "top-titles", span: 6, rows: 3, range: "day" }, + ]); + }); + + // The leaderboards do not offer an hour, and an unranged widget must not + // start carrying a window because one was stored for it. + it("replaces a window the widget does not offer and drops one it cannot use", () => { + writeStored([ + { id: "top-profiles", span: 6, rows: 3, range: "hour" }, + { id: "egress-24h", span: 6, rows: 3, range: "fortnight" }, + { id: "concurrent-streams-24h", span: 6, rows: 3, range: 7 }, + { id: "users", span: 5, rows: 4, range: "month" }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([ + { id: "top-profiles", span: 6, rows: 3, range: "week" }, + { id: "egress-24h", span: 6, rows: 3, range: "day" }, + { id: "concurrent-streams-24h", span: 6, rows: 3, range: "day" }, + { id: "users", span: 5, rows: 4 }, + ]); + expect(result.current.entries[3]).not.toHaveProperty("range"); + }); + + it("setWidgetRange changes the window and persists", () => { + writeStored([{ id: "egress-24h", span: 6, rows: 3, range: "day" }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.setWidgetRange("egress-24h", "week"); + }); + + expect(result.current.entries).toEqual([{ id: "egress-24h", span: 6, rows: 3, range: "week" }]); + expect(readStored().entries).toEqual([{ id: "egress-24h", span: 6, rows: 3, range: "week" }]); + }); + + // Picking a window is an everyday viewing action, not an arrangement one. + it("setWidgetRange works outside customize mode", () => { + writeStored([{ id: "top-titles", span: 6, rows: 3, range: "week" }]); + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.isCustomizing).toBe(false); + + act(() => { + result.current.setWidgetRange("top-titles", "month"); + }); + + expect(result.current.entries).toEqual([ + { id: "top-titles", span: 6, rows: 3, range: "month" }, ]); + expect(readStored().entries).toEqual([{ id: "top-titles", span: 6, rows: 3, range: "month" }]); + }); + + it("setWidgetRange ignores a window the widget does not offer", () => { + writeStored([ + { id: "top-titles", span: 6, rows: 3, range: "week" }, + { id: "users", span: 5, rows: 4 }, + ]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.setWidgetRange("top-titles", "hour"); + result.current.setWidgetRange("users", "month"); + }); + + expect(result.current.entries).toEqual([ + { id: "top-titles", span: 6, rows: 3, range: "week" }, + { id: "users", span: 5, rows: 4 }, + ]); + }); + + it("round-trips a chosen window through localStorage", () => { + const first = renderHook(() => useDashboardLayout()); + act(() => { + first.result.current.setWidgetRange("concurrent-streams-24h", "month"); + first.result.current.setWidgetRange("top-profiles", "day"); + }); + const saved = first.result.current.entries; + first.unmount(); + + const second = renderHook(() => useDashboardLayout()); + + expect(second.result.current.entries).toEqual(saved); + expect(second.result.current.entries).toContainEqual({ + id: "concurrent-streams-24h", + span: 6, + rows: 3, + range: "month", + }); + expect(second.result.current.entries).toContainEqual({ + id: "top-profiles", + span: 6, + rows: 3, + range: "day", + }); + }); + + it("addWidget starts a ranged widget on its default window", () => { + writeStored([{ id: "libraries", span: 7, rows: 4 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.addWidget("playback-reliability"); + }); + + expect(result.current.entries).toContainEqual({ + id: "playback-reliability", + span: 6, + rows: 2, + range: "day", + }); }); it("exposes hidden widgets in registry order", () => { writeStored([ - { id: "users", span: 5 }, - { id: "stat-storage", span: 3 }, + { id: "users", span: 5, rows: 4 }, + { id: "stat-storage", span: 3, rows: 1 }, ]); const { result } = renderHook(() => useDashboardLayout()); @@ -98,8 +315,8 @@ describe("useDashboardLayout", () => { ); }); - it("addWidget appends with the default span and persists", () => { - writeStored([{ id: "libraries", span: 7 }]); + it("addWidget appends with the default span and rows and persists", () => { + writeStored([{ id: "libraries", span: 7, rows: 4 }]); const { result } = renderHook(() => useDashboardLayout()); act(() => { @@ -107,8 +324,8 @@ describe("useDashboardLayout", () => { }); const expected = [ - { id: "libraries", span: 7 }, - { id: "now-playing", span: 12 }, + { id: "libraries", span: 7, rows: 4 }, + { id: "now-playing", span: 12, rows: 4 }, ]; expect(result.current.entries).toEqual(expected); expect(readStored()).toEqual({ version: 1, entries: expected }); @@ -122,15 +339,15 @@ describe("useDashboardLayout", () => { }); expect(result.current.entries.some((entry) => entry.id === "trakt-sync")).toBe(false); - expect(result.current.hiddenWidgets.map((w) => w.id)).toEqual(["trakt-sync"]); + expect(result.current.hiddenWidgets.map((w) => w.id)).toEqual(hiddenWidgetIds("trakt-sync")); expect(readStored().entries.some((entry) => entry.id === "trakt-sync")).toBe(false); }); it("moveWidget inserts before the target and persists", () => { writeStored([ - { id: "libraries", span: 7 }, - { id: "users", span: 5 }, - { id: "recent-activity", span: 12 }, + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, + { id: "recent-activity", span: 12, rows: 4 }, ]); const { result } = renderHook(() => useDashboardLayout()); @@ -152,8 +369,8 @@ describe("useDashboardLayout", () => { it("moveWidget with a null beforeId moves to the end", () => { writeStored([ - { id: "libraries", span: 7 }, - { id: "users", span: 5 }, + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, ]); const { result } = renderHook(() => useDashboardLayout()); @@ -166,25 +383,70 @@ describe("useDashboardLayout", () => { }); it("resizeWidget clamps the span and persists", () => { - writeStored([{ id: "users", span: 5 }]); + writeStored([{ id: "users", span: 5, rows: 4 }]); const { result } = renderHook(() => useDashboardLayout()); act(() => { - result.current.resizeWidget("users", 6); + result.current.resizeWidget("users", { span: 6 }); }); - expect(result.current.entries).toEqual([{ id: "users", span: 6 }]); - expect(readStored().entries).toEqual([{ id: "users", span: 6 }]); + expect(result.current.entries).toEqual([{ id: "users", span: 6, rows: 4 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 6, rows: 4 }]); act(() => { - result.current.resizeWidget("users", 99); + result.current.resizeWidget("users", { span: 99 }); }); - expect(result.current.entries).toEqual([{ id: "users", span: 8 }]); + expect(result.current.entries).toEqual([{ id: "users", span: 8, rows: 4 }]); act(() => { - result.current.resizeWidget("users", 1); + result.current.resizeWidget("users", { span: 1 }); }); - expect(result.current.entries).toEqual([{ id: "users", span: 4 }]); - expect(readStored().entries).toEqual([{ id: "users", span: 4 }]); + expect(result.current.entries).toEqual([{ id: "users", span: 4, rows: 4 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 4, rows: 4 }]); + }); + + it("resizeWidget clamps rows and leaves the span alone", () => { + writeStored([{ id: "users", span: 5, rows: 4 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", { rows: 6 }); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 5, rows: 6 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 5, rows: 6 }]); + + act(() => { + result.current.resizeWidget("users", { rows: 99 }); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 5, rows: 8 }]); + + act(() => { + result.current.resizeWidget("users", { rows: 0 }); + }); + expect(result.current.entries).toEqual([{ id: "users", span: 5, rows: 2 }]); + }); + + it("resizeWidget changes both axes at once", () => { + writeStored([{ id: "users", span: 5, rows: 4 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", { span: 8, rows: 2 }); + }); + + expect(result.current.entries).toEqual([{ id: "users", span: 8, rows: 2 }]); + expect(readStored().entries).toEqual([{ id: "users", span: 8, rows: 2 }]); + }); + + // trakt-sync pins both axes at 1 row, so a resize of a pinned axis is a no-op. + it("resizeWidget keeps an axis a widget pins", () => { + writeStored([{ id: "trakt-sync", span: 9, rows: 1 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("trakt-sync", { span: 12, rows: 5 }); + }); + + expect(result.current.entries).toEqual([{ id: "trakt-sync", span: 12, rows: 1 }]); }); it("resetLayout restores the defaults and clears storage", () => { @@ -192,7 +454,7 @@ describe("useDashboardLayout", () => { act(() => { result.current.removeWidget("users"); - result.current.resizeWidget("libraries", 12); + result.current.resizeWidget("libraries", { span: 12, rows: 6 }); }); expect(result.current.entries).not.toEqual(DEFAULT_LAYOUT); @@ -208,7 +470,8 @@ describe("useDashboardLayout", () => { const first = renderHook(() => useDashboardLayout()); act(() => { first.result.current.removeWidget("stat-shows"); - first.result.current.resizeWidget("trakt-sync", 12); + first.result.current.resizeWidget("trakt-sync", { span: 12 }); + first.result.current.resizeWidget("recent-errors", { span: 8, rows: 6 }); first.result.current.moveWidget("recent-activity", "now-playing"); }); const saved = first.result.current.entries; @@ -216,6 +479,191 @@ describe("useDashboardLayout", () => { const second = renderHook(() => useDashboardLayout()); expect(second.result.current.entries).toEqual(saved); - expect(second.result.current.hiddenWidgets.map((w) => w.id)).toEqual(["stat-shows"]); + expect(second.result.current.entries).toContainEqual({ + id: "recent-errors", + span: 8, + rows: 6, + }); + expect(second.result.current.hiddenWidgets.map((w) => w.id)).toEqual( + hiddenWidgetIds("stat-shows"), + ); + }); +}); + +describe("useDashboardLayout server persistence", () => { + beforeEach(() => { + window.localStorage.clear(); + mocks.query = { data: undefined, isSuccess: false }; + mocks.save.mockReset(); + mocks.reset.mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("adopts the server layout and mirrors it to localStorage", () => { + writeStored([{ id: "users", span: 5, rows: 4 }]); + serverLayout([ + { id: "libraries", span: 7, rows: 6 }, + { id: "now-playing", span: 12, rows: 2 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + const expected = [ + { id: "libraries", span: 7, rows: 6 }, + { id: "now-playing", span: 12, rows: 2 }, + ]; + expect(result.current.entries).toEqual(expected); + expect(readStored()).toEqual({ version: 1, entries: expected }); + expect(mocks.save).not.toHaveBeenCalled(); + }); + + it("sanitizes the adopted server layout", () => { + serverLayout([ + { id: "not-a-widget", span: 6, rows: 3 }, + { id: "users", span: 99, rows: 99 }, + { id: "users", span: 4, rows: 4 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([{ id: "users", span: 8, rows: 8 }]); + }); + + it("fills in default rows for a server layout saved before row heights", () => { + serverLayout([ + { id: "libraries", span: 7 }, + { id: "users", span: 5 }, + ]); + + const { result } = renderHook(() => useDashboardLayout()); + + const expected = [ + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, + ]; + expect(result.current.entries).toEqual(expected); + expect(readStored()).toEqual({ version: 1, entries: expected }); + }); + + it("keeps the local layout when the server document is not a v1 layout", () => { + writeStored([{ id: "users", span: 5, rows: 4 }]); + mocks.query = { + data: { layout: { version: 99, entries: [] }, updated_at: "2026-08-26T10:00:00Z" }, + isSuccess: true, + }; + + const { result } = renderHook(() => useDashboardLayout()); + + expect(result.current.entries).toEqual([{ id: "users", span: 5, rows: 4 }]); + }); + + it("migrates a local-only layout to the server exactly once", () => { + writeStored([{ id: "users", span: 5, rows: 6 }]); + serverNoLayout(); + + const { result, rerender } = renderHook(() => useDashboardLayout()); + + expect(mocks.save).toHaveBeenCalledTimes(1); + expect(mocks.save).toHaveBeenCalledWith({ + version: 1, + entries: [{ id: "users", span: 5, rows: 6 }], + }); + + rerender(); + expect(mocks.save).toHaveBeenCalledTimes(1); + expect(result.current.entries).toEqual([{ id: "users", span: 5, rows: 6 }]); + }); + + it("does not migrate when the browser has no stored layout", () => { + serverNoLayout(); + + const { result } = renderHook(() => useDashboardLayout()); + + expect(mocks.save).not.toHaveBeenCalled(); + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + }); + + it("does not adopt a server layout over an edit made while the query was in flight", () => { + const { result, rerender } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.removeWidget("users"); + }); + const edited = result.current.entries; + + serverLayout([{ id: "libraries", span: 7, rows: 4 }]); + rerender(); + + expect(result.current.entries).toEqual(edited); + }); + + it("debounces mutations into a single save of the full layout", () => { + vi.useFakeTimers(); + writeStored([ + { id: "libraries", span: 7, rows: 4 }, + { id: "users", span: 5, rows: 4 }, + ]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", { span: 4 }); + result.current.resizeWidget("users", { span: 6 }); + result.current.resizeWidget("users", { rows: 3 }); + result.current.removeWidget("libraries"); + }); + + expect(mocks.save).not.toHaveBeenCalled(); + + act(() => { + vi.advanceTimersByTime(DASHBOARD_LAYOUT_SAVE_DEBOUNCE_MS); + }); + + expect(mocks.save).toHaveBeenCalledTimes(1); + expect(mocks.save).toHaveBeenCalledWith({ + version: 1, + entries: [{ id: "users", span: 6, rows: 3 }], + }); + }); + + it("flushes a queued save when the dashboard unmounts", () => { + vi.useFakeTimers(); + writeStored([{ id: "users", span: 5, rows: 4 }]); + const { result, unmount } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", { span: 6, rows: 5 }); + }); + expect(mocks.save).not.toHaveBeenCalled(); + + unmount(); + + expect(mocks.save).toHaveBeenCalledTimes(1); + expect(mocks.save).toHaveBeenCalledWith({ + version: 1, + entries: [{ id: "users", span: 6, rows: 5 }], + }); + }); + + it("resetLayout deletes the server layout and drops the queued save", () => { + vi.useFakeTimers(); + writeStored([{ id: "users", span: 5, rows: 4 }]); + const { result } = renderHook(() => useDashboardLayout()); + + act(() => { + result.current.resizeWidget("users", { span: 6 }); + result.current.resetLayout(); + }); + + act(() => { + vi.advanceTimersByTime(DASHBOARD_LAYOUT_SAVE_DEBOUNCE_MS * 2); + }); + + expect(mocks.reset).toHaveBeenCalledTimes(1); + expect(mocks.save).not.toHaveBeenCalled(); + expect(result.current.entries).toEqual(DEFAULT_LAYOUT); + expect(window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY)).toBeNull(); }); }); diff --git a/web/src/components/admin/dashboard/useDashboardLayout.ts b/web/src/components/admin/dashboard/useDashboardLayout.ts index 0ea8cd4bd..ad70eaa20 100644 --- a/web/src/components/admin/dashboard/useDashboardLayout.ts +++ b/web/src/components/admin/dashboard/useDashboardLayout.ts @@ -1,9 +1,30 @@ -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import type { AdminDashboardLayoutDocument } from "@/api/types"; +import { + useAdminDashboardLayout, + useResetAdminDashboardLayout, + useSaveAdminDashboardLayout, +} from "@/hooks/queries/admin/dashboardLayout"; import { DASHBOARD_WIDGETS, DEFAULT_LAYOUT, findDashboardWidget } from "./registry"; -import type { DashboardLayoutEntry, DashboardWidgetDefinition, WidgetId } from "./types"; +import type { + DashboardLayoutEntry, + DashboardWidgetDefinition, + WidgetId, + WidgetRange, +} from "./types"; export const DASHBOARD_LAYOUT_STORAGE_KEY = "silo.admin-dashboard-layout.v1"; +// Edits are bursty — a drag emits several moves, a resize several spans — so +// the server write waits for the burst to settle. Local state and localStorage +// are updated synchronously, so the delay is never visible. +export const DASHBOARD_LAYOUT_SAVE_DEBOUNCE_MS = 800; + +// Version 1 is still version 1 with row heights and per-widget windows in it: +// `rows` and `range` are additive fields that older documents simply omit, and +// sanitizing fills them from the widget's defaults. A bump would only be needed +// for a change that makes an existing field mean something new. interface StoredLayout { version: 1; entries: DashboardLayoutEntry[]; @@ -16,40 +37,102 @@ function clampSpan(span: unknown, widget: DashboardWidgetDefinition): number { return Math.min(widget.maxSpan, Math.max(widget.minSpan, Math.round(span))); } -function loadStoredLayout(): DashboardLayoutEntry[] { +/** + * Row heights were added after the first layouts were saved, so an entry + * without them is the common case rather than a corrupt one: it predates the + * field and takes the widget's default height. + */ +function clampRows(rows: unknown, widget: DashboardWidgetDefinition): number { + if (typeof rows !== "number" || !Number.isFinite(rows)) { + return widget.defaultRows; + } + return Math.min(widget.maxRows, Math.max(widget.minRows, Math.round(rows))); +} + +/** + * The window a stored entry asks for, or the widget's default. + * + * A widget that offers no ranges never carries one, so a value stored while it + * did — or a range removed from its allowed list since — is dropped rather than + * requested from an endpoint that would clamp it into something else. + */ +function sanitizeRange(range: unknown, widget: DashboardWidgetDefinition): WidgetRange | undefined { + const ranges = widget.ranges; + if (!ranges) { + return undefined; + } + if (typeof range === "string" && (ranges.allowed as readonly string[]).includes(range)) { + return range as WidgetRange; + } + return ranges.default; +} + +/** + * Validates a layout document from any source — localStorage or the server — + * and drops what this build cannot render. Returns null when the value is not + * a v1 layout document at all, which callers read as "there is no layout here" + * rather than "the layout is empty". + */ +function sanitizeLayoutDocument(value: unknown): DashboardLayoutEntry[] | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const parsed = value as Partial; + if (parsed.version !== 1 || !Array.isArray(parsed.entries)) { + return null; + } + const seen = new Set(); + const entries: DashboardLayoutEntry[] = []; + for (const entry of parsed.entries) { + if (!entry || typeof entry !== "object" || typeof entry.id !== "string") { + continue; + } + const widget = findDashboardWidget(entry.id); + if (!widget || seen.has(widget.id)) { + continue; + } + seen.add(widget.id); + const sanitized: DashboardLayoutEntry = { + id: widget.id, + span: clampSpan(entry.span, widget), + rows: clampRows(entry.rows, widget), + }; + // Written only when the widget has ranges, so a layout document never + // carries `"range": undefined` for the widgets that do not. + const range = sanitizeRange(entry.range, widget); + if (range) { + sanitized.range = range; + } + entries.push(sanitized); + } + return entries; +} + +function readStoredLayout(): DashboardLayoutEntry[] | null { let raw: string | null = null; try { raw = window.localStorage.getItem(DASHBOARD_LAYOUT_STORAGE_KEY); } catch { - return [...DEFAULT_LAYOUT]; + return null; } if (!raw) { - return [...DEFAULT_LAYOUT]; + return null; } try { - const parsed = JSON.parse(raw) as Partial | null; - if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.entries)) { - return [...DEFAULT_LAYOUT]; - } - const seen = new Set(); - const entries: DashboardLayoutEntry[] = []; - for (const entry of parsed.entries) { - if (!entry || typeof entry !== "object" || typeof entry.id !== "string") { - continue; - } - const widget = findDashboardWidget(entry.id); - if (!widget || seen.has(widget.id)) { - continue; - } - seen.add(widget.id); - entries.push({ id: widget.id, span: clampSpan(entry.span, widget) }); - } - return entries; + return sanitizeLayoutDocument(JSON.parse(raw)); } catch { - return [...DEFAULT_LAYOUT]; + return null; } } +function loadStoredLayout(): DashboardLayoutEntry[] { + return readStoredLayout() ?? [...DEFAULT_LAYOUT]; +} + +function toLayoutDocument(entries: DashboardLayoutEntry[]): AdminDashboardLayoutDocument { + return { version: 1, entries }; +} + function persistLayout(entries: DashboardLayoutEntry[]) { try { const stored: StoredLayout = { version: 1, entries }; @@ -59,22 +142,128 @@ function persistLayout(entries: DashboardLayoutEntry[]) { } } +function clearStoredLayout() { + try { + window.localStorage.removeItem(DASHBOARD_LAYOUT_STORAGE_KEY); + } catch { + // Ignore storage failures; in-memory state still resets. + } +} + +/** A resize of one or both axes; an omitted axis keeps its current value. */ +export interface DashboardWidgetSize { + span?: number; + rows?: number; +} + export interface DashboardLayout { entries: DashboardLayoutEntry[]; hiddenWidgets: DashboardWidgetDefinition[]; isCustomizing: boolean; setCustomizing: (customizing: boolean) => void; moveWidget: (id: WidgetId, beforeId: WidgetId | null) => void; - resizeWidget: (id: WidgetId, span: number) => void; + resizeWidget: (id: WidgetId, size: DashboardWidgetSize) => void; + /** + * Change a widget's window. Unlike moving and resizing this is an everyday + * viewing action, not an arrangement one, so it works outside customize mode + * — and rides the same debounced save, because where an admin left a chart is + * part of the layout they expect to find again. + */ + setWidgetRange: (id: WidgetId, range: WidgetRange) => void; removeWidget: (id: WidgetId) => void; addWidget: (id: WidgetId) => void; resetLayout: () => void; } +/** + * Owns the admin's widget arrangement. + * + * localStorage is the instant-paint and offline copy; the server row is the + * source of truth across browsers. The hook paints from localStorage, adopts + * the server layout once it arrives, migrates a local-only layout up to the + * server the first time it finds none there, and debounces subsequent writes. + * A failed write never rolls back local state — the arrangement the admin sees + * is the one they just made. + */ export function useDashboardLayout(): DashboardLayout { const [entries, setEntries] = useState(loadStoredLayout); const [isCustomizing, setCustomizing] = useState(false); + const remote = useAdminDashboardLayout(); + const saveLayout = useSaveAdminDashboardLayout(); + const resetRemoteLayout = useResetAdminDashboardLayout(); + + const saveMutate = saveLayout.mutate; + const resetMutate = resetRemoteLayout.mutate; + + // The server response is adopted at most once per mount, and never over an + // edit the admin already made in this session. + const settledRef = useRef(false); + const editedRef = useRef(false); + const pendingRef = useRef(null); + const timerRef = useRef | null>(null); + + const flushSave = useCallback(() => { + if (timerRef.current !== null) { + clearTimeout(timerRef.current); + timerRef.current = null; + } + const pending = pendingRef.current; + pendingRef.current = null; + if (pending) { + saveMutate(toLayoutDocument(pending)); + } + }, [saveMutate]); + + const scheduleSave = useCallback( + (next: DashboardLayoutEntry[]) => { + pendingRef.current = next; + if (timerRef.current !== null) { + clearTimeout(timerRef.current); + } + timerRef.current = setTimeout(flushSave, DASHBOARD_LAYOUT_SAVE_DEBOUNCE_MS); + }, + [flushSave], + ); + + const cancelPendingSave = useCallback(() => { + if (timerRef.current !== null) { + clearTimeout(timerRef.current); + timerRef.current = null; + } + pendingRef.current = null; + }, []); + + // Send a queued write before this page goes away rather than dropping it. + useEffect(() => flushSave, [flushSave]); + + const remoteData = remote.data; + const remoteSettled = remote.isSuccess; + + useEffect(() => { + if (!remoteSettled || settledRef.current) { + return; + } + settledRef.current = true; + // An edit made while the query was in flight is newer than the response; + // its own debounced save carries it to the server. + if (editedRef.current) { + return; + } + const serverEntries = sanitizeLayoutDocument(remoteData?.layout); + if (serverEntries) { + setEntries(serverEntries); + persistLayout(serverEntries); + return; + } + // No server layout yet: hand this browser's arrangement up once so the + // admin's other browsers inherit it instead of starting from defaults. + const local = readStoredLayout(); + if (local) { + saveMutate(toLayoutDocument(local)); + } + }, [remoteSettled, remoteData, saveMutate]); + const update = useCallback( (updater: (prev: DashboardLayoutEntry[]) => DashboardLayoutEntry[]) => { setEntries((prev) => { @@ -82,11 +271,13 @@ export function useDashboardLayout(): DashboardLayout { if (next === prev) { return prev; } + editedRef.current = true; persistLayout(next); + scheduleSave(next); return next; }); }, - [], + [scheduleSave], ); const moveWidget = useCallback( @@ -114,20 +305,51 @@ export function useDashboardLayout(): DashboardLayout { ); const resizeWidget = useCallback( - (id: WidgetId, span: number) => { + (id: WidgetId, size: DashboardWidgetSize) => { update((prev) => { const widget = findDashboardWidget(id); if (!widget) { return prev; } - const nextSpan = clampSpan(span, widget); let changed = false; const next = prev.map((entry) => { - if (entry.id !== id || entry.span === nextSpan) { + if (entry.id !== id) { + return entry; + } + // Each axis is clamped to its own range, and an axis the caller left + // out keeps the value it already had rather than snapping to a + // default — a column drag must not silently restore a row height. + const nextSpan = size.span === undefined ? entry.span : clampSpan(size.span, widget); + const nextRows = size.rows === undefined ? entry.rows : clampRows(size.rows, widget); + if (nextSpan === entry.span && nextRows === entry.rows) { return entry; } changed = true; - return { ...entry, span: nextSpan }; + return { ...entry, span: nextSpan, rows: nextRows }; + }); + return changed ? next : prev; + }); + }, + [update], + ); + + const setWidgetRange = useCallback( + (id: WidgetId, range: WidgetRange) => { + update((prev) => { + const widget = findDashboardWidget(id); + // A range the widget does not offer is ignored rather than stored: the + // endpoints clamp their windows, so the chart would silently disagree + // with the picker. + if (!widget?.ranges || !widget.ranges.allowed.includes(range)) { + return prev; + } + let changed = false; + const next = prev.map((entry) => { + if (entry.id !== id || entry.range === range) { + return entry; + } + changed = true; + return { ...entry, range }; }); return changed ? next : prev; }); @@ -151,20 +373,30 @@ export function useDashboardLayout(): DashboardLayout { if (!widget || prev.some((entry) => entry.id === id)) { return prev; } - return [...prev, { id: widget.id, span: widget.defaultSpan }]; + const added: DashboardLayoutEntry = { + id: widget.id, + span: widget.defaultSpan, + rows: widget.defaultRows, + }; + if (widget.ranges) { + added.range = widget.ranges.default; + } + return [...prev, added]; }); }, [update], ); const resetLayout = useCallback(() => { - try { - window.localStorage.removeItem(DASHBOARD_LAYOUT_STORAGE_KEY); - } catch { - // Ignore storage failures; state still resets below. - } + // Drop the queued write first: saving the arrangement the admin just threw + // away would resurrect it on the next load. + cancelPendingSave(); + clearStoredLayout(); + editedRef.current = true; + settledRef.current = true; setEntries([...DEFAULT_LAYOUT]); - }, []); + resetMutate(); + }, [cancelPendingSave, resetMutate]); const hiddenWidgets = useMemo(() => { const visible = new Set(entries.map((entry) => entry.id)); @@ -178,6 +410,7 @@ export function useDashboardLayout(): DashboardLayout { setCustomizing, moveWidget, resizeWidget, + setWidgetRange, removeWidget, addWidget, resetLayout, diff --git a/web/src/components/admin/dashboard/widgetChrome.tsx b/web/src/components/admin/dashboard/widgetChrome.tsx new file mode 100644 index 000000000..a1910992c --- /dev/null +++ b/web/src/components/admin/dashboard/widgetChrome.tsx @@ -0,0 +1,59 @@ +import { createContext, useContext, useMemo, type ReactNode } from "react"; + +import type { WidgetId, WidgetRange } from "./types"; + +/** + * What a widget knows about its own frame. + * + * Widgets are rendered by the grid with no props — the registry stores a bare + * component type — so the one thing they need from their placement, the window + * the admin picked, arrives through context instead. + */ +export interface WidgetChrome { + /** The placed widget's id, or null when rendered outside the grid. */ + id: WidgetId | null; + range: WidgetRange; + setRange: (range: WidgetRange) => void; +} + +const WidgetChromeContext = createContext(null); + +/** + * The window a widget reads outside the grid: a plain day. + * + * Widgets are also rendered by unit tests and could be by any future host, and + * a chart that threw because nobody wrapped it would be a worse failure than + * one that quietly shows the usual day. + */ +const FALLBACK_CHROME: WidgetChrome = { + id: null, + range: "day", + setRange: () => {}, +}; + +export function WidgetChromeProvider({ + id, + range, + setRange, + children, +}: { + id: WidgetId; + range: WidgetRange | undefined; + setRange: (id: WidgetId, range: WidgetRange) => void; + children: ReactNode; +}) { + const value = useMemo( + () => ({ + id, + range: range ?? FALLBACK_CHROME.range, + setRange: (next: WidgetRange) => setRange(id, next), + }), + [id, range, setRange], + ); + return {children}; +} + +/** The window this widget is showing, and how to change it. */ +export function useWidgetRange(): WidgetChrome { + return useContext(WidgetChromeContext) ?? FALLBACK_CHROME; +} diff --git a/web/src/components/admin/dashboard/widgets/ConcurrentStreamsWidget.tsx b/web/src/components/admin/dashboard/widgets/ConcurrentStreamsWidget.tsx new file mode 100644 index 000000000..a6c2e952e --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/ConcurrentStreamsWidget.tsx @@ -0,0 +1,65 @@ +import { useMemo } from "react"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAdminTimeseries } from "@/hooks/queries/admin/dashboardInsights"; +import { + formatRangeTimestamp, + rangeEdgeLabels, + rangeHours, + rangePhrase, + rangeTitle, +} from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; +import { TimeseriesChartBody } from "./timeseriesChart"; +import { buildTimeseriesPoints } from "./timeseriesSeries"; + +/** + * Concurrent playback sessions over the chosen window, one point per bucket the + * server returned. Buckets the sampler missed break the line instead of + * dropping to zero — "the server was down" and "nobody was watching" are + * different facts. + * + * A wide window is bucketed server-side and each bucket reports its peak + * minute, so the summit of a spike survives being zoomed out. + */ +export function ConcurrentStreamsWidget() { + const { range } = useWidgetRange(); + const query = useAdminTimeseries(rangeHours(range)); + const points = useMemo( + () => buildTimeseriesPoints(query.data, (point) => point.streams), + [query.data], + ); + + const peak = points.reduce((max, point) => Math.max(max, point.value ?? 0), 0); + + return ( + + + + {rangeTitle("Concurrent streams", range)} + +
+ + Peak {peak.toLocaleString()} + + +
+ + + formatRangeTimestamp(range, t, { withTime: true })} + edgeLabels={rangeEdgeLabels(range)} + minTickStep={1} + fill + /> + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/EgressWidget.tsx b/web/src/components/admin/dashboard/widgets/EgressWidget.tsx new file mode 100644 index 000000000..4494aee17 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/EgressWidget.tsx @@ -0,0 +1,65 @@ +import { useMemo } from "react"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAdminTimeseries } from "@/hooks/queries/admin/dashboardInsights"; +import { formatMbps, formatMbpsValue } from "../format"; +import { + formatRangeTimestamp, + rangeEdgeLabels, + rangeHours, + rangePhrase, + rangeTitle, +} from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; +import { TimeseriesChartBody } from "./timeseriesChart"; +import { buildTimeseriesPoints } from "./timeseriesSeries"; + +/** + * Egress the deployment served over the chosen window, in Mbps. + * + * The sampler mixes two sources into one number: the rolling average each + * stream node reports, and the exact bytes each API process served itself. A + * node-less single-server install charts the second alone. Wide windows are + * bucketed server-side to the peak minute in each bucket, so "Peak" means the + * same thing at every range. + */ +export function EgressWidget() { + const { range } = useWidgetRange(); + const query = useAdminTimeseries(rangeHours(range)); + const points = useMemo( + () => buildTimeseriesPoints(query.data, (point) => point.egress_kbps / 1_000), + [query.data], + ); + + const peak = points.reduce((max, point) => Math.max(max, point.value ?? 0), 0); + + return ( + + + {rangeTitle("Egress", range)} +
+ + Peak {formatMbps(peak)} + + +
+ + + formatRangeTimestamp(range, t, { withTime: true })} + edgeLabels={rangeEdgeLabels(range)} + fill + /> + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/HealthStripWidget.test.tsx b/web/src/components/admin/dashboard/widgets/HealthStripWidget.test.tsx new file mode 100644 index 000000000..e1c6c9cda --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/HealthStripWidget.test.tsx @@ -0,0 +1,171 @@ +import { render, screen } from "@testing-library/react"; +import { MemoryRouter } from "react-router"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AdminServerStatus, StreamNode } from "@/api/types"; + +const mocks = vi.hoisted(() => ({ + useAdminServerStatus: vi.fn(), + useBuildInfo: vi.fn(), + useAdminNodes: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/settings", () => ({ + useAdminServerStatus: mocks.useAdminServerStatus, +})); +vi.mock("@/hooks/queries/admin/system", () => ({ + useBuildInfo: mocks.useBuildInfo, +})); +vi.mock("@/hooks/queries/admin/nodes", () => ({ + useAdminNodes: mocks.useAdminNodes, +})); + +import { formatLatency, formatUptime } from "../format"; +import { HealthStripWidget } from "./HealthStripWidget"; + +function status(overrides: Partial = {}): AdminServerStatus { + return { + started_at: new Date(Date.now() - 3 * 3_600_000).toISOString(), + restart_required: false, + restart_requested: false, + health: { + postgres: { configured: true, ok: true, latency_ms: 1.42 }, + redis: { configured: true, ok: true, latency_ms: 0.31 }, + errors_24h: 4, + warnings_24h: 12, + }, + ...overrides, + }; +} + +function node(overrides: Partial = {}): StreamNode { + return { + id: 1, + name: "node-1", + type: "transcode", + url: "http://node-1:8091", + enabled: true, + healthy: true, + active_jobs: 0, + group: null, + max_jobs: 4, + max_bandwidth_kbps: null, + egress_kbps: 0, + last_health_check: new Date().toISOString(), + created_at: new Date().toISOString(), + ...overrides, + }; +} + +function renderStrip() { + return render( + + + , + ); +} + +describe("formatUptime", () => { + const now = Date.parse("2026-08-26T12:00:00Z"); + + it("steps down to the coarsest useful unit", () => { + expect(formatUptime("2026-08-26T11:59:30Z", now)).toBe("30s"); + expect(formatUptime("2026-08-26T11:18:00Z", now)).toBe("42m"); + expect(formatUptime("2026-08-26T06:48:00Z", now)).toBe("5h 12m"); + expect(formatUptime("2026-08-26T07:00:00Z", now)).toBe("5h"); + expect(formatUptime("2026-08-23T08:00:00Z", now)).toBe("3d 4h"); + expect(formatUptime("2026-08-23T12:00:00Z", now)).toBe("3d"); + }); + + it("reports an em dash rather than a negative or bogus uptime", () => { + expect(formatUptime(undefined, now)).toBe("—"); + expect(formatUptime("not-a-date", now)).toBe("—"); + expect(formatUptime("2026-08-26T12:00:30Z", now)).toBe("0s"); + }); +}); + +describe("formatLatency", () => { + it("keeps decimals only where they carry information", () => { + expect(formatLatency(1.42)).toBe("1.42 ms"); + expect(formatLatency(0.31)).toBe("0.31 ms"); + expect(formatLatency(148.6)).toBe("149 ms"); + }); +}); + +describe("HealthStripWidget", () => { + beforeEach(() => { + mocks.useAdminServerStatus.mockReset(); + mocks.useBuildInfo.mockReset(); + mocks.useAdminNodes.mockReset(); + mocks.useBuildInfo.mockReturnValue({ data: { display: "v0.9.1" } }); + mocks.useAdminNodes.mockReturnValue({ data: [], isLoading: false, error: null }); + }); + + it("composes version, uptime, dependencies and error count", () => { + mocks.useAdminServerStatus.mockReturnValue({ + data: status(), + isLoading: false, + error: null, + }); + mocks.useAdminNodes.mockReturnValue({ + data: [node(), node({ id: 2, name: "node-2", healthy: false })], + isLoading: false, + error: null, + }); + + renderStrip(); + + expect(screen.getByText("v0.9.1")).toBeTruthy(); + expect(screen.getByText("3h")).toBeTruthy(); + expect(screen.getByText("1.42 ms")).toBeTruthy(); + expect(screen.getByText("0.31 ms")).toBeTruthy(); + expect(screen.getByText("1/2")).toBeTruthy(); + expect(screen.getByText("4")).toBeTruthy(); + expect(screen.getByText("12 warnings")).toBeTruthy(); + }); + + it("separates an unconfigured dependency from a broken one", () => { + mocks.useAdminServerStatus.mockReturnValue({ + data: status({ + health: { + postgres: { configured: true, ok: false }, + redis: { configured: false }, + errors_24h: 0, + warnings_24h: 0, + }, + }), + isLoading: false, + error: null, + }); + + renderStrip(); + + expect(screen.getByText("Unreachable")).toBeTruthy(); + expect(screen.getByText("Not configured")).toBeTruthy(); + }); + + it("says where transcodes run when no stream nodes are registered", () => { + mocks.useAdminServerStatus.mockReturnValue({ + data: status(), + isLoading: false, + error: null, + }); + + renderStrip(); + + expect(screen.getByText("none")).toBeTruthy(); + expect(screen.getByText("this server transcodes")).toBeTruthy(); + }); + + it("surfaces a failed status load", () => { + mocks.useAdminServerStatus.mockReturnValue({ + data: undefined, + isLoading: false, + error: new Error("boom"), + }); + + renderStrip(); + + expect(screen.getByText("Failed to load server health.")).toBeTruthy(); + }); +}); diff --git a/web/src/components/admin/dashboard/widgets/HealthStripWidget.tsx b/web/src/components/admin/dashboard/widgets/HealthStripWidget.tsx new file mode 100644 index 000000000..cb5223382 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/HealthStripWidget.tsx @@ -0,0 +1,170 @@ +import type { ComponentType, ReactNode } from "react"; +import { Link } from "react-router"; +import { AlertTriangle, Clock, Database, Server, Tag, Zap } from "lucide-react"; + +import type { AdminHealthComponent } from "@/api/types"; +import { Card, CardContent } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminNodes } from "@/hooks/queries/admin/nodes"; +import { useAdminServerStatus } from "@/hooks/queries/admin/settings"; +import { useBuildInfo } from "@/hooks/queries/admin/system"; +import { cn } from "@/lib/utils"; +import { SectionError } from "../feedback"; +import { formatLatency, formatUptime } from "../format"; + +/** + * One-line deployment health. + * + * Version, uptime and node health are composed here rather than served by + * `/admin/server/status`: the build info and the node list already have their + * own endpoints, and duplicating them into the status payload would give the + * dashboard two answers that can disagree. + */ +export function HealthStripWidget() { + const statusQuery = useAdminServerStatus(); + const buildQuery = useBuildInfo(); + const nodesQuery = useAdminNodes(); + + const status = statusQuery.data; + const health = status?.health; + const nodes = nodesQuery.data ?? []; + const healthyNodes = nodes.filter((node) => node.enabled && node.healthy).length; + + if (statusQuery.isLoading) { + return ; + } + + if (statusQuery.error || !status) { + return ( + + + + + + ); + } + + return ( + // One row tall by default, so the card's own padding is the compact one the + // loading skeleton has always promised rather than the Card default. + + + + + + + 0 && healthyNodes < nodes.length ? "warn" : undefined} + /> + 0 ? "error" : undefined} + to="/admin/logs" + /> + + + ); +} + +/** + * A dependency reports whether it is configured before whether it is up: a + * single-node deployment with no Redis is healthy, and rendering that the same + * as an unreachable Redis would invent an outage. + */ +function DependencyCell({ + icon, + label, + component, +}: { + icon: ComponentType<{ className?: string }>; + label: string; + component: AdminHealthComponent | undefined; +}) { + if (!component) { + return ; + } + if (!component.configured) { + return ; + } + if (!component.ok) { + return ; + } + return ( + + ); +} + +function HealthCell({ + icon: Icon, + label, + value, + detail, + tone, + to, +}: { + icon: ComponentType<{ className?: string }>; + label: string; + value: string; + detail?: string; + tone?: "warn" | "error"; + to?: string; +}) { + const body = ( + <> +
+ + {label} +
+
+ {value} +
+ {detail ? ( +
{detail}
+ ) : null} + + ); + + return {body}; +} + +function CellShell({ to, children }: { to?: string; children: ReactNode }) { + const className = "bg-surface border-border min-w-0 rounded-lg border p-2.5"; + if (to) { + return ( + + {children} + + ); + } + return
{children}
; +} diff --git a/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx b/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx index b53e78da0..663885cb9 100644 --- a/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx +++ b/web/src/components/admin/dashboard/widgets/LibrariesWidget.tsx @@ -42,7 +42,7 @@ export function LibrariesWidget() { return ( - + Libraries - + {librariesQuery.isLoading ? ( ) : librariesQuery.error ? ( diff --git a/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx b/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx index 9c90a5614..8c38fdd94 100644 --- a/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx +++ b/web/src/components/admin/dashboard/widgets/NowPlayingWidget.tsx @@ -18,8 +18,8 @@ export function NowPlayingWidget() { const sessions = sessionsQuery.data ?? []; return ( -
-
+
+
Now Playing
{sessions.length > 0 && ( )}
- {sessionsQuery.isLoading ? ( -
- {Array.from({ length: 2 }).map((_, i) => ( - - ))} -
- ) : sessionsQuery.error ? ( - - ) : sessions.length === 0 ? ( -
No active streams.
- ) : ( - <> + {/* The stream cards scroll inside the widget: a short widget shows two of + them rather than spilling out of its row. */} +
+ {sessionsQuery.isLoading ? (
- {sessions.slice(0, 4).map((session) => ( - + {Array.from({ length: 2 }).map((_, i) => ( + ))}
- {sessions.length > 4 && ( - - +{sessions.length - 4} more active streams - - )} - - )} + ) : sessionsQuery.error ? ( + + ) : sessions.length === 0 ? ( +
No active streams.
+ ) : ( + <> +
+ {sessions.slice(0, 4).map((session) => ( + + ))} +
+ {sessions.length > 4 && ( + + +{sessions.length - 4} more active streams + + )} + + )} +
); } diff --git a/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.test.tsx b/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.test.tsx new file mode 100644 index 000000000..0871fff44 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.test.tsx @@ -0,0 +1,203 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AdminPlaybackActivity } from "@/api/types"; + +const mocks = vi.hoisted(() => ({ + useAdminPlaybackActivity: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/dashboardInsights", () => ({ + useAdminPlaybackActivity: mocks.useAdminPlaybackActivity, +})); + +import { WidgetChromeProvider } from "../widgetChrome"; +import { PlaybackActivityWidget } from "./PlaybackActivityWidget"; +import { buildPlaybackActivityColumns } from "./playbackActivitySeries"; + +const HOUR_MS = 3_600_000; +const DAY_MS = 86_400_000; +const NOW = Date.parse("2026-08-26T14:37:00Z"); +const CURRENT_HOUR = Math.floor(NOW / HOUR_MS) * HOUR_MS; +const CURRENT_DAY = Math.floor(NOW / DAY_MS) * DAY_MS; + +function isoHour(hoursAgo: number): string { + return new Date(CURRENT_HOUR - hoursAgo * HOUR_MS).toISOString(); +} + +function isoDay(daysAgo: number): string { + return new Date(CURRENT_DAY - daysAgo * DAY_MS).toISOString(); +} + +function activity(overrides: Partial = {}): AdminPlaybackActivity { + return { + hours: 24, + bucket_seconds: 3600, + buckets: [], + reliability: { + sessions_started: 0, + transcode_starts: 0, + finalized_sessions: 0, + completed_sessions: 0, + completion_rate: 0, + unique_profiles: 0, + }, + profiles_active_24h: 0, + ...overrides, + }; +} + +describe("buildPlaybackActivityColumns", () => { + it("zero-fills the whole window when the response is empty", () => { + const columns = buildPlaybackActivityColumns([], { now: NOW }); + + expect(columns).toHaveLength(24); + expect(columns[0]?.t).toBe(CURRENT_HOUR - 23 * HOUR_MS); + expect(columns[23]?.t).toBe(CURRENT_HOUR); + for (const column of columns) { + expect([...column.segments]).toEqual([0, 0, 0]); + } + }); + + it("places sparse buckets at their hour and leaves the quiet hours at zero", () => { + const columns = buildPlaybackActivityColumns( + [ + { hour: isoHour(2), direct: 4, remux: 1, transcode: 2 }, + { hour: isoHour(23), direct: 1, remux: 0, transcode: 0 }, + ], + { now: NOW }, + ); + + expect(columns).toHaveLength(24); + expect([...(columns[21]?.segments ?? [])]).toEqual([4, 1, 2]); + expect([...(columns[0]?.segments ?? [])]).toEqual([1, 0, 0]); + expect([...(columns[22]?.segments ?? [])]).toEqual([0, 0, 0]); + expect([...(columns[23]?.segments ?? [])]).toEqual([0, 0, 0]); + }); + + it("ignores buckets outside the window and unparseable timestamps", () => { + const columns = buildPlaybackActivityColumns( + [ + { hour: isoHour(48), direct: 9, remux: 9, transcode: 9 }, + { hour: "not-a-timestamp", direct: 7, remux: 7, transcode: 7 }, + ], + { now: NOW }, + ); + + expect(columns).toHaveLength(24); + expect(columns.every((column) => column.segments.every((value) => value === 0))).toBe(true); + }); + + it("treats a missing response as an empty window", () => { + expect(buildPlaybackActivityColumns(undefined, { now: NOW })).toHaveLength(24); + }); + + // Past two days the endpoint groups by day, and zero-filling on an hourly + // grid would scatter those columns across empty hours. + it("zero-fills a week on the daily grid the server bucketed by", () => { + const columns = buildPlaybackActivityColumns( + [{ hour: isoDay(3), direct: 5, remux: 0, transcode: 1 }], + { hours: 168, bucketSeconds: 86_400, now: NOW }, + ); + + expect(columns).toHaveLength(7); + expect(columns[0]?.t).toBe(CURRENT_DAY - 6 * DAY_MS); + expect(columns[6]?.t).toBe(CURRENT_DAY); + expect([...(columns[3]?.segments ?? [])]).toEqual([5, 0, 1]); + expect([...(columns[4]?.segments ?? [])]).toEqual([0, 0, 0]); + }); + + it("covers a month with 31 daily columns", () => { + const columns = buildPlaybackActivityColumns([], { + hours: 744, + bucketSeconds: 86_400, + now: NOW, + }); + + expect(columns).toHaveLength(31); + }); + + it("falls back to hourly buckets when the response omits the width", () => { + const columns = buildPlaybackActivityColumns([], { hours: 24, bucketSeconds: 0, now: NOW }); + + expect(columns).toHaveLength(24); + }); +}); + +describe("PlaybackActivityWidget", () => { + beforeEach(() => { + mocks.useAdminPlaybackActivity.mockReset(); + }); + + it("renders one column per hour of the window", () => { + mocks.useAdminPlaybackActivity.mockReturnValue({ + data: activity({ + buckets: [ + { hour: new Date(Date.now() - HOUR_MS).toISOString(), direct: 2, remux: 0, transcode: 1 }, + ], + }), + isLoading: false, + error: null, + }); + + render(); + + const chart = screen.getByRole("img", { name: /playback sessions per hour/i }); + expect(chart.querySelectorAll(":scope > div")).toHaveLength(24); + expect(screen.getByText("3 sessions")).toBeTruthy(); + expect(screen.getByText("Direct stream")).toBeTruthy(); + expect(screen.getByText("Playback activity · last 24 h")).toBeTruthy(); + }); + + // A month asks for 744 hours and gets daily buckets back, so the chart has to + // draw 31 columns rather than 744 near-empty hourly ones. + it("renders one column per day when the server bucketed daily", () => { + mocks.useAdminPlaybackActivity.mockReturnValue({ + data: activity({ + hours: 744, + bucket_seconds: 86_400, + buckets: [ + { hour: new Date(Date.now() - DAY_MS).toISOString(), direct: 4, remux: 0, transcode: 2 }, + ], + }), + isLoading: false, + error: null, + }); + + render( + {}}> + + , + ); + + expect(mocks.useAdminPlaybackActivity).toHaveBeenCalledWith(744); + const chart = screen.getByRole("img", { name: /playback sessions per day/i }); + expect(chart.querySelectorAll(":scope > div")).toHaveLength(31); + expect(screen.getByText("6 sessions")).toBeTruthy(); + expect(screen.getByText("Playback activity · last 30 d")).toBeTruthy(); + }); + + it("says the window was quiet instead of drawing an empty axis", () => { + mocks.useAdminPlaybackActivity.mockReturnValue({ + data: activity(), + isLoading: false, + error: null, + }); + + render(); + + expect(screen.getByText("No playback in the last 24 hours")).toBeTruthy(); + }); + + it("surfaces a failed load", () => { + mocks.useAdminPlaybackActivity.mockReturnValue({ + data: undefined, + isLoading: false, + error: new Error("boom"), + }); + + render(); + + expect(screen.getByText("Failed to load playback activity.")).toBeTruthy(); + }); +}); diff --git a/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.tsx b/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.tsx new file mode 100644 index 000000000..25520a778 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/PlaybackActivityWidget.tsx @@ -0,0 +1,74 @@ +import { useMemo } from "react"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { formatTime } from "@/lib/datetime"; +import { useAdminPlaybackActivity } from "@/hooks/queries/admin/dashboardInsights"; +import { ChartEmptyState, ChartSkeleton, StackedColumnChart } from "../charts"; +import { SectionError } from "../feedback"; +import { formatDayLabel, rangeHours, rangePhrase, rangeTitle } from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; +import { + buildPlaybackActivityColumns, + DEFAULT_PLAYBACK_BUCKET_SECONDS, + PLAYBACK_SERIES_LABELS, +} from "./playbackActivitySeries"; + +/** Playback starts per bucket over the chosen window, stacked by play method. */ +export function PlaybackActivityWidget() { + const { range } = useWidgetRange(); + const hours = rangeHours(range); + const query = useAdminPlaybackActivity(hours); + const bucketSeconds = query.data?.bucket_seconds || DEFAULT_PLAYBACK_BUCKET_SECONDS; + const columns = useMemo( + () => buildPlaybackActivityColumns(query.data?.buckets, { hours, bucketSeconds }), + [query.data, hours, bucketSeconds], + ); + const total = columns.reduce( + (sum, column) => sum + column.segments.reduce((columnSum, value) => columnSum + value, 0), + 0, + ); + // Hourly buckets are read as clock times; daily ones as dates, since every + // column of a month would otherwise be labelled midnight. + const isDaily = bucketSeconds >= 86_400; + + return ( + + + + {rangeTitle("Playback activity", range)} + +
+ + {total.toLocaleString()} {total === 1 ? "session" : "sessions"} + + +
+ + + {query.isLoading ? ( + + ) : query.error ? ( + + ) : total === 0 ? ( + + ) : ( + + isDaily ? formatDayLabel(t) : formatTime(t, { hour: "numeric", minute: undefined }) + } + totalLabel="Sessions" + /> + )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/PlaybackReliabilityWidget.tsx b/web/src/components/admin/dashboard/widgets/PlaybackReliabilityWidget.tsx new file mode 100644 index 000000000..4b0e13449 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/PlaybackReliabilityWidget.tsx @@ -0,0 +1,92 @@ +import type { ReactNode } from "react"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminPlaybackActivity } from "@/hooks/queries/admin/dashboardInsights"; +import { SectionError } from "../feedback"; +import { rangeHours, rangeTitle } from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; + +/** + * Coarse playback health for the chosen window. + * + * Deliberately four counts and one rate: time-to-first-frame and failed-start + * counts would belong here, but nothing records playback *start* events yet + * (playback_history_admin only gains a row when a session finalizes). Adding + * them means capturing start telemetry in internal/playback first — until then + * this widget shows what the server actually knows rather than an estimate + * reverse-engineered from logs. See docs/admin-api.md. + */ +export function PlaybackReliabilityWidget() { + const { range } = useWidgetRange(); + const query = useAdminPlaybackActivity(rangeHours(range)); + const reliability = query.data?.reliability; + + return ( + + + + {rangeTitle("Playback reliability", range)} + + + + + {query.isLoading ? ( +
+ {Array.from({ length: 4 }).map((_, index) => ( + + ))} +
+ ) : query.error || !reliability ? ( + + ) : ( +
+ + + 0 + ? `${reliability.completed_sessions.toLocaleString()} of ${reliability.finalized_sessions.toLocaleString()} finished` + : "No finished sessions yet" + } + /> + +
+ )} + + + ); +} + +function MiniStat({ label, value, detail }: { label: string; value: string; detail?: ReactNode }) { + return ( +
+
+ {value} +
+
{label}
+ {detail ?
{detail}
: null} +
+ ); +} + +/** + * Completion is measured over finalized sessions only — sessions still playing + * have not had the chance to finish, so counting them would drag the rate down + * simply because someone is watching right now. + */ +function formatCompletionRate(rate: number, finalizedSessions: number): string { + if (finalizedSessions <= 0 || !Number.isFinite(rate)) { + return "—"; + } + return `${Math.round(Math.min(Math.max(rate, 0), 1) * 100)}%`; +} diff --git a/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx b/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx index 043132b98..c83509f85 100644 --- a/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx +++ b/web/src/components/admin/dashboard/widgets/RecentActivityWidget.tsx @@ -14,7 +14,7 @@ export function RecentActivityWidget() { return ( - + Recent Activity - + {sessionsQuery.isLoading ? ( ) : sessionsQuery.error ? ( diff --git a/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.test.tsx b/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.test.tsx new file mode 100644 index 000000000..159742e9d --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.test.tsx @@ -0,0 +1,105 @@ +import { render, screen } from "@testing-library/react"; +import { MemoryRouter } from "react-router"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { OperationalLogEntry } from "@/api/types"; + +const mocks = vi.hoisted(() => ({ + useOperationalLogs: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/logs", () => ({ + useOperationalLogs: mocks.useOperationalLogs, +})); + +import { RecentErrorsWidget } from "./RecentErrorsWidget"; + +function entry(overrides: Partial = {}): OperationalLogEntry { + return { + id: 1, + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + level: "error", + component: "playback", + message: "transcode session failed to start", + ...overrides, + }; +} + +function renderWidget() { + return render( + + + , + ); +} + +describe("RecentErrorsWidget", () => { + beforeEach(() => { + mocks.useOperationalLogs.mockReset(); + }); + + it("asks for both levels in one request", () => { + mocks.useOperationalLogs.mockReturnValue({ data: undefined, isLoading: true, error: null }); + + renderWidget(); + + expect(mocks.useOperationalLogs).toHaveBeenCalledWith({ level: "error,warn", limit: 8 }); + }); + + it("labels each level with a word, not only a color", () => { + mocks.useOperationalLogs.mockReturnValue({ + data: { + entries: [ + entry(), + entry({ id: 2, level: "warn", component: "scanner", message: "path is unreadable" }), + ], + }, + isLoading: false, + error: null, + }); + + renderWidget(); + + expect(screen.getByText("Error")).toBeTruthy(); + expect(screen.getByText("Warn")).toBeTruthy(); + expect(screen.getByText("transcode session failed to start")).toBeTruthy(); + expect(screen.getByText(/playback · 5m ago/)).toBeTruthy(); + expect(screen.getByText(/scanner/)).toBeTruthy(); + }); + + it("falls back to a readable label for an unexpected level", () => { + mocks.useOperationalLogs.mockReturnValue({ + data: { entries: [entry({ level: "debug" })] }, + isLoading: false, + error: null, + }); + + renderWidget(); + + expect(screen.getByText("debug")).toBeTruthy(); + }); + + it("says the log is quiet rather than rendering an empty list", () => { + mocks.useOperationalLogs.mockReturnValue({ + data: { entries: [] }, + isLoading: false, + error: null, + }); + + renderWidget(); + + expect(screen.getByText("No errors or warnings logged.")).toBeTruthy(); + }); + + it("surfaces a failed load", () => { + mocks.useOperationalLogs.mockReturnValue({ + data: undefined, + isLoading: false, + error: new Error("boom"), + }); + + renderWidget(); + + expect(screen.getByText("Failed to load recent errors.")).toBeTruthy(); + }); +}); diff --git a/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.tsx b/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.tsx new file mode 100644 index 000000000..05e97f8ab --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/RecentErrorsWidget.tsx @@ -0,0 +1,100 @@ +import { Link } from "react-router"; +import { AlertTriangle, Info, XCircle } from "lucide-react"; + +import type { OperationalLogEntry } from "@/api/types"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useOperationalLogs } from "@/hooks/queries/admin/logs"; +import { formatRelativeTime } from "@/lib/date"; +import { cn } from "@/lib/utils"; +import { SectionError, UserSkeletonRows } from "../feedback"; + +const ROW_LIMIT = 8; +/** One request for both levels — `level` takes a comma-separated list. */ +const LEVELS = "error,warn"; + +/** + * The newest error and warning lines from the operational log. + * + * Warnings sit beside errors on purpose: the log line that explains an error + * is usually a warning logged moments earlier, and splitting them across two + * widgets would hide that pairing. + */ +export function RecentErrorsWidget() { + const logsQuery = useOperationalLogs({ level: LEVELS, limit: ROW_LIMIT }); + const entries = logsQuery.data?.entries ?? []; + + return ( + + + Recent errors + + All logs › + + + + {logsQuery.isLoading ? ( + + ) : logsQuery.error ? ( + + ) : entries.length === 0 ? ( +
+ No errors or warnings logged. +
+ ) : ( +
+ {entries.slice(0, ROW_LIMIT).map((entry) => ( + + ))} +
+ )} + + + ); +} + +function LogRow({ entry }: { entry: OperationalLogEntry }) { + const tone = levelTone(entry.level); + const Icon = tone.icon; + return ( +
+ {/* Icon + word: the level has to survive being read by someone who + cannot tell the tints apart, and by a screenshot in grayscale. */} + + +
+
+ {entry.message} +
+
+ {entry.component || "server"} + {" · "} + {formatRelativeTime(entry.timestamp, { rounding: "floor", justNowLabel: "Just now" }) ?? + entry.timestamp} +
+
+
+ ); +} + +function levelTone(level: string) { + switch (level.toLowerCase()) { + case "error": + case "fatal": + return { label: "Error", icon: XCircle, className: "text-destructive" }; + case "warn": + case "warning": + return { label: "Warn", icon: AlertTriangle, className: "text-amber-500" }; + default: + return { label: level || "Log", icon: Info, className: "text-muted-foreground" }; + } +} diff --git a/web/src/components/admin/dashboard/widgets/ScanActivityWidget.tsx b/web/src/components/admin/dashboard/widgets/ScanActivityWidget.tsx new file mode 100644 index 000000000..9694a3fb0 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/ScanActivityWidget.tsx @@ -0,0 +1,129 @@ +import { useMemo } from "react"; +import { Link } from "react-router"; +import { AlertTriangle, CheckCircle2, CircleSlash, Clock, Loader2 } from "lucide-react"; + +import type { AutoscanScanStatus } from "@/api/types"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { useAdminLibraries } from "@/hooks/queries/admin/libraries"; +import { useAutoscanScans } from "@/hooks/queries/useAutoscan"; +import { formatActiveScanMode, formatActiveScanTrigger } from "@/lib/scanRuns"; +import { cn } from "@/lib/utils"; +import { SectionError, UserSkeletonRows } from "../feedback"; +import { formatScanDuration } from "../format"; + +const ROW_LIMIT = 8; + +/** + * Recently finished (and in-flight) scan runs. + * + * Duration is derived here rather than read from the row: the scans endpoint + * reports the two timestamps and nothing else, and a scan that is still + * running has no end to subtract from. + */ +export function ScanActivityWidget() { + const scansQuery = useAutoscanScans({ limit: ROW_LIMIT }); + const librariesQuery = useAdminLibraries(); + const scans = scansQuery.data?.rows ?? []; + + const libraryNames = useMemo(() => { + const names = new Map(); + for (const library of librariesQuery.data ?? []) { + names.set(library.id, library.name); + } + return names; + }, [librariesQuery.data]); + + return ( + + + Scan activity + + All scans › + + + + {scansQuery.isLoading ? ( + + ) : scansQuery.error ? ( + + ) : scans.length === 0 ? ( +
No scans yet.
+ ) : ( + + + + Library + Mode + Trigger + Status + Duration + + + + {scans.slice(0, ROW_LIMIT).map((scan) => ( + + + {libraryNames.get(scan.library_id) ?? `Library #${scan.library_id}`} + + + {formatActiveScanMode(scan)} + + + {formatActiveScanTrigger(scan.trigger)} + + + + + + {formatScanDuration(scan)} + + + ))} + +
+ )} + + + ); +} + +function ScanStatus({ status }: { status: AutoscanScanStatus }) { + const tone = scanStatusTone(status); + const Icon = tone.icon; + return ( + + + ); +} + +/** Icon and word together — the tint is a second signal, never the only one. */ +function scanStatusTone(status: AutoscanScanStatus) { + switch (status) { + case "completed": + return { label: "Completed", icon: CheckCircle2, className: "text-emerald-500" }; + case "failed": + return { label: "Failed", icon: AlertTriangle, className: "text-destructive" }; + case "running": + return { label: "Running", icon: Loader2, className: "text-sky-500" }; + case "cancelled": + return { label: "Cancelled", icon: CircleSlash, className: "text-amber-500" }; + case "accepted": + return { label: "Queued", icon: Clock, className: "text-muted-foreground" }; + } +} diff --git a/web/src/components/admin/dashboard/widgets/ScannerWidget.tsx b/web/src/components/admin/dashboard/widgets/ScannerWidget.tsx new file mode 100644 index 000000000..2305848f8 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/ScannerWidget.tsx @@ -0,0 +1,180 @@ +import { useMemo } from "react"; +import { Link } from "react-router"; +import { CheckCircle2, CircleSlash, ScanLine } from "lucide-react"; + +import type { ScanRun } from "@/api/types"; +import { useEventChannel } from "@/components/realtimeEventsContext"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminLibraries } from "@/hooks/queries/admin/libraries"; +import { useActiveScans } from "@/hooks/queries/admin/scans"; +import { useAutoscanStatus } from "@/hooks/queries/useAutoscan"; +import { formatRelativeTime } from "@/lib/date"; +import { compareActiveScans } from "@/lib/scanRuns"; +import { cn } from "@/lib/utils"; +import { SectionError } from "../feedback"; +import { formatDashboardLibraryScanProgress } from "../format"; + +const VISIBLE_LIBRARIES = 4; + +interface LibraryScanGroup { + libraryID: number; + primary: ScanRun; + count: number; +} + +/** + * Groups the live scans by library and picks the one worth showing. + * + * Grouping is what makes the "+N more" the progress formatter appends true: + * it counts other scans on the *same* library, which is what a reader assumes + * when the row is titled with a library name. + */ +function groupActiveScansByLibrary(scans: ScanRun[]): LibraryScanGroup[] { + const grouped = new Map(); + for (const scan of scans) { + if (scan.status !== "accepted" && scan.status !== "running") { + continue; + } + const existing = grouped.get(scan.library_id); + if (existing) { + existing.push(scan); + continue; + } + grouped.set(scan.library_id, [scan]); + } + + const groups: LibraryScanGroup[] = []; + for (const [libraryID, libraryScans] of grouped) { + libraryScans.sort(compareActiveScans); + const primary = libraryScans[0]; + if (!primary) { + continue; + } + groups.push({ libraryID, primary, count: libraryScans.length }); + } + return groups; +} + +/** + * Live scanner state. + * + * Active scans arrive only over the `scans` WebSocket channel — there is no + * REST equivalent — so this subscribes to the channel and reads the cache + * RealtimeEventsProvider hydrates. Queue depth and autoscan health come from + * the autoscan status endpoint, which is the authority on work this process + * has accepted but not started. + */ +export function ScannerWidget() { + useEventChannel("scans"); + const { data: activeScans = [] } = useActiveScans(); + const statusQuery = useAutoscanStatus(); + const librariesQuery = useAdminLibraries(); + const status = statusQuery.data; + + const libraryNames = useMemo(() => { + const names = new Map(); + for (const library of librariesQuery.data ?? []) { + names.set(library.id, library.name); + } + return names; + }, [librariesQuery.data]); + + const groups = useMemo(() => groupActiveScansByLibrary(activeScans), [activeScans]); + const hiddenGroups = Math.max(0, groups.length - VISIBLE_LIBRARIES); + const runningPolls = status?.running_polls?.length ?? 0; + + return ( + + + Scanner + + Activity › + + + + {statusQuery.isLoading && groups.length === 0 ? ( + + ) : statusQuery.error && groups.length === 0 ? ( + + ) : ( + <> + {groups.length === 0 ? ( +
+
+ ) : ( +
+ {groups.slice(0, VISIBLE_LIBRARIES).map((group) => ( +
+
+ {libraryNames.get(group.libraryID) ?? `Library #${group.libraryID}`} +
+
+ {formatDashboardLibraryScanProgress(group.primary, group.count)} +
+
+ ))} + {hiddenGroups > 0 ? ( +
+ + {hiddenGroups} more {hiddenGroups === 1 ? "library" : "libraries"} +
+ ) : null} +
+ )} + +
+ + + +
+ +
+ + {status?.enabled ? ( + <> + · + + last event{" "} + {formatRelativeTime(status.latest_event_at, { rounding: "floor" }) ?? "never"} + + + ) : null} +
+ + )} + + + ); +} + +function QueueCount({ label, value }: { label: string; value: number | undefined }) { + return ( +
+
+ {value === undefined ? "—" : value.toLocaleString()} +
+
{label}
+
+ ); +} + +function AutoscanState({ enabled }: { enabled: boolean | undefined }) { + if (enabled === undefined) { + return Autoscan status unknown; + } + const Icon = enabled ? CheckCircle2 : CircleSlash; + return ( + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/TopProfilesWidget.tsx b/web/src/components/admin/dashboard/widgets/TopProfilesWidget.tsx new file mode 100644 index 000000000..663e946cf --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/TopProfilesWidget.tsx @@ -0,0 +1,74 @@ +import { useMemo } from "react"; +import { Link } from "react-router"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAdminTopActivity } from "@/hooks/queries/admin/dashboardInsights"; +import { BarList, BarListSkeleton } from "../charts"; +import type { BarListItem } from "../charts"; +import { SectionError } from "../feedback"; +import { formatWatchTime } from "../format"; +import { rangeDays, rangePhrase, rangeTitle } from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; + +const ROW_LIMIT = 8; + +/** + * Most active household profiles of the chosen window. + * + * Rows are per profile, not per account: several profiles share one login, so + * the label carries both when the account name adds anything. + */ +export function TopProfilesWidget() { + const { range } = useWidgetRange(); + const query = useAdminTopActivity(rangeDays(range)); + const items = useMemo( + () => + (query.data?.profiles ?? []).slice(0, ROW_LIMIT).map((profile) => { + const account = profile.username || `User #${profile.user_id}`; + const name = profile.profile_name || profile.profile_id || account; + return { + id: `${profile.user_id}:${profile.profile_id}`, + label: name === account ? account : `${name} · ${account}`, + value: profile.plays, + secondary: formatWatchTime(profile.total_seconds), + to: `/admin/history?user_id=${profile.user_id}${ + profile.profile_id ? `&profile_id=${encodeURIComponent(profile.profile_id)}` : "" + }`, + }; + }), + [query.data], + ); + + return ( + + + + {rangeTitle("Most active profiles", range)} + +
+ + All history › + + +
+ + + {query.isLoading ? ( + + ) : query.error ? ( + + ) : ( + `${plays.toLocaleString()} ${plays === 1 ? "play" : "plays"}`} + emptyLabel={`No profile activity in ${rangePhrase(range)}`} + /> + )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/TopTitlesWidget.tsx b/web/src/components/admin/dashboard/widgets/TopTitlesWidget.tsx new file mode 100644 index 000000000..bd6110fc6 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/TopTitlesWidget.tsx @@ -0,0 +1,68 @@ +import { useMemo } from "react"; +import { Link } from "react-router"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAdminTopActivity } from "@/hooks/queries/admin/dashboardInsights"; +import { BarList, BarListSkeleton } from "../charts"; +import type { BarListItem } from "../charts"; +import { SectionError } from "../feedback"; +import { formatWatchTime } from "../format"; +import { rangeDays, rangePhrase, rangeTitle } from "../range"; +import { useWidgetRange } from "../widgetChrome"; +import { WidgetRangePicker } from "../WidgetRangePicker"; + +const ROW_LIMIT = 8; + +/** + * Most-played titles of the chosen window. Episodes are rolled up to their + * series by the endpoint, so a show appears once with the plays of all its + * episodes. + */ +export function TopTitlesWidget() { + const { range } = useWidgetRange(); + const query = useAdminTopActivity(rangeDays(range)); + const items = useMemo( + () => + (query.data?.titles ?? []).slice(0, ROW_LIMIT).map((title) => ({ + id: title.media_item_id, + label: title.title || title.media_item_id, + value: title.plays, + secondary: formatWatchTime(title.total_seconds), + // Link to the catalog item, not a filtered history view: for TV the id is + // a series content id while history rows store episode ids, so a history + // filter would match nothing. + to: title.media_item_id ? `/item/${encodeURIComponent(title.media_item_id)}` : undefined, + })), + [query.data], + ); + + return ( + + + {rangeTitle("Top titles", range)} +
+ + All history › + + +
+ + + {query.isLoading ? ( + + ) : query.error ? ( + + ) : ( + `${plays.toLocaleString()} ${plays === 1 ? "play" : "plays"}`} + emptyLabel={`No plays in ${rangePhrase(range)}`} + /> + )} + + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/TranscodeNodesWidget.tsx b/web/src/components/admin/dashboard/widgets/TranscodeNodesWidget.tsx new file mode 100644 index 000000000..8f06d14ee --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/TranscodeNodesWidget.tsx @@ -0,0 +1,140 @@ +import { Link } from "react-router"; +import { CheckCircle2, CircleSlash, XCircle } from "lucide-react"; + +import type { StreamNode } from "@/api/types"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAdminNodes } from "@/hooks/queries/admin/nodes"; +import { formatRelativeTime } from "@/lib/date"; +import { SectionError } from "../feedback"; +import { formatMbps } from "../format"; + +/** + * Remote transcode/streaming workers from `nodepool`. + * + * A deployment with no stream nodes is the normal single-server shape, not a + * misconfiguration, so the empty state says where transcodes actually run + * rather than reading as a missing dependency. + */ +export function TranscodeNodesWidget() { + const nodesQuery = useAdminNodes(); + const nodes = nodesQuery.data ?? []; + + return ( + + + Transcode nodes + + Manage › + + + + {nodesQuery.isLoading ? ( + <> + {Array.from({ length: 2 }).map((_, i) => ( + + ))} + + ) : nodesQuery.error ? ( + + ) : nodes.length === 0 ? ( +
+ No stream nodes — transcodes run on this server +
+ ) : ( + nodes.map((node) => ) + )} + + + ); +} + +function NodeRow({ node }: { node: StreamNode }) { + const status = nodeStatus(node); + const StatusIcon = status.icon; + const jobs = jobLoad(node); + const lastCheck = + formatRelativeTime(node.last_health_check, { + rounding: "floor", + justNowLabel: "Just now", + }) ?? "never checked"; + + return ( +
+
+
+
{node.name}
+
+ {node.type} + {node.group ? ` · ${node.group}` : ""} · checked {lastCheck} +
+
+ {/* Status is an icon plus the word: the tint alone would be the only + signal for anyone who cannot separate the hues. */} + + +
+
+
+
+
+
+
+
+ {jobs.label} + · + {formatMbps(node.egress_kbps / 1_000)} +
+
+
+ ); +} + +function nodeStatus(node: StreamNode) { + if (!node.enabled) { + return { label: "Disabled", icon: CircleSlash, className: "text-muted-foreground" }; + } + if (!node.healthy) { + return { label: "Unhealthy", icon: XCircle, className: "text-destructive" }; + } + return { label: "Healthy", icon: CheckCircle2, className: "text-emerald-500" }; +} + +/** + * A node with no `max_jobs` is uncapped, so there is no fraction to draw — the + * meter then reflects nothing and stays empty rather than inventing a ceiling. + */ +function jobLoad(node: StreamNode): { percent: number; label: string; max: number | null } { + const max = node.max_jobs && node.max_jobs > 0 ? node.max_jobs : null; + if (max === null) { + return { + percent: 0, + label: `${node.active_jobs.toLocaleString()} jobs`, + max: null, + }; + } + const percent = Math.max(0, Math.min(100, Math.round((node.active_jobs / max) * 100))); + return { + percent, + label: `${node.active_jobs.toLocaleString()}/${max.toLocaleString()} jobs`, + max, + }; +} diff --git a/web/src/components/admin/dashboard/widgets/UsersWidget.tsx b/web/src/components/admin/dashboard/widgets/UsersWidget.tsx index 418621382..c43a8819a 100644 --- a/web/src/components/admin/dashboard/widgets/UsersWidget.tsx +++ b/web/src/components/admin/dashboard/widgets/UsersWidget.tsx @@ -9,17 +9,30 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import type { AdminUser } from "@/api/types"; +import { formatRelativeTime } from "@/lib/date"; import { useAdminUsers } from "@/hooks/queries/admin/users"; import { SectionError, UserSkeletonRows } from "../feedback"; +// Most recently active first; users with no recorded activity sink to the +// bottom. The list endpoint itself returns account-creation order. +function byLastActive(a: AdminUser, b: AdminUser): number { + const at = a.last_active_at ? Date.parse(a.last_active_at) : 0; + const bt = b.last_active_at ? Date.parse(b.last_active_at) : 0; + if (at !== bt) { + return bt - at; + } + return a.username.localeCompare(b.username); +} + export function UsersWidget() { const navigate = useNavigate(); const usersQuery = useAdminUsers(); - const users = usersQuery.data ?? []; + const users = [...(usersQuery.data ?? [])].sort(byLastActive); return ( - + Users - + {usersQuery.isLoading ? ( ) : usersQuery.error ? ( @@ -40,7 +53,7 @@ export function UsersWidget() { User - Role + Last active Status @@ -59,16 +72,26 @@ export function UsersWidget() { > {u.username.charAt(0).toUpperCase()}
-
-
{u.username}
-
+
+
+ {u.username} + {u.role === "admin" && ( + + admin + + )} +
+
{u.email}
- - {u.role} + + {formatRelativeTime(u.last_active_at ?? null, { + rounding: "floor", + justNowLabel: "Just now", + }) ?? "—"} diff --git a/web/src/components/admin/dashboard/widgets/playbackActivitySeries.ts b/web/src/components/admin/dashboard/widgets/playbackActivitySeries.ts new file mode 100644 index 000000000..1d12c1129 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/playbackActivitySeries.ts @@ -0,0 +1,67 @@ +import type { AdminPlaybackActivityBucket } from "@/api/types"; +import { timeBuckets } from "../charts"; +import type { StackedColumnBucket } from "../charts"; + +const HOUR_MS = 3_600_000; + +/** Bucket width the endpoint falls back to when a response predates the field. */ +export const DEFAULT_PLAYBACK_BUCKET_SECONDS = 3600; + +/** + * Series order is fixed by entity, not by size: direct play always sits at the + * baseline in `--chart-1`, direct stream above it, transcode on top. "Remux" is + * the server's word for a direct stream; the dashboard uses the operator's. + */ +export const PLAYBACK_SERIES_LABELS = ["Direct play", "Direct stream", "Transcode"] as const; + +const EMPTY_SEGMENTS: readonly number[] = [0, 0, 0]; + +export interface PlaybackActivityColumnOptions { + /** Window length in hours, matching the `hours` the endpoint was asked for. */ + hours?: number; + /** Bucket width the endpoint grouped by, from `bucket_seconds`. */ + bucketSeconds?: number; + now?: number; +} + +/** + * The columns of the window, oldest first. + * + * The endpoint returns only buckets that saw a session, so quiet ones have to + * be filled in here — a stacked column chart that silently skipped them would + * compress the window and misplace every remaining column on the axis. The + * bucket width comes from the response rather than being assumed hourly: past + * two days the server groups by day, and zero-filling on the wrong grid would + * scatter the real columns between empty ones. + */ +export function buildPlaybackActivityColumns( + buckets: readonly AdminPlaybackActivityBucket[] | undefined, + options: PlaybackActivityColumnOptions = {}, +): StackedColumnBucket[] { + const { hours = 24, bucketSeconds = DEFAULT_PLAYBACK_BUCKET_SECONDS, now = Date.now() } = options; + const stepMs = + bucketSeconds > 0 ? bucketSeconds * 1_000 : DEFAULT_PLAYBACK_BUCKET_SECONDS * 1_000; + const columns = Math.max(1, Math.ceil((hours * HOUR_MS) / stepMs)); + + const to = Math.floor(now / stepMs) * stepMs; + const from = to - (columns - 1) * stepMs; + const samples = (buckets ?? []).flatMap((bucket) => { + const t = Date.parse(bucket.hour); + if (!Number.isFinite(t)) { + return []; + } + return [ + { + t: Math.floor(t / stepMs) * stepMs, + value: [bucket.direct, bucket.remux, bucket.transcode] as readonly number[], + }, + ]; + }); + + return timeBuckets(from, to, stepMs, samples, EMPTY_SEGMENTS).map( + (bucket) => ({ + t: bucket.t, + segments: bucket.value, + }), + ); +} diff --git a/web/src/components/admin/dashboard/widgets/statTiles.tsx b/web/src/components/admin/dashboard/widgets/statTiles.tsx index 22465de6e..1b1663acf 100644 --- a/web/src/components/admin/dashboard/widgets/statTiles.tsx +++ b/web/src/components/admin/dashboard/widgets/statTiles.tsx @@ -1,8 +1,14 @@ import type { ReactNode } from "react"; -import { Activity, Film, HardDrive, Tv, Users } from "lucide-react"; +import { Activity, Film, Gauge, HardDrive, Tv, UserCheck, Users, Zap } from "lucide-react"; import { Skeleton } from "@/components/ui/skeleton"; import { useAdminSessions, useAdminStats } from "@/hooks/queries/admin/stats"; -import { formatFileCount } from "../format"; +import { + useAdminPlaybackActivity, + useAdminTimeseries, +} from "@/hooks/queries/admin/dashboardInsights"; +import { classifyActivityMethod } from "@/pages/adminActivityPresentation"; +import { formatFileCount, formatMbps } from "../format"; +import { latestFreshPoint } from "./timeseriesSeries"; function StatTile({ label, @@ -20,13 +26,16 @@ function StatTile({ error: unknown; }) { if (isLoading) { - return ; + return ; } return ( -
-
-
{label}
+ // A stat tile is one grid row tall, so its content is centered in whatever + // height the row gives it and the padding is trimmed to the ~96px the + // loading skeleton has always reserved. +
+
+
{label}
{icon}
{error ? ( @@ -58,6 +67,70 @@ export function ActiveStreamsStatWidget() { ); } +// The sampler writes one row a minute, so a sample older than two minutes means +// the sampler is behind (or stopped) — show nothing rather than a stale rate +// presented as the current one. +const EGRESS_SAMPLE_MAX_AGE_MS = 2 * 60_000; + +export function EgressNowStatWidget() { + const timeseriesQuery = useAdminTimeseries(1); + const latest = latestFreshPoint(timeseriesQuery.data, EGRESS_SAMPLE_MAX_AGE_MS); + return ( + } + isLoading={timeseriesQuery.isLoading} + error={timeseriesQuery.error} + /> + ); +} + +export function TranscodeShareStatWidget() { + const sessionsQuery = useAdminSessions(); + const sessions = sessionsQuery.data ?? []; + // Same reduction the activity page uses: the server-computed + // effective_play_method when present, otherwise the per-stream decisions. + // Audio-only transcodes stay out of the count — this tile is about the + // sessions burning video encode capacity. + const transcoding = sessions.filter( + (session) => classifyActivityMethod(session) === "transcode", + ).length; + const share = sessions.length > 0 ? Math.round((transcoding / sessions.length) * 100) : null; + return ( + 0 + ? `${transcoding.toLocaleString()} of ${sessions.length.toLocaleString()} streams` + : "no active streams" + } + icon={} + isLoading={sessionsQuery.isLoading} + error={sessionsQuery.error} + /> + ); +} + +export function ProfilesActiveStatWidget() { + const activityQuery = useAdminPlaybackActivity(24); + const profiles = activityQuery.data?.profiles_active_24h; + return ( + } + isLoading={activityQuery.isLoading} + error={activityQuery.error} + /> + ); +} + export function MoviesStatWidget() { const statsQuery = useAdminStats(); const stats = statsQuery.data; diff --git a/web/src/components/admin/dashboard/widgets/timeseriesChart.tsx b/web/src/components/admin/dashboard/widgets/timeseriesChart.tsx new file mode 100644 index 000000000..cc70433a4 --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/timeseriesChart.tsx @@ -0,0 +1,85 @@ +import type { UseQueryResult } from "@tanstack/react-query"; + +import type { AdminTimeseries } from "@/api/types"; +import { ChartEmptyState, ChartSkeleton, LineChart } from "../charts"; +import type { LineChartPoint } from "../charts"; +import { SectionError } from "../feedback"; + +export interface TimeseriesChartBodyProps { + query: UseQueryResult; + points: readonly LineChartPoint[]; + seriesLabel: string; + ariaLabel: string; + errorMessage: string; + emptyMessage?: string; + formatValue?: (value: number) => string; + formatTick?: (value: number) => string; + formatTimestamp?: (t: number) => string; + /** How far back the window reaches, and where it ends; see LineChart. */ + edgeLabels?: { start: string; end: string }; + minTickStep?: number; + height?: number; + /** + * Fill the card's content area instead of a fixed `height`. Widget rows are + * resizable, so the plot follows the card rather than the other way round. + */ + fill?: boolean; +} + +/** + * Loading / error / collecting / plotted states for a sampled line chart, + * shared by the concurrent-streams and egress widgets. + * + * Never renders nothing: a widget with no data still says why, because the + * sampler only has history for the time the server was actually up. + */ +export function TimeseriesChartBody({ + query, + points, + seriesLabel, + ariaLabel, + errorMessage, + emptyMessage = "No samples yet", + formatValue, + formatTick, + formatTimestamp, + edgeLabels, + minTickStep, + height = 160, + fill = false, +}: TimeseriesChartBodyProps) { + if (query.isLoading) { + return ; + } + if (query.error) { + return ; + } + + const hasSample = points.some((point) => point.value !== null); + if (!hasSample) { + return ( + + ); + } + + return ( + + ); +} diff --git a/web/src/components/admin/dashboard/widgets/timeseriesSeries.ts b/web/src/components/admin/dashboard/widgets/timeseriesSeries.ts new file mode 100644 index 000000000..905ec708b --- /dev/null +++ b/web/src/components/admin/dashboard/widgets/timeseriesSeries.ts @@ -0,0 +1,74 @@ +import type { AdminTimeseries, AdminTimeseriesPoint } from "@/api/types"; +import { timeBuckets } from "../charts"; +import type { LineChartPoint } from "../charts"; + +/** + * Shaping for the widgets that plot `GET /admin/stats/timeseries` (concurrent + * streams, egress). Both read the same response and differ only in which field + * they select and how they format it, so the window arithmetic lives here. + */ + +const DEFAULT_RESOLUTION_SECONDS = 60; + +/** + * Dense minute series for the response window. + * + * The endpoint omits minutes the sampler never wrote — a restart, a process + * that was down — so the window is re-expanded here and missing minutes become + * `null`. That distinction is the whole point: a gap breaks the line, while a + * sampled minute with no streams draws a real zero. + */ +export function buildTimeseriesPoints( + series: AdminTimeseries | undefined, + select: (point: AdminTimeseriesPoint) => number, +): LineChartPoint[] { + if (!series) { + return []; + } + const resolution = + series.resolution_seconds > 0 ? series.resolution_seconds : DEFAULT_RESOLUTION_SECONDS; + const stepMs = resolution * 1_000; + const samples = series.points.flatMap((point) => { + const t = Date.parse(point.t); + if (!Number.isFinite(t)) { + return []; + } + const value = select(point); + return [{ t, value: Number.isFinite(value) ? value : null }]; + }); + + const from = Date.parse(series.from); + const to = Date.parse(series.to); + if (!Number.isFinite(from) || !Number.isFinite(to) || to < from) { + // Malformed window: plot whatever samples arrived rather than nothing. + return samples.map((sample) => ({ t: sample.t, value: sample.value })); + } + + const start = Math.floor(from / stepMs) * stepMs; + const end = Math.floor(to / stepMs) * stepMs; + return timeBuckets(start, end, stepMs, samples, null).map((bucket) => ({ + t: bucket.t, + value: bucket.present ? bucket.value : null, + })); +} + +/** + * The most recent sample, but only while it is fresh enough to still describe + * "now" — a stalled sampler must read as unknown, never as the last value it + * happened to write. + */ +export function latestFreshPoint( + series: AdminTimeseries | undefined, + maxAgeMs: number, + now: number = Date.now(), +): AdminTimeseriesPoint | null { + const point = series?.points[series.points.length - 1]; + if (!point) { + return null; + } + const t = Date.parse(point.t); + if (!Number.isFinite(t) || now - t > maxAgeMs) { + return null; + } + return point; +} diff --git a/web/src/hooks/queries/admin/dashboardInsights.test.ts b/web/src/hooks/queries/admin/dashboardInsights.test.ts new file mode 100644 index 000000000..ab5b56dd5 --- /dev/null +++ b/web/src/hooks/queries/admin/dashboardInsights.test.ts @@ -0,0 +1,154 @@ +import { createElement, type ReactNode } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + api: vi.fn(), +})); + +vi.mock("@/api/client", () => ({ + api: mocks.api, +})); + +import { adminKeys } from "../keys"; +import { + adminPlaybackActivityPath, + adminTimeseriesPath, + adminTopActivityPath, + normalizeInsightHours, + normalizeTopActivityDays, + useAdminPlaybackActivity, + useAdminTimeseries, + useAdminTopActivity, +} from "./dashboardInsights"; + +function createQueryClient() { + return new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); +} + +function createWrapper(queryClient: QueryClient) { + return function Wrapper({ children }: { children: ReactNode }) { + return createElement(QueryClientProvider, { client: queryClient }, children); + }; +} + +describe("dashboard insight windows", () => { + it.each([ + [24, 24], + [1, 1], + [168, 168], + [744, 744], + [0, 1], + [-5, 1], + [500, 500], + [5_000, 744], + [24.9, 24], + [Number.NaN, 24], + ])("clamps %p hours to %p", (input, expected) => { + expect(normalizeInsightHours(input)).toBe(expected); + }); + + it.each([ + [7, 7], + [1, 1], + [30, 30], + [0, 1], + [90, 30], + [Number.NaN, 7], + ])("clamps %p days to %p", (input, expected) => { + expect(normalizeTopActivityDays(input)).toBe(expected); + }); + + it("builds request paths from the clamped window", () => { + expect(adminTimeseriesPath(24)).toBe("/admin/stats/timeseries?hours=24"); + expect(adminTimeseriesPath(1)).toBe("/admin/stats/timeseries?hours=1"); + expect(adminTimeseriesPath(9_999)).toBe("/admin/stats/timeseries?hours=744"); + expect(adminPlaybackActivityPath(24)).toBe("/admin/stats/playback-activity?hours=24"); + expect(adminTopActivityPath(7)).toBe("/admin/stats/top-activity?days=7"); + expect(adminTopActivityPath(0)).toBe("/admin/stats/top-activity?days=1"); + }); + + it("keys every window under the prefix the dashboard refresh invalidates", () => { + const roots = [ + [adminKeys.dashboardTimeseriesRoot(), adminKeys.dashboardTimeseries(24)], + [adminKeys.playbackActivityRoot(), adminKeys.playbackActivity(24)], + [adminKeys.topActivityRoot(), adminKeys.topActivity(7)], + ] as const; + for (const [root, leaf] of roots) { + expect(leaf.slice(0, root.length)).toEqual([...root]); + } + }); +}); + +describe("dashboard insight hooks", () => { + beforeEach(() => { + mocks.api.mockReset(); + mocks.api.mockResolvedValue({}); + }); + + it("requests the timeseries window it was asked for and caches it by window", async () => { + const queryClient = createQueryClient(); + const { result } = renderHook(() => useAdminTimeseries(1), { + wrapper: createWrapper(queryClient), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(mocks.api).toHaveBeenCalledWith("/admin/stats/timeseries?hours=1"); + expect(queryClient.getQueryData(adminKeys.dashboardTimeseries(1))).toEqual({}); + expect(queryClient.getQueryData(adminKeys.dashboardTimeseries(24))).toBeUndefined(); + }); + + it("defaults playback activity to a 24 hour window", async () => { + const queryClient = createQueryClient(); + const { result } = renderHook(() => useAdminPlaybackActivity(), { + wrapper: createWrapper(queryClient), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(mocks.api).toHaveBeenCalledWith("/admin/stats/playback-activity?hours=24"); + expect(queryClient.getQueryData(adminKeys.playbackActivity(24))).toEqual({}); + }); + + it("defaults top activity to a 7 day window", async () => { + const queryClient = createQueryClient(); + const { result } = renderHook(() => useAdminTopActivity(), { + wrapper: createWrapper(queryClient), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(mocks.api).toHaveBeenCalledWith("/admin/stats/top-activity?days=7"); + expect(queryClient.getQueryData(adminKeys.topActivity(7))).toEqual({}); + }); + + it("paces from the dashboard loop: stale just under 60s, never self-polling", async () => { + const queryClient = createQueryClient(); + const wrapper = createWrapper(queryClient); + renderHook(() => useAdminTimeseries(24), { wrapper }); + renderHook(() => useAdminPlaybackActivity(24), { wrapper }); + renderHook(() => useAdminTopActivity(7), { wrapper }); + + await waitFor(() => expect(queryClient.getQueryCache().getAll()).toHaveLength(3)); + + const staleTimes = new Map( + queryClient + .getQueryCache() + .getAll() + .map((query) => [ + JSON.stringify(query.queryKey), + query.options as { staleTime?: number; refetchInterval?: unknown }, + ]), + ); + + expect(staleTimes.get(JSON.stringify(adminKeys.dashboardTimeseries(24)))?.staleTime).toBe( + 55_000, + ); + expect(staleTimes.get(JSON.stringify(adminKeys.playbackActivity(24)))?.staleTime).toBe(55_000); + expect(staleTimes.get(JSON.stringify(adminKeys.topActivity(7)))?.staleTime).toBe(300_000); + for (const options of staleTimes.values()) { + expect(options.refetchInterval).toBeUndefined(); + } + }); +}); diff --git a/web/src/hooks/queries/admin/dashboardInsights.ts b/web/src/hooks/queries/admin/dashboardInsights.ts new file mode 100644 index 000000000..8102fa5ee --- /dev/null +++ b/web/src/hooks/queries/admin/dashboardInsights.ts @@ -0,0 +1,93 @@ +import { useQuery } from "@tanstack/react-query"; + +import { api } from "@/api/client"; +import type { AdminPlaybackActivity, AdminTimeseries, AdminTopActivity } from "@/api/types"; +import { adminKeys } from "../keys"; + +/** + * Read hooks for the admin dashboard's aggregate insight endpoints. + * + * None of them carries a `refetchInterval` on purpose: the dashboard page owns + * pacing through its visibility-gated 60s loop (and the manual Refresh button), + * so a per-hook timer would only add fetches while the tab is hidden. The stale + * times sit just under that loop so a widget mounting mid-cycle serves from + * cache instead of firing a second request for the same window. + */ + +/** Just under the dashboard's 60s refresh cadence. */ +const SAMPLED_SERIES_STALE_TIME = 55_000; +/** Seven-day rollups move slowly; the server caches them for 5 minutes too. */ +const TOP_ACTIVITY_STALE_TIME = 5 * 60_000; + +export const DEFAULT_INSIGHT_HOURS = 24; +export const DEFAULT_TOP_ACTIVITY_DAYS = 7; + +// Mirrors the server-side clamps (internal/api/handlers/admin_stats_*.go). The +// client applies them before building the key so two requests the server would +// answer identically share one cache entry instead of splitting into a key the +// response does not actually match. +const MIN_HOURS = 1; +// 744 hours is 31 days: the sampler's retention window, and the widest window +// the endpoints answer. +const MAX_HOURS = 744; +const MIN_DAYS = 1; +const MAX_DAYS = 30; + +function clampWindow(value: number, min: number, max: number, fallback: number): number { + if (!Number.isFinite(value)) { + return fallback; + } + return Math.min(Math.max(Math.trunc(value), min), max); +} + +/** Hours window accepted by the timeseries and playback-activity endpoints. */ +export function normalizeInsightHours(hours: number = DEFAULT_INSIGHT_HOURS): number { + return clampWindow(hours, MIN_HOURS, MAX_HOURS, DEFAULT_INSIGHT_HOURS); +} + +/** Days window accepted by the top-activity endpoint. */ +export function normalizeTopActivityDays(days: number = DEFAULT_TOP_ACTIVITY_DAYS): number { + return clampWindow(days, MIN_DAYS, MAX_DAYS, DEFAULT_TOP_ACTIVITY_DAYS); +} + +export function adminTimeseriesPath(hours: number): string { + return `/admin/stats/timeseries?hours=${normalizeInsightHours(hours)}`; +} + +export function adminPlaybackActivityPath(hours: number): string { + return `/admin/stats/playback-activity?hours=${normalizeInsightHours(hours)}`; +} + +export function adminTopActivityPath(days: number): string { + return `/admin/stats/top-activity?days=${normalizeTopActivityDays(days)}`; +} + +/** Minute-resolution stream counts and egress for the last `hours`. */ +export function useAdminTimeseries(hours: number = DEFAULT_INSIGHT_HOURS) { + const window = normalizeInsightHours(hours); + return useQuery({ + queryKey: adminKeys.dashboardTimeseries(window), + queryFn: () => api(adminTimeseriesPath(window)), + staleTime: SAMPLED_SERIES_STALE_TIME, + }); +} + +/** Hourly playback counts by method, plus reliability and profile scalars. */ +export function useAdminPlaybackActivity(hours: number = DEFAULT_INSIGHT_HOURS) { + const window = normalizeInsightHours(hours); + return useQuery({ + queryKey: adminKeys.playbackActivity(window), + queryFn: () => api(adminPlaybackActivityPath(window)), + staleTime: SAMPLED_SERIES_STALE_TIME, + }); +} + +/** Most-played titles and most-active profiles over the last `days`. */ +export function useAdminTopActivity(days: number = DEFAULT_TOP_ACTIVITY_DAYS) { + const window = normalizeTopActivityDays(days); + return useQuery({ + queryKey: adminKeys.topActivity(window), + queryFn: () => api(adminTopActivityPath(window)), + staleTime: TOP_ACTIVITY_STALE_TIME, + }); +} diff --git a/web/src/hooks/queries/admin/dashboardLayout.test.ts b/web/src/hooks/queries/admin/dashboardLayout.test.ts new file mode 100644 index 000000000..670fd8053 --- /dev/null +++ b/web/src/hooks/queries/admin/dashboardLayout.test.ts @@ -0,0 +1,163 @@ +import { createElement, type ReactNode } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AdminDashboardLayoutResponse } from "@/api/types"; + +const mocks = vi.hoisted(() => ({ + api: vi.fn(), + toastError: vi.fn(), + toastSuccess: vi.fn(), +})); + +vi.mock("@/api/client", () => ({ + api: mocks.api, +})); + +vi.mock("sonner", () => ({ + toast: { + error: mocks.toastError, + success: mocks.toastSuccess, + }, +})); + +import { + useAdminDashboardLayout, + useResetAdminDashboardLayout, + useSaveAdminDashboardLayout, +} from "./dashboardLayout"; + +function createQueryClient() { + return new QueryClient({ + defaultOptions: { mutations: { retry: false }, queries: { retry: false } }, + }); +} + +function createWrapper(queryClient: QueryClient) { + return function Wrapper({ children }: { children: ReactNode }) { + return createElement(QueryClientProvider, { client: queryClient }, children); + }; +} + +const layoutDocument = { + version: 1, + entries: [{ id: "libraries", span: 7, rows: 4 }], +}; + +describe("useAdminDashboardLayout", () => { + beforeEach(() => { + mocks.api.mockReset(); + mocks.toastError.mockReset(); + mocks.toastSuccess.mockReset(); + }); + + it("reads the saved layout for the current admin", async () => { + const response: AdminDashboardLayoutResponse = { + layout: layoutDocument, + updated_at: "2026-08-26T10:00:00Z", + }; + mocks.api.mockResolvedValue(response); + const { result } = renderHook(() => useAdminDashboardLayout(), { + wrapper: createWrapper(createQueryClient()), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + + expect(mocks.api).toHaveBeenCalledWith("/admin/dashboard/layout"); + expect(result.current.data).toEqual(response); + }); + + it("reports a never-saved layout as null rather than an error", async () => { + mocks.api.mockResolvedValue({ layout: null, updated_at: null }); + const { result } = renderHook(() => useAdminDashboardLayout(), { + wrapper: createWrapper(createQueryClient()), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + + expect(result.current.data).toEqual({ layout: null, updated_at: null }); + }); +}); + +describe("useSaveAdminDashboardLayout", () => { + beforeEach(() => { + mocks.api.mockReset(); + mocks.toastError.mockReset(); + mocks.toastSuccess.mockReset(); + }); + + it("PUTs the layout document and seeds the query cache", async () => { + const queryClient = createQueryClient(); + mocks.api.mockResolvedValue(undefined); + const { result } = renderHook(() => useSaveAdminDashboardLayout(), { + wrapper: createWrapper(queryClient), + }); + + await act(async () => { + await result.current.mutateAsync(layoutDocument); + }); + + expect(mocks.api).toHaveBeenCalledWith("/admin/dashboard/layout", { + method: "PUT", + body: JSON.stringify({ layout: layoutDocument }), + }); + const cached = queryClient.getQueryData([ + "admin", + "dashboard", + "layout", + ]); + expect(cached?.layout).toEqual(layoutDocument); + expect(cached?.updated_at).toEqual(expect.any(String)); + expect(mocks.toastError).not.toHaveBeenCalled(); + }); + + it("surfaces a save failure once without discarding local state", async () => { + const queryClient = createQueryClient(); + mocks.api.mockRejectedValue(new Error("offline")); + const { result } = renderHook(() => useSaveAdminDashboardLayout(), { + wrapper: createWrapper(queryClient), + }); + + await act(async () => { + try { + await result.current.mutateAsync(layoutDocument); + } catch { + // The hook reports the failure through a toast; local state is kept. + } + }); + + expect(mocks.toastError).toHaveBeenCalledTimes(1); + expect(queryClient.getQueryData(["admin", "dashboard", "layout"])).toBeUndefined(); + }); +}); + +describe("useResetAdminDashboardLayout", () => { + beforeEach(() => { + mocks.api.mockReset(); + mocks.toastError.mockReset(); + mocks.toastSuccess.mockReset(); + }); + + it("DELETEs the layout and clears the cached document", async () => { + const queryClient = createQueryClient(); + queryClient.setQueryData(["admin", "dashboard", "layout"], { + layout: layoutDocument, + updated_at: "2026-08-26T10:00:00Z", + }); + mocks.api.mockResolvedValue(undefined); + const { result } = renderHook(() => useResetAdminDashboardLayout(), { + wrapper: createWrapper(queryClient), + }); + + await act(async () => { + await result.current.mutateAsync(); + }); + + expect(mocks.api).toHaveBeenCalledWith("/admin/dashboard/layout", { method: "DELETE" }); + expect(queryClient.getQueryData(["admin", "dashboard", "layout"])).toEqual({ + layout: null, + updated_at: null, + }); + }); +}); diff --git a/web/src/hooks/queries/admin/dashboardLayout.ts b/web/src/hooks/queries/admin/dashboardLayout.ts new file mode 100644 index 000000000..1495d890f --- /dev/null +++ b/web/src/hooks/queries/admin/dashboardLayout.ts @@ -0,0 +1,67 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; + +import { api } from "@/api/client"; +import type { AdminDashboardLayoutDocument, AdminDashboardLayoutResponse } from "@/api/types"; +import { adminKeys } from "../keys"; + +const DASHBOARD_LAYOUT_PATH = "/admin/dashboard/layout"; + +// A single toast id per concern: a burst of failed saves (offline, server +// down) collapses into one message instead of stacking one per attempt. +const SAVE_TOAST_ID = "admin-dashboard-layout-save"; +const RESET_TOAST_ID = "admin-dashboard-layout-reset"; + +/** + * Reads this admin account's saved dashboard arrangement. + * + * `staleTime: Infinity` on purpose: the layout only changes when this admin + * edits it, and every edit writes the new document straight into the cache, so + * there is nothing for a refetch to discover. The dashboard paints from + * localStorage first and adopts this result when it arrives. + */ +export function useAdminDashboardLayout() { + return useQuery({ + queryKey: adminKeys.dashboardLayout(), + queryFn: () => api(DASHBOARD_LAYOUT_PATH), + staleTime: Infinity, + gcTime: Infinity, + }); +} + +export function useSaveAdminDashboardLayout() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (layout: AdminDashboardLayoutDocument) => + api(DASHBOARD_LAYOUT_PATH, { + method: "PUT", + body: JSON.stringify({ layout }), + }), + onSuccess: (_data, layout) => { + queryClient.setQueryData(adminKeys.dashboardLayout(), { + layout, + updated_at: new Date().toISOString(), + }); + }, + onError: () => { + // The layout still works from local state, so this is informational. + toast.error("Failed to save the dashboard layout on the server", { id: SAVE_TOAST_ID }); + }, + }); +} + +export function useResetAdminDashboardLayout() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: () => api(DASHBOARD_LAYOUT_PATH, { method: "DELETE" }), + onSuccess: () => { + queryClient.setQueryData(adminKeys.dashboardLayout(), { + layout: null, + updated_at: null, + }); + }, + onError: () => { + toast.error("Failed to reset the dashboard layout on the server", { id: RESET_TOAST_ID }); + }, + }); +} diff --git a/web/src/hooks/queries/admin/logs.ts b/web/src/hooks/queries/admin/logs.ts index 238dd2b13..3988f7a0b 100644 --- a/web/src/hooks/queries/admin/logs.ts +++ b/web/src/hooks/queries/admin/logs.ts @@ -6,6 +6,7 @@ import { adminKeys } from "../keys"; export interface AdminLogQuery { cursor?: string; limit?: number; + /** One level, or a comma-separated list of them (e.g. `"error,warn"`). */ level?: string; component?: string; node_id?: string; diff --git a/web/src/hooks/queries/keys.ts b/web/src/hooks/queries/keys.ts index 8dd6b00b3..0cd1d874f 100644 --- a/web/src/hooks/queries/keys.ts +++ b/web/src/hooks/queries/keys.ts @@ -376,6 +376,16 @@ export const adminKeys = { sessions: () => ["admin", "sessions"] as const, serverSettings: () => ["admin", "serverSettings"] as const, serverStatus: () => ["admin", "serverStatus"] as const, + dashboardLayout: () => ["admin", "dashboard", "layout"] as const, + // Dashboard insight endpoints are keyed by their window so a 1h tile and a + // 24h chart cache separately; the matching `*Root` key is the prefix the + // dashboard's Refresh invalidates, which covers every window at once. + dashboardTimeseriesRoot: () => ["admin", "dashboard", "timeseries"] as const, + dashboardTimeseries: (hours: number) => ["admin", "dashboard", "timeseries", hours] as const, + playbackActivityRoot: () => ["admin", "dashboard", "playback-activity"] as const, + playbackActivity: (hours: number) => ["admin", "dashboard", "playback-activity", hours] as const, + topActivityRoot: () => ["admin", "dashboard", "top-activity"] as const, + topActivity: (days: number) => ["admin", "dashboard", "top-activity", days] as const, catalogSearchStatus: () => ["admin", "catalogSearchStatus"] as const, jellyfinCompatStatus: () => ["admin", "jellyfinCompatStatus"] as const, requestsRoot: () => ["admin", "requests"] as const, @@ -397,6 +407,7 @@ export const adminKeys = { }) => ["admin", "playbackHistory", params] as const, userIPs: (userId: number, days?: number) => ["admin", "users", userId, "ips", days] as const, ipUsers: (ip: string, days?: number) => ["admin", "ips", ip, days] as const, + operationalLogsRoot: () => ["admin", "logs", "app"] as const, operationalLogs: (params: Record) => ["admin", "logs", "app", params] as const, auditLogs: (params: Record) => ["admin", "logs", "audit", params] as const, diagnosticStatus: () => ["diagnostics", "status"] as const, @@ -462,6 +473,7 @@ export const adminKeys = { autoscanSources: () => ["admin", "autoscan", "sources"] as const, autoscanScanSourcePlugins: () => ["admin", "autoscan", "scan-source-plugins"] as const, autoscanStatus: () => ["admin", "autoscan", "status"] as const, + autoscanScansRoot: () => ["admin", "autoscan", "scans"] as const, autoscanScans: (params?: Record) => ["admin", "autoscan", "scans", params ?? {}] as const, autoscanEvents: (params?: Record) => diff --git a/web/src/pages/AdminDashboard.tsx b/web/src/pages/AdminDashboard.tsx index c30cbe0b6..654903cc4 100644 --- a/web/src/pages/AdminDashboard.tsx +++ b/web/src/pages/AdminDashboard.tsx @@ -14,6 +14,20 @@ import { adminKeys } from "@/hooks/queries/keys"; import { usePageActivity } from "@/hooks/usePageActivity"; import { buildAdminCommandNavSections } from "@/lib/adminNavigation"; +// Query prefixes the dashboard's stats/sessions/libraries/users refetch does +// not already cover. Widgets fetch these themselves, so Refresh only has to +// mark them stale; mounted widgets refetch, hidden ones stay cheap. +const DASHBOARD_WIDGET_QUERY_PREFIXES = [ + adminKeys.dashboardTimeseriesRoot(), + adminKeys.playbackActivityRoot(), + adminKeys.topActivityRoot(), + adminKeys.serverStatus(), + adminKeys.nodes(), + adminKeys.autoscanStatus(), + adminKeys.autoscanScansRoot(), + adminKeys.operationalLogsRoot(), +]; + const REFRESH_SPINNER_MIN_VISIBLE_MS = 1_000; const DASHBOARD_AUTO_REFRESH_MS = 60_000; const RELATIVE_UPDATED_LABEL_TICK_MS = 30_000; @@ -96,6 +110,13 @@ export default function AdminDashboard() { queryClient.invalidateQueries({ queryKey: adminKeys.sessions(), refetchType: "none" }), queryClient.invalidateQueries({ queryKey: adminKeys.libraries(), refetchType: "none" }), queryClient.invalidateQueries({ queryKey: adminKeys.users(), refetchType: "none" }), + // Widgets that own their own data: invalidate by prefix so every + // window variant is covered, and let the default refetchType refetch + // the ones actually mounted. Nothing here is refetched by hand — the + // widget's own hook does that when its query goes stale. + ...DASHBOARD_WIDGET_QUERY_PREFIXES.map((queryKey) => + queryClient.invalidateQueries({ queryKey }), + ), ]); const nextStats = await fetchAdminStats({ refresh: true }); queryClient.setQueryData(adminKeys.stats(), nextStats); @@ -246,7 +267,7 @@ export default function AdminDashboard() { Add widget - Drag a widget to move it · drag its right edge to resize · × removes it + Drag a widget to move it · drag its corner to resize · × removes it

+ {/* + No claim about what the cluster default *is*: it may be unset (each + node auto-discovers) or an explicit device list, and this form does + not know which. + */} {selectedCount === 0 - ? "Using the cluster default (auto-discover this node's devices)." + ? "Using the cluster-wide device setting." : selectedCount === 1 ? "All transcodes on this node run on the selected device." : "Transcodes on this node balance across the selected devices (least loaded first)."} @@ -697,11 +702,7 @@ function NodeForm({ id="node-hw-device-override" value={hwDeviceOverride} onChange={(e) => setHwDeviceOverride(e.target.value)} - placeholder={ - usesCUDADevices - ? "Cluster default (CUDA device 0)" - : "Cluster default (auto-discover)" - } + placeholder="Cluster default" /> {/* Each branch is a whole sentence rather than a shared tail: an @@ -710,6 +711,12 @@ function NodeForm({ NVENC addresses), so "no inventory yet" is only true of the other branch — and splitting one sentence across the conditional also lets JSX drop the space before "empty". + + Neither branch says what leaving this empty resolves to. Empty + inherits the cluster-wide playback.hw_device verbatim, and this + form does not know that value — naming a default here would be + a guess, and on the NVENC branch a dangerous one, since an + inherited /dev/dri path reaches NVENC as a CUDA identity. */}

{usesCUDADevices ? ( @@ -718,15 +725,16 @@ function NodeForm({ 0 or{" "} GPU-a1b2c3d4). NVENC addresses GPUs by CUDA identity, not by /dev/dri render path, so - the device picker does not apply to it. Leave empty to use the cluster default - (CUDA device 0). + the device picker does not apply to it. Leaving this empty inherits the + cluster-wide device setting, which must itself be a CUDA identity for NVENC to + use it — set one here when the cluster is configured with render paths. ) : ( <> Optional. Comma-separated render device paths this node transcodes on (e.g.{" "} /dev/dri/renderD128,/dev/dri/renderD129). This node has reported no device inventory yet, so there is nothing to pick - from. Leave empty to use the cluster default (auto-discover). + from. Leave empty to inherit the cluster-wide device setting. )}

From 8463622206f6617b9d878f8bb8fc2e2639f99c14 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:41:36 -0400 Subject: [PATCH 024/163] fix(app): keep library mounts reported when the path query fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sampler treats the media-root set it is given as the whole truth: refreshDisks prunes every path outside it — dropping the cached capacity readings with it — and diskStats omits them from the sample. Returning nothing on a failed DistinctLibraryPaths therefore did not "keep the previous pass's mounts reporting" as its comment claimed. A two-second database hiccup blanked every library mount from the admin resource panel and from Prometheus, and left the next pass reporting them unavailable until fresh probes landed, all while the mounts themselves were healthy. The provider now reuses the last set the database actually answered with. An empty result it genuinely returned is cached like any other, since an operator who removed their last library has no roots. The caching wrapper is split out so it is testable without a database. Swept the sampler's other providers for the same shape while here: none need it. FFmpegChildren reads this process's own /proc, DeviceIdentities globs /dev/dri, and DeviceSessions is an in-memory snapshot — their failures are permanent rather than transient, so there is no last-good answer to hold. Recorded that on Options.MediaRoots so the asymmetry reads as deliberate. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 45 ++++++++++++++++----- cmd/silo/main_test.go | 69 +++++++++++++++++++++++++++++++++ internal/nodemetrics/sampler.go | 9 +++++ 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 4c0c0d399..35163c334 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -16,6 +16,7 @@ import ( "os/signal" "path/filepath" "runtime/debug" + "slices" "sort" "strconv" "strings" @@ -167,23 +168,49 @@ func nodeCapabilityFetcher(jwtSecret string) nodepool.CapabilityFetcher { // libraryPathProvider adapts the folder repository to the sampler's media-root // provider. // -// It runs on the sampling goroutine every interval, so it is bounded -// separately from the caller's context: a database that has stopped answering -// must cost one interval's disk sampling, not the sampler. A failed read -// reports no roots for that pass rather than an error, because the previous -// pass's mounts are still tracked and keep reporting. +// It runs on the sampling goroutine every interval, so it is bounded separately +// from the caller's context: a database that has stopped answering must cost one +// interval's disk sampling, not the sampler. func libraryPathProvider(repo *catalog.FolderRepository) func(context.Context) []string { if repo == nil { return nil } - return func(ctx context.Context) []string { + return cachedLibraryPaths(func(ctx context.Context) ([]string, error) { queryCtx, cancel := context.WithTimeout(ctx, libraryPathQueryTimeout) defer cancel() - paths, err := repo.DistinctLibraryPaths(queryCtx) + return repo.DistinctLibraryPaths(queryCtx) + }) +} + +// cachedLibraryPaths wraps a library-root query so a failed read reuses the last +// set that succeeded. +// +// Returning nothing on error is not the harmless degradation it looks like. The +// sampler treats the returned set as the whole truth: refreshDisks prunes every +// path outside it — dropping the cached capacity readings with it — and +// diskStats omits them from the sample. A two-second database hiccup would +// therefore blank every library mount from the admin resource panel and from +// Prometheus, and leave the next pass reporting them unavailable until fresh +// probes land, all while the mounts themselves are perfectly healthy. Only the +// query failed, so the previous answer is still the best one available. +// +// An empty result that the database actually returned is cached like any other: +// an operator who removed their last library has genuinely no roots. +func cachedLibraryPaths(query func(context.Context) ([]string, error)) func(context.Context) []string { + var ( + mu sync.Mutex + lastGood []string + ) + return func(ctx context.Context) []string { + paths, err := query(ctx) + mu.Lock() + defer mu.Unlock() if err != nil { - slog.DebugContext(ctx, "library paths unavailable for resource sampling", "component", "app", "error", err) - return nil + slog.DebugContext(ctx, "library paths unavailable for resource sampling; reusing the last known set", + "component", "app", "error", err, "roots", len(lastGood)) + return slices.Clone(lastGood) } + lastGood = slices.Clone(paths) return paths } } diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index c9b6cce07..9103fc2d5 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -294,3 +295,71 @@ func TestReloadWatchSyncPluginProvidersDropsStaleProvidersOnCapabilityReadFailur t.Fatalf("stale provider %q remained registered", provider.Key()) } } + +// The sampler treats the returned root set as the whole truth: paths outside it +// are pruned — losing their cached capacity readings — and omitted from the +// sample. Returning nothing on a transient database error would therefore blank +// every library mount from the admin panel and from Prometheus, and leave the +// next pass reporting them unavailable until fresh probes land, all while the +// mounts are healthy. +func TestCachedLibraryPathsReusesTheLastGoodSetOnError(t *testing.T) { + var ( + paths []string + err error + ) + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { return paths, err }) + + paths = []string{"/mnt/movies", "/mnt/shows"} + if got := provider(context.Background()); !slices.Equal(got, paths) { + t.Fatalf("first read = %v, want %v", got, paths) + } + + paths, err = nil, errors.New("database is not answering") + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies", "/mnt/shows"}) { + t.Fatalf("read after an error = %v, want the last good set", got) + } + + // Recovery replaces it rather than merging. + paths, err = []string{"/mnt/movies"}, nil + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies"}) { + t.Fatalf("read after recovery = %v, want the fresh set", got) + } +} + +// An empty set the database actually returned is a real answer: an operator who +// removed their last library has no roots, and holding the old ones would keep +// reporting mounts the deployment no longer has. +func TestCachedLibraryPathsCachesADeliberateEmptyResult(t *testing.T) { + var ( + paths = []string{"/mnt/movies"} + err error + ) + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { return paths, err }) + provider(context.Background()) + + paths = nil + if got := provider(context.Background()); len(got) != 0 { + t.Fatalf("read = %v, want the deliberate empty result", got) + } + // And that empty result is what a later failure falls back to. + err = errors.New("database is not answering") + if got := provider(context.Background()); len(got) != 0 { + t.Fatalf("read after an error = %v, want the cached empty result", got) + } +} + +// The caller must not be able to mutate the cache through the slice it is given. +func TestCachedLibraryPathsDoesNotShareItsCachedSlice(t *testing.T) { + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { + return []string{"/mnt/movies"}, nil + }) + first := provider(context.Background()) + first[0] = "/tmp/clobbered" + + failing := cachedLibraryPaths(func(context.Context) ([]string, error) { + return nil, errors.New("boom") + }) + if got := failing(context.Background()); got != nil && len(got) != 0 { + t.Fatalf("an empty cache returned %v", got) + } +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index a471e85fc..bc61bb6b7 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -30,6 +30,15 @@ type Options struct { // MediaRoots returns library folder paths to sample alongside the scratch // dir. It is called on the sampling goroutine each pass, so it must be cheap // and must respect ctx; a nil provider samples the scratch dir only. + // + // What it returns is taken as the whole truth: paths outside the set are + // pruned, losing their cached readings, and omitted from the sample. A + // provider backed by anything fallible therefore has to answer with its last + // known set rather than nothing, or a momentary failure reads as every + // library mount disappearing. It is the only provider here that crosses a + // database or network boundary — the others read local state whose failure + // is permanent rather than transient — which is why it is the only one that + // needs to. MediaRoots func(ctx context.Context) []string // FFmpegChildren returns the pids whose DRM fdinfo counts as this node's GPU // work. The default is this process's direct ffmpeg children. From f757dfd2c934aa77cfaab164f9c82fbbd80ef512 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:46:49 -0400 Subject: [PATCH 025/163] fix(api): report an unconfirmed node reload after a policy edit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reloadNodeConfig swallowed its outcome, so an override edit whose nudge timed out or was refused looked identical to one the node had adopted. It now returns whether the node confirmed, and the caller names the consequence: until that node's own poll catches up, its backend comes from this server's pool while its device still comes from its own configuration, so a start dispatched to it in that window can pair the two wrongly and fail. The policy is still published when the node does not confirm. Withholding it would leave an override the operator has saved, and can see stored on the row, never reaching dispatch at all — nothing else re-reads that column — which is a silent permanent misconfiguration rather than a loud one bounded by the poll interval and self-healing. That trade is recorded at the call site. This narrows the window rather than closing it. Closing it means sending the effective device alongside the backend so both come from one source instead of two mechanisms with different timing; that is a change to the node start contract and to the recipe-card rebuild path, which deliberately re-reads HWAccel/HWDevice from node config today. Left for a maintainer decision rather than folded into a policy edit. Found by review of #794 (CodeRabbit). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 7 +++- internal/api/handlers/nodes.go | 34 ++++++++++++++---- internal/api/handlers/nodes_test.go | 53 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 39a1989c0..195c873ac 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -85,7 +85,12 @@ API also drops its own cached view of what the node can do, so the next session is planned against the new backend's tone-map executors rather than the previous one's. It is also best effort — a node that is unreachable still applies the change on its -next config reload (within 60 seconds). Either way the node re-advertises +next config reload (within 60 seconds). When it does not confirm, the policy is +published anyway and a warning names the node: withholding it would leave a +stored override never reaching dispatch, since nothing else re-reads the column. +Until that node's poll catches up its backend comes from this server while its +device comes from its own configuration, so a start dispatched to it in that +window can pair the two wrongly and fail. Either way the node re-advertises `capabilities.resolved` at its next capability snapshot (every 15 minutes). Two things do wait for a restart: the hardware encoder warmup that ran at boot, which stays primed for the old backend, and sessions already transcoding, which diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 555c9d90a..c9e792593 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -211,7 +211,23 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { // (QSV on a render node to NVENC on a CUDA index, say) gets up to a minute // of requests pairing the new backend with the old device. if nodeAccelerationChanged(previous, node) { - h.reloadNodeConfig(r.Context(), node) + if !h.reloadNodeConfig(r.Context(), node) { + // The node did not confirm. Its backend now comes from this + // server's pool while its device still comes from its own + // configuration, so until its poll catches up (within 60s) a start + // request can pair the new backend with the old device and fail. + // + // The policy is published anyway. Withholding it would leave an + // override an operator has saved and can see stored never reaching + // dispatch at all — nothing else re-reads the column — which is a + // silent permanent misconfiguration rather than a loud, bounded, + // self-healing one. Closing the window properly means sending the + // effective device alongside the backend so both come from one + // source; that is a change to the node start contract, not + // something to slip into a policy edit. + slog.WarnContext(r.Context(), "node has not adopted its new acceleration policy yet; transcodes dispatched to it may fail until its next config poll", + "component", "api", "node_id", node.ID, "name", node.Name) + } // And drop what this server believes the node can do. The v3 planning // cache holds the tone-map executors and transformation inventory the // *old* backend advertised, and it stays valid for its own TTL — so @@ -250,7 +266,8 @@ func sameOptionalString(a, b *string) bool { return *a == *b } -// reloadNodeConfig asks one node to re-read its configuration now. +// reloadNodeConfig asks one node to re-read its configuration now, reporting +// whether it confirmed. // // It targets /admin/reload-config, never /admin/force-reload: the latter tears // down every live playback session on a transcode node, which is a reasonable @@ -261,23 +278,24 @@ func sameOptionalString(a, b *string) bool { // watcher poll is the backstop, so a node that is unreachable or predates the // route must not turn a successful update into a failed one. It is bounded // tightly for the same reason — this runs after the response is written, but -// still on the request's goroutine and context. -func (h *NodeHandler) reloadNodeConfig(ctx context.Context, node *nodepool.Node) { +// still on the request's goroutine and context. The caller uses the result to +// say how far out of step the node may be, not to fail the update. +func (h *NodeHandler) reloadNodeConfig(ctx context.Context, node *nodepool.Node) bool { if node == nil || node.URL == "" { - return + return false } ctx, cancel := context.WithTimeout(ctx, nodeConfigReloadTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/reload-config", nil) if err != nil { - return + return false } req.Header.Set("Authorization", "Bearer "+h.jwtSecret) resp, err := (&http.Client{Timeout: nodeConfigReloadTimeout}).Do(req) if err != nil { slog.WarnContext(ctx, "node did not reload after an acceleration override change; it will pick it up on its next config poll", "component", "api", "node_id", node.ID, "name", node.Name, "error", logredact.SanitizeURLError(err)) - return + return false } defer func() { _ = resp.Body.Close() }() _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) @@ -287,7 +305,9 @@ func (h *NodeHandler) reloadNodeConfig(ctx context.Context, node *nodepool.Node) if resp.StatusCode < 200 || resp.StatusCode > 299 { slog.WarnContext(ctx, "node refused the reload after an acceleration override change; it will pick it up on its next config poll", "component", "api", "node_id", node.ID, "name", node.Name, "status", resp.StatusCode) + return false } + return true } // nodeConfigReloadTimeout bounds the post-update nudge. A config reload is a diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index a6f16f85a..83640a92f 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -325,6 +325,59 @@ func TestHandleUpdateNodeInvalidatesCapabilityCacheAfterAnOverrideChange(t *test } } +// A node that does not confirm the reload is out of step: its backend now comes +// from this server's pool while its device still comes from its own +// configuration. The policy is published regardless — withholding it would +// strand an override the operator can see stored, since nothing else re-reads +// the column — so what this pins down is that the update still succeeds and the +// caller learns the node did not confirm. +func TestReloadNodeConfigReportsAnUnconfirmedNode(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(node.Close) + + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + if handler.reloadNodeConfig(context.Background(), &nodepool.Node{ID: 1, Name: "gpu-1", URL: node.URL}) { + t.Fatal("a 500 from the node reported the reload as confirmed") + } + if handler.reloadNodeConfig(context.Background(), &nodepool.Node{ID: 1, Name: "gpu-1", URL: "http://127.0.0.1:1"}) { + t.Fatal("an unreachable node reported the reload as confirmed") + } +} + +// The update itself still succeeds and the new policy still reaches the pool: a +// stored override that never reaches dispatch is a silent permanent +// misconfiguration, where the mismatch is loud, bounded by the node's poll, and +// self-healing. +func TestHandleUpdateNodePublishesPolicyEvenWhenTheNodeDoesNotConfirm(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(node.Close) + + qsv, nvenc := "qsv", "nvenc" + before := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &qsv} + after := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &nvenc} + repo := &stubNodeRepository{updateResult: after, node: before} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + invalidated := make(chan string, 4) + handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) + + recorder := httptest.NewRecorder() + handler.HandleUpdateNode(recorder, updateNodeRequest(t, `{"hw_accel_override":"nvenc"}`)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want the update to succeed anyway", recorder.Code) + } + select { + case <-invalidated: + default: + t.Fatal("the capability cache was not dropped when the node did not confirm") + } +} + // An edit that moves neither override leaves the cache alone: re-probing every // node on every rename would put ffmpeg execs behind an unrelated form save. func TestHandleUpdateNodeKeepsCapabilityCacheWithoutAnOverrideChange(t *testing.T) { From 940abeafe8c70a2b69086237b1bbac3d1f4b5947 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:51:52 -0400 Subject: [PATCH 026/163] fix(nodepool): stop a skipped backend counting as capability drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit computeCapabilityDrift flattened Skipped into "not verified", so a backend that had passed its probe and is now skipped was recorded as lost. Skipped means no probe ran, because the node cannot open that backend's configured devices — a statement about access rather than about hardware, and one the GPU column already reports on its own. It also contradicted hardwareProbesClean, which counts a skipped backend as clean: the note was set by one rule and cleared by the other on the next hash change, so it flapped with nothing having changed. The two now agree. The distinction is "could not try" against "tried and the driver said no". A backend that was probed and failed is still a loss, and so is one that stopped being reported at all, which is what a card disappearing looks like — both are pinned down by tests beside the new one. Found by review of #794 (CodeRabbit). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 9 ++-- internal/nodepool/health.go | 22 ++++++++-- internal/nodepool/health_drift_test.go | 61 ++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 195c873ac..34cae836d 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -239,9 +239,12 @@ Semantics worth knowing: that probed nothing at all does not clear it either: a GPU that disappeared completely leaves no candidate backend to fail, and the absence of anything to probe is not evidence of recovery. -- A backend reported as `skipped` does not hold the note open. Skipping means - the node cannot open the devices, which is a statement about access rather - than about hardware. +- A backend reported as `skipped` neither sets the note nor holds it open. + Skipping means no probe ran because the node cannot open the backend's + configured devices, which is a statement about access rather than about + hardware — the GPU column reports that state separately. A backend that was + probed and *failed* is a loss, as is one that stopped being reported at all, + which is what a card disappearing looks like. - Only a loss is reported. Added hardware is not drift. - A node's first stored report carries none — there is nothing to compare it against. diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index bd7e4846f..c21bc237e 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -701,14 +701,28 @@ func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, pars } drift.previousResolved = previous.Resolved drift.resolved = current.Resolved - verifiedNow := make(map[string]bool, len(current.DetectedBackends)) + // Skipped is carried, not flattened into "not verified": it means no probe + // ran because the node cannot open the backend's configured devices, which + // is a statement about access rather than about hardware. Treating it as a + // loss also contradicted hardwareProbesClean, which counts a skipped + // backend as clean — the note would be set by one rule and cleared by the + // other on the next hash change, flapping without anything having changed. + type probeOutcome struct{ verified, skipped bool } + now := make(map[string]probeOutcome, len(current.DetectedBackends)) for _, backend := range current.DetectedBackends { - verifiedNow[backend.Backend] = backend.Verified + now[backend.Backend] = probeOutcome{verified: backend.Verified, skipped: backend.Skipped} } for _, backend := range previous.DetectedBackends { - if backend.Verified && !verifiedNow[backend.Backend] { - drift.lostBackends = append(drift.lostBackends, backend.Backend) + if !backend.Verified { + continue + } + // A backend missing from the new report entirely had no candidate + // hardware left to probe, which is the GPU-disappeared case and is a + // genuine loss. + if outcome, reported := now[backend.Backend]; reported && (outcome.verified || outcome.skipped) { + continue } + drift.lostBackends = append(drift.lostBackends, backend.Backend) } drift.lostDevices = lostRenderDevices(previous, current) return drift, true diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 2375e1353..38446e15b 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -421,3 +421,64 @@ func TestComputeCapabilityDriftReportsAReplacedCardInTheSameSlot(t *testing.T) { t.Fatalf("lostDevices = %v, want the replaced card reported gone", drift.lostDevices) } } + +// Skipped means no probe ran, because the node cannot open the backend's +// configured devices — a statement about access, not about hardware. Counting +// it as a loss also contradicted hardwareProbesClean, which treats a skipped +// backend as clean: the note would be set by one rule and cleared by the other +// on the next hash change, flapping with nothing having changed. +func TestComputeCapabilityDriftDoesNotTreatASkippedBackendAsLost(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":false,"skipped":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostBackends) != 0 { + t.Fatalf("lostBackends = %v, want a skipped backend not counted as lost", drift.lostBackends) + } + // And the pair round-trips: what does not set the note must not be held + // open by it either. + standing := "verified hardware backends lost: qsv" + if got := resolveDriftNote(&standing, drift, parsed, []byte(after)); got == nil || *got != standing { + t.Fatalf("resolveDriftNote = %v, want a skipped report to leave a standing note alone", got) + } +} + +// A backend that fails its probe outright is still a loss — the distinction is +// "could not try" versus "tried and the driver said no". +func TestComputeCapabilityDriftStillCatchesAFailedBackend(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":false,"reason":"h264_qsv smoke encode failed"}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostBackends) != 1 || drift.lostBackends[0] != "qsv" { + t.Fatalf("lostBackends = %v, want the failing backend reported", drift.lostBackends) + } +} + +// A backend that vanishes from the report had no candidate hardware left to +// probe at all, which is the GPU-disappeared case and a genuine loss. +func TestComputeCapabilityDriftCatchesABackendThatStoppedBeingReported(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":[],"detected_backends":[]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostBackends) != 1 || drift.lostBackends[0] != "qsv" { + t.Fatalf("lostBackends = %v, want the vanished backend reported", drift.lostBackends) + } +} From b3c4179b123c42cf20636d2f841f801d1830c171 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:02:10 -0400 Subject: [PATCH 027/163] style(app): drop a redundant nil check staticcheck flagged len() on a nil slice is zero, so the guard added beside it in 8463622 said nothing. Caught by the changed-lines lint gate, which I had not been able to run locally; golangci-lint is installed now and the same invocation CI uses reports no issues across this branch. Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index 9103fc2d5..4f4c93e74 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -359,7 +359,7 @@ func TestCachedLibraryPathsDoesNotShareItsCachedSlice(t *testing.T) { failing := cachedLibraryPaths(func(context.Context) ([]string, error) { return nil, errors.New("boom") }) - if got := failing(context.Background()); got != nil && len(got) != 0 { + if got := failing(context.Background()); len(got) != 0 { t.Fatalf("an empty cache returned %v", got) } } From d5783f4044a7c0ea884a9aa30c4c50b02abf0c56 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:22:19 -0400 Subject: [PATCH 028/163] fix(nodes): require regained hardware to clear drift; reserve the auto-detected device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from the review of 940abea. Clearing a drift note asked only that every backend the node probed passed. On a multi-GPU node that lost one card the survivor passes just as cleanly, and once the degraded report is stored the delta finds nothing lost ever again — so the next unrelated hash change erased a standing regression while the card was still missing. Clearing now also requires the report to have regained something the stored one lacked: a backend that verifies again, or a device identity that is back. The empty-inventory case fixed earlier was the same defect seen from one side; this is the general rule. Separately, an explicitly configured backend short-circuits resolution, so a host running hw_accel=qsv with no hw_device never walks its hardware and had no verified device to reserve — reporting zero GPU sessions for every transcode it ran, the same hole the auto path had, on the branch that never probes. It now falls back to the render node execution is about to pick anyway. That fallback made two existing tests host-dependent: both asserted an empty setting stays unresolved, which was only true because the machine running them has no /dev/dri. They now point at an empty device directory so they assert the invariant rather than the host. Swept the other AcquireHWDevice call sites in tests: the rest are either already hermetic or NVENC, which returns before this branch. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 16 +++---- docs/wiki/admin/monitoring-nodes.md | 8 ++-- internal/nodepool/health.go | 27 ++++++++++++ internal/nodepool/health_drift_test.go | 46 ++++++++++++++++++--- internal/playback/gpudetect_publish_test.go | 43 +++++++++++++------ internal/playback/hwdevice.go | 10 +++++ internal/playback/hwdevice_test.go | 24 ++++++++--- 7 files changed, 141 insertions(+), 33 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 34cae836d..ddc074bc0 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -174,7 +174,7 @@ Each entry in `last_stats.gpu`: |---|---|---| | `device` | string | The render node path (`/dev/dri/renderD128`), or `cuda:N` for an NVIDIA GPU with no readable DRM node. | | `vendor` | string | `intel`, `nvidia` or `amd`. Omitted when sysfs names a vendor we do not recognize. | -| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. With no `playback.hw_device` configured the workload is counted against the render device auto-detection verified the backend on; it goes uncounted only when no probe has verified one, which is the state of a node whose backend was named explicitly and never walked. | +| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. With no `playback.hw_device` configured the workload is counted against the render device the transcode will actually open — the one auto-detection verified the backend on, or the first available render node when the backend was named explicitly and no detection walk ran. It goes uncounted only on a host with no render device at all. | | `video_busy_pct`, `render_busy_pct` | int | Engine busy percentages over the sampling interval. | | `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. Present only with an enrichment source — absent is not zero, and must not be rendered as an idle GPU. | | `vram_used_mb`, `vram_total_mb` | int | GPU memory, on the same terms as `total_busy_pct`. | @@ -230,15 +230,17 @@ long before anyone reads a log. Semantics worth knowing: - Setting it is a comparison; clearing it is not. The note appears when a refetch - loses something, and stays until a refetch produces a report that probed at - least one backend and every backend it probed passed. A refetch that finds + loses something, and clearing takes positive evidence of recovery: a report + that probed at least one backend, had every backend it probed pass, *and* + regained something the stored report lacked — a backend that now verifies and + did not, or a device identity that is present and was not. A refetch that finds nothing *newly* lost leaves it alone, because a delta against an already-degraded report always finds nothing — a reboot moves `boot_id`, a reworded FFmpeg failure moves the probe reason, and either would otherwise - erase a standing regression and report a broken node as repaired. A report - that probed nothing at all does not clear it either: a GPU that disappeared - completely leaves no candidate backend to fail, and the absence of anything to - probe is not evidence of recovery. + erase a standing regression and report a broken node as repaired. Two cases + make the "regained" half necessary rather than pedantic: a GPU that disappeared + completely leaves no candidate backend to fail, and a multi-GPU node that lost + one card keeps probing the survivor perfectly cleanly. - A backend reported as `skipped` neither sets the note nor holds it open. Skipping means no probe ran because the node cannot open the backend's configured devices, which is a statement about access rather than about diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index 92d229803..b06b24e11 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -84,10 +84,12 @@ spread the work at all. **Drift**, an amber badge, means a capability refetch found the node's hardware got *worse* than the report it replaced: a backend that used to pass its probe -now fails, or a render device is gone. Hover for the note. The badge stays until -a refetch produces a report whose probes all pass — it is not erased by the next +now fails, or a render device is gone. Hover for the note. The badge stays until a +refetch both probes cleanly *and* regains something the stored report lacked — a +backend that verifies again, or a device that is back. It is not erased by a refetch that merely loses nothing further, so a reboot or a reworded FFmpeg error -cannot make a standing regression look repaired. Re-probing the node is the +cannot make a standing regression look repaired, and on a multi-GPU node the +surviving card probing cleanly does not speak for the one that went away. Re-probing the node is the direct way to ask whether it is still true. It is a warning, not a routing input — nothing in node selection reads it. diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index c21bc237e..664fa7314 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -564,6 +564,11 @@ type capabilityDrift struct { // lostDevices are render devices present in the previous report and absent // from this one. lostDevices []string + // regained reports that this refetch found hardware the stored report did + // not have: a backend that now verifies and did not, or a device identity + // that is present and was not. It is the only evidence that a standing + // regression actually recovered — see resolveDriftNote. + regained bool // previousResolved and resolved are the backend each report resolved to; // carried for the log line, which is where an operator reads the effect. previousResolved string @@ -653,6 +658,15 @@ func resolveDriftNote(stored *string, drift capabilityDrift, parsed bool, payloa // Nothing new was lost, but this report is not evidence of recovery. return stored } + if !drift.regained { + // Every probe that ran passed — but on a multi-GPU node the surviving + // card passes just as cleanly with its sibling still missing, and once + // the degraded report is stored the delta finds nothing lost forever + // after. A clean sweep of what remains is not evidence that what went + // away came back; only hardware appearing that the stored report lacked + // is. + return stored + } return nil } @@ -725,6 +739,19 @@ func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, pars drift.lostBackends = append(drift.lostBackends, backend.Backend) } drift.lostDevices = lostRenderDevices(previous, current) + // The mirror comparison. Once a degraded report is stored, every later delta + // is degraded-to-degraded and finds nothing lost; growth is what separates + // "still broken" from "came back". + drift.regained = len(lostRenderDevices(current, previous)) > 0 + verifiedBefore := make(map[string]bool, len(previous.DetectedBackends)) + for _, backend := range previous.DetectedBackends { + verifiedBefore[backend.Backend] = backend.Verified + } + for _, backend := range current.DetectedBackends { + if backend.Verified && !verifiedBefore[backend.Backend] { + drift.regained = true + } + } return drift, true } diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 38446e15b..014a03107 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -162,17 +162,51 @@ func TestResolveDriftNoteKeepsNoteWhenNoProbePassed(t *testing.T) { } } -// The complement: a report with a backend that actually passed its probe is the +// The complement: a report that gains back what the stored one lacked is the // evidence recovery needs, and clears the note. -func TestResolveDriftNoteClearsOnAPassingProbe(t *testing.T) { - const recoveredPayload = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + +func TestResolveDriftNoteClearsWhenHardwareComesBack(t *testing.T) { + const degraded = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"vaapi","verified":false,"reason":"no such device"}]}` + const recovered = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + `"detected_backends":[{"backend":"vaapi","verified":true},` + `{"backend":"qsv","verified":false,"skipped":true}]}` + standing := "verified hardware backends lost: vaapi" - payload := []byte(recoveredPayload) + drift, parsed := computeCapabilityDrift([]byte(degraded), []byte(recovered)) + if !drift.regained { + t.Fatal("a report that regained a verified backend and a device was not seen as recovery") + } + if got := resolveDriftNote(&standing, drift, parsed, []byte(recovered)); got != nil { + t.Fatalf("capability_drift = %q, want recovered hardware to clear it", *got) + } +} + +// A multi-GPU node that lost one card keeps probing the survivor cleanly, and +// once the degraded report is stored the delta finds nothing lost ever again. +// Clearing on that generic success told an operator the node recovered while +// one of its cards was still missing. +func TestResolveDriftNoteKeepsNoteWhileASiblingGPUIsStillMissing(t *testing.T) { + // Two cards before the loss, one after — and the survivor verifies, so + // every probe that ran passes. + const degraded = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD129" + payload := []byte(degraded) + // The next refetch is degraded-to-degraded: nothing newly lost, nothing + // regained, every probe clean. drift, parsed := computeCapabilityDrift(payload, payload) - if got := resolveDriftNote(&standing, drift, parsed, payload); got != nil { - t.Fatalf("capability_drift = %q, want a verified backend to clear it", *got) + if !hardwareProbesClean(payload) { + t.Fatal("the surviving card should probe cleanly; that is the point") + } + got := resolveDriftNote(&standing, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared while the lost card was still missing") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) } } diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index f6ddcaf16..74806747c 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -102,18 +102,6 @@ func TestAcquireHWDeviceUsesTheVerifiedRenderDevice(t *testing.T) { } } -// With nothing verified — a backend named explicitly and never walked — the -// device stays unresolved and ffmpeg picks one downstream, exactly as before. -func TestAcquireHWDeviceLeavesTheDeviceUnsetWithoutAVerifiedProbe(t *testing.T) { - setupHWAccelTest(t) - - device, workload, release := acquireHWDevice("", transcodeHWQSV, "") - defer release() - if device != "" || workload != "" { - t.Fatalf("acquireHWDevice() = (%q, %q), want both empty with no verified device", device, workload) - } -} - // Invalidation has to supersede a probe already in flight, not merely clear the // map in front of it. The operator-facing re-probe exists to force a cold // re-verification; if a probe that started before the invalidation could hand @@ -232,3 +220,34 @@ func TestNVENCConfiguredDeviceIsProbedNotSkipped(t *testing.T) { } t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) } + +// An explicitly configured backend short-circuits resolution, so the detection +// walk never runs and nothing is ever recorded as verified. Without a fallback +// the workload went uncounted and the node reported zero GPU sessions while it +// transcoded — the same reporting hole the auto path had, on the branch that +// never probes. +func TestAcquireHWDeviceCountsTheAutoDetectedDeviceWithoutAProbe(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + + // No walk has run, so nothing is verified — exactly the state a host with + // hw_accel=qsv and no hw_device is in. + if got := VerifiedHWDevice(transcodeHWQSV); got != "" { + t.Fatalf("VerifiedHWDevice = %q, want nothing verified for this test", got) + } + + device, release := AcquireHWDevice("", transcodeHWQSV) + defer release() + + want := env.devicePath("renderD128") + if device != want { + t.Fatalf("AcquireHWDevice() = %q, want the device execution will pick, %q", device, want) + } + if got := hwDeviceActiveCount(want); got != 1 { + t.Fatalf("active workloads on %s = %d, want the transcode counted", want, got) + } + release() + if got := hwDeviceActiveCount(want); got != 0 { + t.Fatalf("active workloads after release = %d, want 0", got) + } +} diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index 6a1b87ebf..0e1ec8e68 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -269,6 +269,16 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, w if verified := VerifiedHWDevice(resolvedHWAccel); verified != "" { return verified, verified, countHWDeviceWorkload(verified) } + // Nothing verified. That is not only the cold-process case: an + // explicitly configured backend short-circuits resolution, so a host + // running hw_accel=qsv with no hw_device never walks its hardware at all + // and would otherwise never name a device here — reporting zero GPU + // sessions for every transcode it runs. Fall back to the device + // execution is about to pick anyway, which is this same first render + // node; appendHWAccelArgs resolves an empty value the same way. + if detected := detectRenderDevice(defaultDRIDir); detected != "" { + return detected, detected, countHWDeviceWorkload(detected) + } return "", "", noop } // Select and reserve in one critical section so concurrent workload starts diff --git a/internal/playback/hwdevice_test.go b/internal/playback/hwdevice_test.go index 27394b210..321c93e58 100644 --- a/internal/playback/hwdevice_test.go +++ b/internal/playback/hwdevice_test.go @@ -58,12 +58,20 @@ func TestParseHWDeviceSet(t *testing.T) { } } -func TestAcquireHWDeviceEmptyValueStaysEmpty(t *testing.T) { +// An empty setting resolves to nothing only when there is nothing to resolve to. +// Pointed at an empty device directory for the same reason as the counting test +// below: with a real /dev/dri present this now returns the device execution +// would pick, which is the point of resolving it here. +func TestAcquireHWDeviceEmptyValueStaysEmptyWithoutDevices(t *testing.T) { resetDeviceLoad(t) + original := defaultDRIDir + defaultDRIDir = t.TempDir() + t.Cleanup(func() { defaultDRIDir = original }) + device, release := AcquireHWDevice("", "qsv") defer release() if device != "" { - t.Fatalf("device = %q, want empty so auto-detection applies", device) + t.Fatalf("device = %q, want empty so auto-detection applies downstream", device) } } @@ -102,11 +110,17 @@ func TestAcquireHWDeviceSingleRenderDeviceIsCounted(t *testing.T) { } } -// With no device configured there is no name to count against: ffmpeg picks the -// device downstream, and inventing a key would report sessions on a device the -// sampler never names. +// With no render device to name, the workload stays uncounted rather than being +// attributed to a device that does not exist. The device directory is pointed at +// an empty temp dir rather than left at the real /dev/dri: unconfigured +// acquisition now falls back to the device execution would pick, so on a host +// that actually has a GPU this would otherwise assert the opposite of the truth. func TestAcquireHWDeviceUnconfiguredRenderDeviceIsNotCounted(t *testing.T) { resetDeviceLoad(t) + original := defaultDRIDir + defaultDRIDir = t.TempDir() + t.Cleanup(func() { defaultDRIDir = original }) + _, release := AcquireHWDevice("", "vaapi") defer release() if got := HWDeviceLoadSnapshot(); len(got) != 0 { From 0273ed7a18b3ff42e76f0467b46d59aeab90cb38 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:36:52 -0400 Subject: [PATCH 029/163] fix(nodepool): clear drift only when the hardware it recorded comes back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third correction to this same rule, and the previous two were wrong in the same way: they tried to infer recovery from a delta. They cannot. Once a degraded report is stored, every later comparison is degraded-to-degraded and finds nothing, so each fix reached for a different proxy for "better than before" — first any passing probe, then any growth in the inventory — and each proxy had a counterexample. A surviving sibling probes perfectly cleanly with its partner still missing; an unrelated GPU added later grows the inventory without repairing anything. The note now records what it is standing for. A new nullable stream_nodes.capability_drift_baseline holds the backends that must verify again and the alias sets of the devices that must reappear, written in the same statement as the note so it always describes it. Clearing checks the current report against that, not against the previous one. Devices are kept as every identity they answered to, so a card returning renumbered — or on a pass where nvidia-smi did not answer — still matches. Successive losses accumulate: two cards going one at a time must both return. A note carried over from before the column existed has nothing recorded to wait for, and a clean report clears it, so an upgrade does not strand one. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 27 +-- docs/wiki/admin/monitoring-nodes.md | 14 +- internal/nodepool/gpuidentity_test.go | 6 +- internal/nodepool/health.go | 167 ++++++++++++++---- internal/nodepool/health_drift_test.go | 119 ++++++++++++- internal/nodepool/health_stats_test.go | 2 +- internal/nodepool/proxy_pool.go | 4 +- internal/nodepool/repository.go | 22 ++- .../nodepool/repository_capabilities_test.go | 12 +- internal/nodepool/transcode_pool.go | 11 +- ...7192932_node_capability_drift_baseline.sql | 27 +++ 11 files changed, 325 insertions(+), 86 deletions(-) create mode 100644 migrations/sql/20260827192932_node_capability_drift_baseline.sql diff --git a/docs/admin-api.md b/docs/admin-api.md index ddc074bc0..f8990052f 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -33,6 +33,7 @@ Always `200 OK` with a JSON array. | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | | `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | | `capability_drift` | string | Human-readable note describing how the node's hardware got worse at the last capability refetch. Omitted when the last refetch found no regression (see below). | +| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": [...], "devices": [[alias, ...], ...]}`. Present with `capability_drift`, absent without it. Each device is every stable name it answered to, so it is recognized if it returns renumbered. | ### Acceleration overrides @@ -230,17 +231,21 @@ long before anyone reads a log. Semantics worth knowing: - Setting it is a comparison; clearing it is not. The note appears when a refetch - loses something, and clearing takes positive evidence of recovery: a report - that probed at least one backend, had every backend it probed pass, *and* - regained something the stored report lacked — a backend that now verifies and - did not, or a device identity that is present and was not. A refetch that finds - nothing *newly* lost leaves it alone, because a delta against an - already-degraded report always finds nothing — a reboot moves `boot_id`, a - reworded FFmpeg failure moves the probe reason, and either would otherwise - erase a standing regression and report a broken node as repaired. Two cases - make the "regained" half necessary rather than pedantic: a GPU that disappeared - completely leaves no candidate backend to fail, and a multi-GPU node that lost - one card keeps probing the survivor perfectly cleanly. + loses something, and it records what it lost in `capability_drift_baseline`. + Clearing requires that specific hardware back: every backend in the baseline + verifying again, and every device in it answering to one of its recorded + aliases. A refetch that finds nothing *newly* lost leaves the note alone, + because a delta against an already-degraded report always finds nothing — a + reboot moves `boot_id`, a reworded FFmpeg failure moves the probe reason, and + either would otherwise erase a standing regression. Three cases make the + baseline necessary rather than pedantic, and none of them are caught by + looking at the current report alone: a GPU that disappeared completely leaves + no candidate backend to fail; a multi-GPU node that lost one card keeps + probing the survivor perfectly cleanly; and adding an unrelated GPU grows the + inventory without repairing anything. Successive losses accumulate, so two + cards going one at a time must both return. +- A note carried over from before the baseline existed has nothing recorded to + wait for, and a clean report clears it. - A backend reported as `skipped` neither sets the note nor holds it open. Skipping means no probe ran because the node cannot open the backend's configured devices, which is a statement about access rather than about diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index b06b24e11..cbc1436df 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -84,12 +84,14 @@ spread the work at all. **Drift**, an amber badge, means a capability refetch found the node's hardware got *worse* than the report it replaced: a backend that used to pass its probe -now fails, or a render device is gone. Hover for the note. The badge stays until a -refetch both probes cleanly *and* regains something the stored report lacked — a -backend that verifies again, or a device that is back. It is not erased by a -refetch that merely loses nothing further, so a reboot or a reworded FFmpeg error -cannot make a standing regression look repaired, and on a multi-GPU node the -surviving card probing cleanly does not speak for the one that went away. Re-probing the node is the +now fails, or a render device is gone. Hover for the note. The badge records what it +lost and stays until exactly that comes back — the backend verifying again, the +card answering to one of the identities it had. It is not erased by a refetch +that merely loses nothing further, so a reboot or a reworded FFmpeg error cannot +make a standing regression look repaired; nor by the surviving card on a +multi-GPU node probing cleanly; nor by an unrelated GPU being added, which grows +the inventory without repairing anything. Cards lost one at a time must all +return. Re-probing the node is the direct way to ask whether it is still true. It is a warning, not a routing input — nothing in node selection reads it. diff --git a/internal/nodepool/gpuidentity_test.go b/internal/nodepool/gpuidentity_test.go index c079ab4bc..0933d2416 100644 --- a/internal/nodepool/gpuidentity_test.go +++ b/internal/nodepool/gpuidentity_test.go @@ -113,7 +113,7 @@ func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { transcodes := NewTranscodePool() transcodes.SetNodes([]*Node{{ID: 1, URL: "http://tc-1", Enabled: true, Healthy: true}}) - transcodes.ApplyCapabilities(1, "http://tc-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil) + transcodes.ApplyCapabilities(1, "http://tc-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil, nil) if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { t.Fatalf("transcode ApplyCapabilities derived %v, want [GPU-aaa]", got) } @@ -121,14 +121,14 @@ func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { // The card was passed through to another host: the node now reports a // device it cannot identify, and must stop claiming the old key. transcodes.ApplyCapabilities(1, "http://tc-1", []byte(`{"boot_id":"boot-2","render_device_details":[{"path":"/dev/dri/renderD128"}]}`), - "sha256:bbb", refreshedAt, nil) + "sha256:bbb", refreshedAt, nil, nil) if got := transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { t.Fatalf("stale identities survived a new report: %v", got) } proxies := NewProxyPool() proxies.SetNodes([]*Node{{ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true}}) - proxies.ApplyCapabilities(2, "http://proxy-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil) + proxies.ApplyCapabilities(2, "http://proxy-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil, nil) if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { t.Fatalf("proxy ApplyCapabilities derived %v, want [GPU-aaa]", got) } diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 664fa7314..90cf7036b 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -240,7 +240,7 @@ func (hc *HealthChecker) Start(ctx context.Context) { type applyHealthFunc func(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) // applyCapabilitiesFunc is a pool's copy-on-write capability writer. -type applyCapabilitiesFunc func(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string) +type applyCapabilitiesFunc func(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) func (hc *HealthChecker) checkAll(ctx context.Context) { var wg sync.WaitGroup @@ -403,13 +403,13 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply // this one replaces, and stored with it so a reader never sees a note // describing a different payload. drift, parsed := computeCapabilityDrift(n.Capabilities, payload) - note := resolveDriftNote(n.CapabilityDrift, drift, parsed, payload) + note, driftBaseline := resolveDriftNote(n.CapabilityDrift, n.CapabilityDriftBaseline, drift, parsed, payload) refreshedAt := time.Now() if hc.repo != nil { // Fenced on the URL this payload was fetched from: the fetch is // detached and bounded at two minutes, so the row may since have been // repointed at a different worker. - if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note); err != nil { + if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline); err != nil { if errors.Is(err, ErrNodeMoved) { // Not a failure to report loudly: the node was edited or // removed while this fetch ran, and the next sweep will fetch @@ -425,7 +425,7 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply } logCapabilityChange(ctx, n, drift, parsed) if applyCapabilities != nil { - applyCapabilities(n.ID, n.URL, payload, hash, refreshedAt, note) + applyCapabilities(n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline) } if onChanged != nil { onChanged(n.URL) @@ -532,15 +532,28 @@ func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { return devices } -// lostRenderDevices names the devices in previous that nothing in current -// answers to. -func lostRenderDevices(previous, current capabilityDriftView) []string { +// lostRenderDeviceEntries returns the devices in previous that nothing in +// current answers to, with their full alias sets: the note displays the path, +// while the drift baseline keeps every identity so the device can be recognized +// when it comes back under a different one. +func lostRenderDeviceEntries(previous, current capabilityDriftView) []renderDeviceAliases { currentDevices := renderDeviceAliasSets(current) - var lost []string + var lost []renderDeviceAliases for _, device := range renderDeviceAliasSets(previous) { if slices.ContainsFunc(currentDevices, device.sameDevice) { continue } + lost = append(lost, device) + } + return lost +} + +// lostRenderDevices names the devices in previous that nothing in current +// answers to. +func lostRenderDevices(previous, current capabilityDriftView) []string { + entries := lostRenderDeviceEntries(previous, current) + lost := make([]string, 0, len(entries)) + for _, device := range entries { name := device.path if name == "" { name = device.aliases[0] @@ -564,11 +577,9 @@ type capabilityDrift struct { // lostDevices are render devices present in the previous report and absent // from this one. lostDevices []string - // regained reports that this refetch found hardware the stored report did - // not have: a backend that now verifies and did not, or a device identity - // that is present and was not. It is the only evidence that a standing - // regression actually recovered — see resolveDriftNote. - regained bool + // lostDeviceAliases carries every identity each lost device answered to, so + // the drift baseline can recognize it if it comes back under another one. + lostDeviceAliases []renderDeviceAliases // previousResolved and resolved are the backend each report resolved to; // carried for the log line, which is where an operator reads the effect. previousResolved string @@ -647,27 +658,117 @@ func truncateDriftNote(note string) string { // backend is still failing its probe, which is the one reading this column must // never produce. So the note is latched, and only a report whose probes all pass // clears it. -func resolveDriftNote(stored *string, drift capabilityDrift, parsed bool, payload []byte) *string { +func resolveDriftNote(stored *string, storedBaseline []byte, drift capabilityDrift, parsed bool, payload []byte) (*string, []byte) { + outstanding := mergeDriftBaseline(storedBaseline, drift) if note := drift.persistedNote(); note != nil { - return note + // A fresh loss extends whatever was already outstanding rather than + // replacing it: two GPUs going one at a time must both have to return. + return note, marshalDriftBaseline(outstanding) } if stored == nil || strings.TrimSpace(*stored) == "" { - return nil + return nil, nil } if !parsed || !hardwareProbesClean(payload) { // Nothing new was lost, but this report is not evidence of recovery. - return stored + return stored, marshalDriftBaseline(outstanding) + } + if !outstanding.recoveredBy(payload) { + // Recovery is the *originally lost* hardware coming back, not the + // inventory merely looking healthy. A surviving sibling probes just as + // cleanly with its partner still missing, and an unrelated GPU added + // later is growth without repair. Only the baseline can tell those from + // a genuine return, which is why it is kept rather than re-derived: once + // the degraded report is stored, every later comparison is + // degraded-to-degraded and finds nothing at all. + return stored, marshalDriftBaseline(outstanding) + } + return nil, nil +} + +// driftBaseline is the hardware a standing capability_drift note is waiting on. +type driftBaseline struct { + // Backends must verify again. + Backends []string `json:"backends,omitempty"` + // Devices are alias sets: any one member reappearing identifies the card, + // so a renumbered render node or a pass without nvidia-smi still matches. + Devices [][]string `json:"devices,omitempty"` +} + +func (b driftBaseline) empty() bool { return len(b.Backends) == 0 && len(b.Devices) == 0 } + +// recoveredBy reports whether every backend and device the note is waiting on is +// accounted for in this report. +func (b driftBaseline) recoveredBy(payload []byte) bool { + if b.empty() { + // Nothing recorded to wait for — a note written before baselines + // existed. A clean report is the best evidence available, and holding + // such a note forever would strand it. + return true + } + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false } - if !drift.regained { - // Every probe that ran passed — but on a multi-GPU node the surviving - // card passes just as cleanly with its sibling still missing, and once - // the degraded report is stored the delta finds nothing lost forever - // after. A clean sweep of what remains is not evidence that what went - // away came back; only hardware appearing that the stored report lacked - // is. - return stored + verified := make(map[string]bool, len(current.DetectedBackends)) + for _, backend := range current.DetectedBackends { + verified[backend.Backend] = backend.Verified } - return nil + for _, backend := range b.Backends { + if !verified[backend] { + return false + } + } + present := make(map[string]bool) + for _, device := range renderDeviceAliasSets(current) { + for _, alias := range device.aliases { + present[alias] = true + } + } + for _, aliases := range b.Devices { + if !slices.ContainsFunc(aliases, func(alias string) bool { return present[alias] }) { + return false + } + } + return true +} + +// mergeDriftBaseline adds this refetch's losses to whatever the note was already +// waiting on. +func mergeDriftBaseline(stored []byte, drift capabilityDrift) driftBaseline { + var baseline driftBaseline + if len(stored) > 0 { + // An unreadable baseline is treated as absent rather than as a reason to + // discard the losses this pass found. + _ = json.Unmarshal(stored, &baseline) + } + for _, backend := range drift.lostBackends { + if !slices.Contains(baseline.Backends, backend) { + baseline.Backends = append(baseline.Backends, backend) + } + } + for _, device := range drift.lostDeviceAliases { + if slices.ContainsFunc(baseline.Devices, func(existing []string) bool { + return slices.ContainsFunc(existing, func(alias string) bool { + return slices.Contains(device.aliases, alias) + }) + }) { + continue + } + baseline.Devices = append(baseline.Devices, slices.Clone(device.aliases)) + } + slices.Sort(baseline.Backends) + return baseline +} + +func marshalDriftBaseline(baseline driftBaseline) []byte { + if baseline.empty() { + return nil + } + encoded, err := json.Marshal(baseline) + if err != nil { + return nil + } + return encoded } // hardwareProbesClean reports whether the node probed at least one backend and @@ -739,19 +840,7 @@ func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, pars drift.lostBackends = append(drift.lostBackends, backend.Backend) } drift.lostDevices = lostRenderDevices(previous, current) - // The mirror comparison. Once a degraded report is stored, every later delta - // is degraded-to-degraded and finds nothing lost; growth is what separates - // "still broken" from "came back". - drift.regained = len(lostRenderDevices(current, previous)) > 0 - verifiedBefore := make(map[string]bool, len(previous.DetectedBackends)) - for _, backend := range previous.DetectedBackends { - verifiedBefore[backend.Backend] = backend.Verified - } - for _, backend := range current.DetectedBackends { - if backend.Verified && !verifiedBefore[backend.Backend] { - drift.regained = true - } - } + drift.lostDeviceAliases = lostRenderDeviceEntries(previous, current) return drift, true } diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 014a03107..92e30c6cf 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -151,7 +151,7 @@ func TestResolveDriftNoteKeepsNoteWhenNoProbePassed(t *testing.T) { // Both sides degraded: the delta finds nothing newly lost, which is // exactly the state in which clearing has to be refused. drift, parsed := computeCapabilityDrift(payload, payload) - got := resolveDriftNote(&standing, drift, parsed, payload) + got, _ := resolveDriftNote(&standing, nil, drift, parsed, payload) if got == nil { t.Fatal("capability_drift was cleared by a report in which no probe passed") } @@ -173,12 +173,94 @@ func TestResolveDriftNoteClearsWhenHardwareComesBack(t *testing.T) { `{"backend":"qsv","verified":false,"skipped":true}]}` standing := "verified hardware backends lost: vaapi" + // The baseline the loss recorded: vaapi has to verify again. + baseline := []byte(`{"backends":["vaapi"]}`) drift, parsed := computeCapabilityDrift([]byte(degraded), []byte(recovered)) - if !drift.regained { - t.Fatal("a report that regained a verified backend and a device was not seen as recovery") + got, gotBaseline := resolveDriftNote(&standing, baseline, drift, parsed, []byte(recovered)) + if got != nil { + t.Fatalf("capability_drift = %q, want the recovered backend to clear it", *got) } - if got := resolveDriftNote(&standing, drift, parsed, []byte(recovered)); got != nil { - t.Fatalf("capability_drift = %q, want recovered hardware to clear it", *got) + if gotBaseline != nil { + t.Fatalf("baseline = %s, want it cleared with the note", gotBaseline) + } +} + +// Growth is not repair. A node standing on a lost GPU that gains an unrelated +// one has a bigger, perfectly clean inventory and still has not got its card +// back — which is why the note keeps what it is waiting for rather than +// re-deriving it from the stored report. +func TestResolveDriftNoteKeepsNoteWhenAnUnrelatedGPUIsAdded(t *testing.T) { + const gained = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD130","pci_address":"0000:09:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + // The lost card, by every identity it answered to. + baseline := []byte(`{"devices":[["0000:03:00.0","/dev/dri/renderD128"]]}`) + payload := []byte(gained) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, gotBaseline := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared because an unrelated GPU appeared") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } + if len(gotBaseline) == 0 { + t.Fatal("the baseline was dropped while the note still stands") + } +} + +// The lost card coming back under a renumbered render node still counts: the +// baseline keeps every identity it answered to, and any one of them matching +// identifies it. +func TestResolveDriftNoteClearsWhenTheLostCardReturnsRenumbered(t *testing.T) { + const back = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[["0000:03:00.0","/dev/dri/renderD128"]]}`) + payload := []byte(back) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want the card at the same slot to clear it", *got) + } +} + +// Two cards going one at a time must both have to return: the second loss +// extends the baseline rather than replacing it. +func TestResolveDriftNoteAccumulatesSuccessiveLosses(t *testing.T) { + const twoCards = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128","/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"},` + + `{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + const oneCard = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + const noCards = `{"resolved":"none","render_devices":[],"detected_backends":[]}` + + firstLoss, parsed := computeCapabilityDrift([]byte(twoCards), []byte(oneCard)) + note, baseline := resolveDriftNote(nil, nil, firstLoss, parsed, []byte(oneCard)) + if note == nil || len(baseline) == 0 { + t.Fatalf("first loss produced note=%v baseline=%s", note, baseline) + } + + secondLoss, parsed := computeCapabilityDrift([]byte(oneCard), []byte(noCards)) + note, baseline = resolveDriftNote(note, baseline, secondLoss, parsed, []byte(noCards)) + if note == nil || len(baseline) == 0 { + t.Fatal("second loss dropped the standing note or its baseline") + } + + // Only the first card returns; the note must stand for the second. + if got, _ := resolveDriftNote(note, baseline, capabilityDrift{}, true, []byte(oneCard)); got == nil { + t.Fatal("capability_drift cleared with one of two lost cards still missing") + } + // Both back clears it. + if got, _ := resolveDriftNote(note, baseline, capabilityDrift{}, true, []byte(twoCards)); got != nil { + t.Fatalf("capability_drift = %q, want both cards returning to clear it", *got) } } @@ -194,14 +276,15 @@ func TestResolveDriftNoteKeepsNoteWhileASiblingGPUIsStillMissing(t *testing.T) { `"detected_backends":[{"backend":"vaapi","verified":true}]}` standing := "render devices gone: /dev/dri/renderD129" + baseline := []byte(`{"devices":[["0000:04:00.0","/dev/dri/renderD129"]]}`) payload := []byte(degraded) - // The next refetch is degraded-to-degraded: nothing newly lost, nothing - // regained, every probe clean. + // The next refetch is degraded-to-degraded: nothing newly lost, and every + // probe that ran passed, because the survivor is fine. drift, parsed := computeCapabilityDrift(payload, payload) if !hardwareProbesClean(payload) { t.Fatal("the surviving card should probe cleanly; that is the point") } - got := resolveDriftNote(&standing, drift, parsed, payload) + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) if got == nil { t.Fatal("capability_drift cleared while the lost card was still missing") } @@ -479,7 +562,7 @@ func TestComputeCapabilityDriftDoesNotTreatASkippedBackendAsLost(t *testing.T) { // And the pair round-trips: what does not set the note must not be held // open by it either. standing := "verified hardware backends lost: qsv" - if got := resolveDriftNote(&standing, drift, parsed, []byte(after)); got == nil || *got != standing { + if got, _ := resolveDriftNote(&standing, nil, drift, parsed, []byte(after)); got == nil || *got != standing { t.Fatalf("resolveDriftNote = %v, want a skipped report to leave a standing note alone", got) } } @@ -516,3 +599,21 @@ func TestComputeCapabilityDriftCatchesABackendThatStoppedBeingReported(t *testin t.Fatalf("lostBackends = %v, want the vanished backend reported", drift.lostBackends) } } + +// A note written before the baseline column existed has nothing recorded to wait +// for. Holding it forever would strand it on an upgraded deployment, so a clean +// report clears it — the best evidence available for a note whose subject was +// never captured. +func TestResolveDriftNoteClearsALegacyNoteWithNoBaseline(t *testing.T) { + const clean = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "verified hardware backends lost: vaapi" + payload := []byte(clean) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, nil, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want a baseline-less note cleared by a clean report", *got) + } +} diff --git a/internal/nodepool/health_stats_test.go b/internal/nodepool/health_stats_test.go index c4df7ede1..b9822c401 100644 --- a/internal/nodepool/health_stats_test.go +++ b/internal/nodepool/health_stats_test.go @@ -213,7 +213,7 @@ func TestApplyCapabilitiesIgnoresAReportForAReplacedWorker(t *testing.T) { pool.ApplyCapabilities(1, "http://original", []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`), - "sha256:stale", time.Now(), nil) + "sha256:stale", time.Now(), nil, nil) stored := pool.Nodes()[0] if len(stored.Capabilities) != 0 || stored.CapabilitiesHash != nil || len(stored.PhysicalGPUKeys) != 0 { diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index ff8680afd..6556776b7 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -71,8 +71,8 @@ func (p *ProxyPool) ApplyHealth(id int, checkedURL string, healthy bool, activeJ // ApplyCapabilities records a freshly fetched capability report by swapping the // node for an updated copy, keeping published *Node values immutable. -func (p *ProxyPool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { +func (p *ProxyPool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { p.mu.Lock() defer p.mu.Unlock() - applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift) + applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 582b456e4..dd29c0745 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -66,6 +66,13 @@ type Node struct { // it. It is written beside Capabilities in one statement, so it always // describes the report stored with it, and nothing routes on it. CapabilityDrift *string `json:"capability_drift,omitempty"` + // CapabilityDriftBaseline records, machine-readably, the backends and device + // identities CapabilityDrift is waiting on. Recovery cannot be derived from + // the stored report alone — once a degraded report is stored every later + // comparison is degraded-to-degraded — so the note keeps what it is standing + // for. Non-nil exactly when CapabilityDrift is, except on a note written + // before this column existed. + CapabilityDriftBaseline json.RawMessage `json:"capability_drift_baseline,omitempty"` // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from // Capabilities rather than stored: it is a pure function of that payload, so // a column would only be a second copy that can disagree with it. Two nodes @@ -250,13 +257,13 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override, capability_drift` +const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override, capability_drift, capability_drift_baseline` func scanNode(row pgx.Row) (*Node, error) { var n Node // jsonb is scanned as raw bytes rather than into json.RawMessage directly so // a NULL column stays nil instead of decoding through the JSON codec. - var capabilities, lastStats []byte + var capabilities, lastStats, driftBaselineBytes []byte err := row.Scan( &n.ID, &n.Name, &n.Type, &n.URL, &n.Enabled, &n.Healthy, &n.ActiveJobs, @@ -266,7 +273,7 @@ func scanNode(row pgx.Row) (*Node, error) { &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, &lastStats, &n.HWAccelOverride, &n.HWDeviceOverride, - &n.CapabilityDrift, + &n.CapabilityDrift, &driftBaselineBytes, ) if err != nil { return nil, err @@ -277,6 +284,9 @@ func scanNode(row pgx.Row) (*Node, error) { if len(lastStats) > 0 { n.LastStats = json.RawMessage(lastStats) } + if len(driftBaselineBytes) > 0 { + n.CapabilityDriftBaseline = json.RawMessage(driftBaselineBytes) + } // Derived here so every reader of a stored row — the admin listing as much // as a pool load — sees the same identities without parsing the payload // again for itself. @@ -456,11 +466,11 @@ func (r *Repository) UpdateHealth(ctx context.Context, id int, checkedURL string // shared-GPU work on that reading until another sweep corrected it. Trailing // slashes are ignored on both sides because the pools normalize URLs and the // column does not. -func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string) error { +func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) error { tag, err := r.pool.Exec(ctx, - `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4, capability_drift = $5 + `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4, capability_drift = $5, capability_drift_baseline = $7 WHERE id = $1 AND rtrim(url, '/') = rtrim($6, '/')`, - id, capabilities, hash, refreshedAt, drift, fetchedFrom) + id, capabilities, hash, refreshedAt, drift, fetchedFrom, driftBaseline) if err != nil { return fmt.Errorf("update node capabilities: %w", err) } diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index 5dbdcf0e6..aadb16dcf 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -64,7 +64,7 @@ func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { payload := json.RawMessage(`{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"]}`) refreshedAt := time.Now().UTC().Truncate(time.Millisecond) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:abc", refreshedAt, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:abc", refreshedAt, nil, nil); err != nil { t.Fatalf("update capabilities: %v", err) } @@ -113,7 +113,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { note := "verified hardware backends lost: nvenc; resolved backend nvenc -> none" payload := json.RawMessage(`{"resolved":"none"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:degraded", time.Now(), ¬e); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:degraded", time.Now(), ¬e, nil); err != nil { t.Fatalf("update capabilities with drift: %v", err) } reloaded, err := repo.GetByID(ctx, node.ID) @@ -125,7 +125,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { } recovered := json.RawMessage(`{"resolved":"nvenc"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, recovered, "sha256:recovered", time.Now(), nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, recovered, "sha256:recovered", time.Now(), nil, nil); err != nil { t.Fatalf("update capabilities without drift: %v", err) } reloaded, err = repo.GetByID(ctx, node.ID) @@ -139,7 +139,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { func TestRepositoryUpdateCapabilitiesUnknownNode(t *testing.T) { repo := NewRepository(newNodeTestPool(t)) - err := repo.UpdateCapabilities(context.Background(), -1, "http://gone", []byte(`{}`), "sha256:abc", time.Now(), nil) + err := repo.UpdateCapabilities(context.Background(), -1, "http://gone", []byte(`{}`), "sha256:abc", time.Now(), nil, nil) if !errors.Is(err, ErrNodeMoved) { t.Fatalf("err = %v, want ErrNodeMoved", err) } @@ -168,7 +168,7 @@ func TestRepositoryUpdateCapabilitiesRefusesAfterAURLEdit(t *testing.T) { t.Fatalf("repoint node: %v", err) } - err = repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"qsv"}`), "sha256:stale", time.Now(), nil) + err = repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"qsv"}`), "sha256:stale", time.Now(), nil, nil) if !errors.Is(err, ErrNodeMoved) { t.Fatalf("err = %v, want ErrNodeMoved after the row was repointed", err) } @@ -198,7 +198,7 @@ func TestRepositoryUpdateCapabilitiesIgnoresATrailingSlash(t *testing.T) { t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) normalized := strings.TrimSuffix(node.URL, "/") - if err := repo.UpdateCapabilities(ctx, node.ID, normalized, []byte(`{"resolved":"qsv"}`), "sha256:ok", time.Now(), nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, normalized, []byte(`{"resolved":"qsv"}`), "sha256:ok", time.Now(), nil, nil); err != nil { t.Fatalf("UpdateCapabilities with a normalized URL: %v", err) } } diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index cd506a887..573ef44ed 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -94,10 +94,10 @@ func (p *TranscodePool) ApplyHealth(id int, checkedURL string, healthy bool, act // ApplyCapabilities records a freshly fetched capability report by swapping the // node for an updated copy, keeping published *Node values immutable. -func (p *TranscodePool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { +func (p *TranscodePool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { p.mu.Lock() defer p.mu.Unlock() - applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift) + applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) } // sameNodeURL compares two node addresses the way the pools store them, so a @@ -145,7 +145,7 @@ func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, act // drift is set verbatim, nil included: the note describes the comparison that // produced this payload, so a node whose hardware recovered must lose the note // at the same moment it gains the clean report. -func applyNodeCapabilities(nodes []*Node, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string) { +func applyNodeCapabilities(nodes []*Node, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { for i, n := range nodes { if n.ID != id || !sameNodeURL(n.URL, fetchedFrom) { continue @@ -155,6 +155,11 @@ func applyNodeCapabilities(nodes []*Node, id int, fetchedFrom string, capabiliti clone.CapabilitiesHash = &hash clone.CapabilitiesRefreshedAt = &refreshedAt clone.CapabilityDrift = drift + if len(driftBaseline) > 0 { + clone.CapabilityDriftBaseline = append(json.RawMessage(nil), driftBaseline...) + } else { + clone.CapabilityDriftBaseline = nil + } // The GPU identities belong to the payload being replaced, so they are // re-derived rather than carried over from the previous report. applyPhysicalGPUKeys(&clone) diff --git a/migrations/sql/20260827192932_node_capability_drift_baseline.sql b/migrations/sql/20260827192932_node_capability_drift_baseline.sql new file mode 100644 index 000000000..75903aa48 --- /dev/null +++ b/migrations/sql/20260827192932_node_capability_drift_baseline.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capability_drift_baseline jsonb; + +COMMENT ON COLUMN stream_nodes.capability_drift_baseline IS + 'Machine-readable record of what capability_drift is standing for: ' + '{"backends": ["qsv"], "devices": [["GPU-uuid", "0000:03:00.0", ' + '"/dev/dri/renderD128"]]} — the backends that must verify again and the ' + 'device alias sets one of whose members must reappear before the note is ' + 'cleared. Each device is stored as every stable name it answered to, so a ' + 'card that comes back renumbered, or whose nvidia-smi uuid is missing on ' + 'the pass that finds it, still matches. NULL exactly when capability_drift ' + 'is NULL. It exists because recovery cannot be derived from the stored ' + 'report alone: once a degraded report is stored, every later comparison is ' + 'degraded-to-degraded and finds nothing lost, and mere growth in the ' + 'inventory is not recovery either — adding an unrelated GPU to a node that ' + 'lost one is not the lost one returning. Written by the node health sweep ' + 'in the same statement as capabilities and capability_drift, so it always ' + 'describes the note beside it. Not a routing input.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capability_drift_baseline; +-- +goose StatementEnd From f3c5c4f017c89f92ce7a3f739bcbf43dc1c6262c Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:46:46 -0400 Subject: [PATCH 030/163] test(app): make the cache-isolation test exercise the cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test named for cache isolation built a second provider whose cache was never populated, so it only asserted that an empty cache returns nothing — it could not have failed if the clone were removed. It now mutates what a successful read returned, forces the query to fail, and checks the fallback still yields the original set; then mutates that fallback and checks again, covering both slices a caller can get its hands on. Also: the admin API table claimed capability_drift_baseline is present whenever capability_drift is, which the same document contradicts three paragraphs later for notes predating the column. Found by review of #794 (CodeRabbit). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main_test.go | 27 +++++++++++++++++++-------- docs/admin-api.md | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index 4f4c93e74..54c76c7ab 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -348,18 +348,29 @@ func TestCachedLibraryPathsCachesADeliberateEmptyResult(t *testing.T) { } } -// The caller must not be able to mutate the cache through the slice it is given. +// The cache is what a failed read falls back to, so a caller scribbling on the +// slice it was handed must not be able to corrupt it — in either direction: the +// value returned from a successful read, or the one returned from the fallback +// itself. func TestCachedLibraryPathsDoesNotShareItsCachedSlice(t *testing.T) { + failing := false provider := cachedLibraryPaths(func(context.Context) ([]string, error) { + if failing { + return nil, errors.New("database is not answering") + } return []string{"/mnt/movies"}, nil }) - first := provider(context.Background()) - first[0] = "/tmp/clobbered" - failing := cachedLibraryPaths(func(context.Context) ([]string, error) { - return nil, errors.New("boom") - }) - if got := failing(context.Background()); len(got) != 0 { - t.Fatalf("an empty cache returned %v", got) + provider(context.Background())[0] = "/tmp/clobbered" + + failing = true + fallback := provider(context.Background()) + if !slices.Equal(fallback, []string{"/mnt/movies"}) { + t.Fatalf("fallback = %v, want the cache untouched by the caller's mutation", fallback) + } + + fallback[0] = "/tmp/clobbered-again" + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies"}) { + t.Fatalf("fallback = %v, want the cache untouched by a mutation of an earlier fallback", got) } } diff --git a/docs/admin-api.md b/docs/admin-api.md index f8990052f..8d4a2a1b2 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -33,7 +33,7 @@ Always `200 OK` with a JSON array. | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | | `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | | `capability_drift` | string | Human-readable note describing how the node's hardware got worse at the last capability refetch. Omitted when the last refetch found no regression (see below). | -| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": [...], "devices": [[alias, ...], ...]}`. Present with `capability_drift`, absent without it. Each device is every stable name it answered to, so it is recognized if it returns renumbered. | +| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": [...], "devices": [[alias, ...], ...]}`. Never present without `capability_drift`; absent with it only for a note written before this field existed (see below). Each device is every stable name it answered to, so it is recognized if it returns renumbered. | ### Acceleration overrides From 157c1f37cb4e5fa5ef735a927eaeda9b17b3e3f2 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:58:55 -0400 Subject: [PATCH 031/163] fix(nodes): stop node identity and policy changes reading as hardware loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings that share one shape: state derived from a node's hardware outliving a change to which node, or which hardware, it describes. Drift treated a backend absent from the report as lost. Detection only probes the backends the configured hw_device gives it candidates for, so repointing a node from a QSV render path to an NVENC index legitimately stops QSV being reported — and that latched a warning demanding QSV verify again on a node deliberately configured for NVENC, which nothing could satisfy. Now only a backend that was probed and *failed* counts. Hardware actually disappearing is caught by the render-device comparison, which reads the host's own inventory and owes nothing to the configuration. Repointing a node's url kept the old worker's capabilities, hash, last_stats and drift, and the caller publishes that row to the pools immediately — so the replacement could be placed using its predecessor's GPU identities and scratch reading. Those columns now clear when the address moves, leaving the row in the state a freshly registered node is in. A node whose row stops matching now keeps the overrides it last read instead of reverting to the cluster settings. On split-horizon deployments the match is by NODE_NAME, so renaming a node in the admin form leaves the worker looking for a name nothing carries while the API still dispatches that row's backend; reverting would pair the new backend with the cluster device for as long as they disagree. /admin/system/hw-accel read the playback settings captured at router construction, so after a settings change it kept probing the previous backend and devices — showing an operator a verification result for the configuration they had just replaced. It now reads through Dependencies.CurrentConfig per request. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 20 ++++- internal/api/handlers/system.go | 39 ++++++--- internal/api/router.go | 20 +++-- internal/nodeconfig/watcher.go | 20 ++++- internal/nodeconfig/watcher_overrides_test.go | 45 ++++++++++ internal/nodepool/health.go | 16 +++- internal/nodepool/health_drift_test.go | 38 +++++++-- internal/nodepool/repository.go | 22 ++++- .../nodepool/repository_capabilities_test.go | 83 +++++++++++++++++++ 9 files changed, 271 insertions(+), 32 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 8d4a2a1b2..9b45ae4f9 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -48,6 +48,13 @@ sets `none` for itself instead of forcing every node onto the lowest common denominator. A homogeneous deployment should leave both unset and configure `playback.hw_accel` once. +Repointing a node's `url` to a different machine clears the identity-bound +state on that row — `capabilities`, `capabilities_hash`, +`capabilities_refreshed_at`, `last_stats`, and the drift note with its baseline +— because all of it describes the worker the old address reached, and the pools +are reloaded from the row immediately. The replacement is treated as newly +registered until its first health check and capability fetch. + A node finds its own row by URL first: `NODE_URL` on the node is matched against `stream_nodes.url`, ignoring a trailing slash on either side. Set `NODE_URL` explicitly on every node. Without it a node guesses @@ -61,7 +68,12 @@ node's own `NODE_URL` is an internal address that never equals it. `name` carries no unique constraint, so an ambiguous match — more than one row sharing that name — identifies nothing and adopts neither row's overrides; registered names should be unique per node, and `NODE_NAME` should equal the -registered name. +registered name. A node whose row *stops* matching — renaming it in the admin +form while the worker's `NODE_NAME` still holds the old value — keeps the last +overrides it read rather than reverting to the cluster settings, since the API +goes on dispatching that row's backend and a row that has gone is not evidence +an operator cleared the override. Fix the mismatch: the node adopts whatever it +finds on its next poll. The node overlays its row onto the cluster-wide playback settings on every config reload, so the override is what that node probes with, advertises in @@ -246,6 +258,12 @@ Semantics worth knowing: cards going one at a time must both return. - A note carried over from before the baseline existed has nothing recorded to wait for, and a clean report clears it. +- Only a backend that was *probed and failed* counts as lost. A backend simply + absent from the report was not asked about — detection probes the backends the + configured `hw_device` gives it candidates for — so repointing a node from a + QSV render path to an NVENC index is not a regression. Hardware actually + disappearing shows up in `render_devices`, which is the host's own inventory + and owes nothing to the configuration. - A backend reported as `skipped` neither sets the note nor holds it open. Skipping means no probe ran because the node cannot open the backend's configured devices, which is a statement about access rather than about diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index 184c18c93..15b623945 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -21,27 +21,34 @@ type resourceSampler interface { Snapshot() nodemetrics.Snapshot } +// playbackSettings reports the playback configuration a local probe should run +// under. It is a function rather than three strings because these settings hot +// reload: frozen at construction, /admin/system/hw-accel would keep probing the +// backend and devices the process started with, and the Playback settings page +// would show an operator a verification result for the configuration they just +// replaced. +type playbackSettings func() (ffmpegPath, hwAccel, hwDevice string) + // SystemHandler serves read-only system inspection endpoints. type SystemHandler struct { transcodePool *nodepool.TranscodePool jwtSecret string - ffmpegPath string - hwAccel string - hwDevice string + playback playbackSettings buildInfo buildinfo.Info resources resourceSampler } -// NewSystemHandler creates a SystemHandler. hwAccel and hwDevice are the -// configured playback settings, so a local probe verifies the same backend and -// devices this host would transcode on. -func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret, ffmpegPath, hwAccel, hwDevice string) *SystemHandler { +// NewSystemHandler creates a SystemHandler. playback supplies the current +// playback settings on each call, so a local probe verifies the backend and +// devices this host would transcode on right now. +func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string, playback playbackSettings) *SystemHandler { + if playback == nil { + playback = func() (string, string, string) { return "", "", "" } + } return &SystemHandler{ transcodePool: transcodePool, jwtSecret: jwtSecret, - ffmpegPath: ffmpegPath, - hwAccel: hwAccel, - hwDevice: hwDevice, + playback: playback, buildInfo: buildinfo.Current(), } } @@ -171,9 +178,17 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, inventory) } -// localHWAccel probes this host against its configured playback settings. +// localHWAccel probes this host against its current playback settings. +// +// A zero-value handler answers with the ffmpeg on PATH and auto-detection +// rather than panicking: tests build one directly, and the accessor is wiring +// this method should not depend on having received. func (h *SystemHandler) localHWAccel() playback.HWAccelInfo { - return playback.DetectHWAccelWithFFmpeg(h.hwAccel, h.ffmpegPath, h.hwDevice) + var ffmpegPath, hwAccel, hwDevice string + if h.playback != nil { + ffmpegPath, hwAccel, hwDevice = h.playback() + } + return playback.DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice) } // HandleBuildInfo handles GET /admin/system/build. diff --git a/internal/api/router.go b/internal/api/router.go index b51fd3049..12c28b3e7 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3191,16 +3191,22 @@ func NewRouter(deps Dependencies) chi.Router { // System inspection. { sysJWTSecret := "" - sysFFmpegPath := "" - sysHWAccel := "" - sysHWDevice := "" if deps.Config != nil { sysJWTSecret = deps.Config.Auth.JWTSecret - sysFFmpegPath = deps.Config.Playback.FFmpegPath - sysHWAccel = deps.Config.Playback.HWAccel - sysHWDevice = deps.Config.Playback.HWDevice } - systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath, sysHWAccel, sysHWDevice) + // Read per request, not captured: playback + // settings hot reload, and a probe against the + // values this process started with would show an + // operator a result for the configuration they + // just replaced. + systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, + func() (string, string, string) { + cfg := deps.CurrentConfig() + if cfg == nil { + return "", "", "" + } + return cfg.Playback.FFmpegPath, cfg.Playback.HWAccel, cfg.Playback.HWDevice + }) if deps.ResourceSampler != nil { systemHandler.SetResourceSampler(deps.ResourceSampler) } diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index 29632255b..8b16f6749 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -291,13 +291,29 @@ func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) w.mu.Lock() first := !w.missingRowLogged w.missingRowLogged = true - w.overrides, w.overridesLoaded = nodeHWOverrides{}, true + previous, hadOverrides := w.overrides, w.overridesLoaded + if !hadOverrides { + w.overrides, w.overridesLoaded = nodeHWOverrides{}, true + } w.mu.Unlock() if first { slog.InfoContext(ctx, "no stream_nodes row for this node; inheriting the cluster acceleration settings", "component", "nodeconfig", "node_url", w.bootstrap.NodeURL, "node_name", w.bootstrap.NodeName) } - return + if !hadOverrides { + return + } + // The row was there and now is not. On a split-horizon deployment the + // match is by NODE_NAME, and renaming a node through the admin form + // leaves this worker's environment pointing at a name nothing carries — + // so "no row" arrives while the API is still dispatching that row's + // overridden backend. Reverting to the cluster device here would pair + // the two wrongly for as long as the names disagree. A row that has + // gone is not evidence an operator cleared the override, so the last + // one read stands, exactly as it does when the lookup errors. + slog.WarnContext(ctx, "this node's stream_nodes row is no longer matchable; keeping the acceleration policy last read from it", + "component", "nodeconfig", "node_url", w.bootstrap.NodeURL, "node_name", w.bootstrap.NodeName) + overrides = previous default: w.mu.Lock() w.overrides, w.overridesLoaded = overrides, true diff --git a/internal/nodeconfig/watcher_overrides_test.go b/internal/nodeconfig/watcher_overrides_test.go index 8a572bc07..70901a698 100644 --- a/internal/nodeconfig/watcher_overrides_test.go +++ b/internal/nodeconfig/watcher_overrides_test.go @@ -243,3 +243,48 @@ func TestApplySettingsOverlayOutlivesBootstrapReapply(t *testing.T) { t.Fatalf("HWAccel = %q, want the node override", cfg.Playback.HWAccel) } } + +// On a split-horizon deployment the row is matched by NODE_NAME, and renaming +// the node through the admin form leaves this worker's environment pointing at a +// name nothing carries. "No row" then arrives while the API is still dispatching +// that row's overridden backend, so reverting to the cluster device here would +// pair the two wrongly for as long as the names disagree. A row that has gone is +// not evidence an operator cleared the override. +func TestApplySettingsKeepsOverrideWhenTheRowStopsMatching(t *testing.T) { + accel, device := "nvenc", "0" + found := true + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + if !found { + return nodeHWOverrides{}, false, nil + } + return nodeHWOverrides{HWAccel: &accel, HWDevice: &device}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("first apply: %v", err) + } + + found = false + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply after the rename: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "nvenc" || cfg.Playback.HWDevice != "0" { + t.Fatalf("effective policy = %q / %q, want the last override read from the row", + cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} + +// A node that never had a row is a different case: there is nothing to keep, so +// the cluster settings stand rather than an invented value. +func TestApplySettingsInheritsClusterWhenNoRowWasEverFound(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{}, false, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 90cf7036b..3f30ce9c2 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -831,10 +831,18 @@ func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, pars if !backend.Verified { continue } - // A backend missing from the new report entirely had no candidate - // hardware left to probe, which is the GPU-disappeared case and is a - // genuine loss. - if outcome, reported := now[backend.Backend]; reported && (outcome.verified || outcome.skipped) { + outcome, reported := now[backend.Backend] + // Absent is not lost. Detection only probes the backends the configured + // hw_device gives it candidates for, so a backend vanishes from the + // report both when its hardware went away *and* when an operator simply + // pointed the node at something else — moving hw_device from a QSV + // render path to an NVENC index legitimately stops QSV being reported. + // Latching drift on that demands a backend verify again that the node is + // deliberately no longer configured for, which nothing can ever satisfy. + // Hardware actually disappearing shows up in render_devices, which is + // the host's own inventory and owes nothing to the configuration, and is + // compared separately below. + if !reported || outcome.verified || outcome.skipped { continue } drift.lostBackends = append(drift.lostBackends, backend.Backend) diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 92e30c6cf..93814a037 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -584,9 +584,10 @@ func TestComputeCapabilityDriftStillCatchesAFailedBackend(t *testing.T) { } } -// A backend that vanishes from the report had no candidate hardware left to -// probe at all, which is the GPU-disappeared case and a genuine loss. -func TestComputeCapabilityDriftCatchesABackendThatStoppedBeingReported(t *testing.T) { +// A GPU that actually disappears is caught by the device comparison, which reads +// the host's own inventory and owes nothing to the configuration. The backend +// going unreported alongside it is a consequence, not separate evidence. +func TestComputeCapabilityDriftCatchesAVanishedGPUAsADeviceLoss(t *testing.T) { const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + `"detected_backends":[{"backend":"qsv","verified":true}]}` const after = `{"resolved":"none","render_devices":[],"detected_backends":[]}` @@ -595,8 +596,35 @@ func TestComputeCapabilityDriftCatchesABackendThatStoppedBeingReported(t *testin if !parsed { t.Fatal("both reports should parse") } - if len(drift.lostBackends) != 1 || drift.lostBackends[0] != "qsv" { - t.Fatalf("lostBackends = %v, want the vanished backend reported", drift.lostBackends) + if !drift.regressed() { + t.Fatalf("drift = %+v, want a vanished GPU recorded", drift) + } + if len(drift.lostDevices) != 1 || drift.lostDevices[0] != "/dev/dri/renderD128" { + t.Fatalf("lostDevices = %v, want the vanished card reported", drift.lostDevices) + } +} + +// Detection only probes the backends the configured hw_device gives it +// candidates for, so repointing a node from a QSV render path to an NVENC index +// legitimately stops QSV being reported. Treating that as a disappeared backend +// latched a warning demanding QSV verify again on a node deliberately configured +// for NVENC — which nothing could ever satisfy, so the false incident could +// never clear. +func TestComputeCapabilityDriftIgnoresABackendThePolicyStoppedProbing(t *testing.T) { + // The host's inventory is unchanged; only what was probed moved. + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if drift.regressed() { + t.Fatalf("drift = %+v, want a policy change not recorded as hardware loss", drift) } } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index dd29c0745..4d9155829 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -210,6 +210,11 @@ const ( hwAccelNone = "none" ) +// sameURL is true when an update leaves the row addressing the same worker. +// Trailing slashes are ignored on both sides because the pools normalize URLs +// and the column does not, so "http://n1/" becoming "http://n1" is not a move. +const sameURL = `rtrim(COALESCE($3, url), '/') = rtrim(url, '/')` + // normalizeGroup trims a group label and converts empty to NULL. func normalizeGroup(group string) *string { g := strings.TrimSpace(group) @@ -392,7 +397,22 @@ func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) max_jobs = CASE WHEN $7::boolean THEN $8::integer ELSE max_jobs END, max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END, hw_accel_override = CASE WHEN $11::boolean THEN $12::text ELSE hw_accel_override END, - hw_device_override = CASE WHEN $13::boolean THEN $14::text ELSE hw_device_override END + hw_device_override = CASE WHEN $13::boolean THEN $14::text ELSE hw_device_override END, + -- Everything below describes the worker the old URL addressed, so + -- repointing the row at a different machine has to drop it. The + -- caller publishes the returned row to the pools immediately, and + -- these are exactly the fields placement reads: the GPU identities + -- behind physical_gpu_keys and the scratch fill behind admission. + -- Keeping them would route work onto the replacement using its + -- predecessor's hardware until a health check and a capability + -- fetch caught up. NULL is the same state a freshly registered node + -- is in, which is the truth here. + capabilities = CASE WHEN `+sameURL+` THEN capabilities END, + capabilities_hash = CASE WHEN `+sameURL+` THEN capabilities_hash END, + capabilities_refreshed_at = CASE WHEN `+sameURL+` THEN capabilities_refreshed_at END, + last_stats = CASE WHEN `+sameURL+` THEN last_stats END, + capability_drift = CASE WHEN `+sameURL+` THEN capability_drift END, + capability_drift_baseline = CASE WHEN `+sameURL+` THEN capability_drift_baseline END WHERE id = $1 RETURNING `+nodeColumns, id, input.Name, input.URL, input.Enabled, diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index aadb16dcf..cd2e7456f 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -202,3 +202,86 @@ func TestRepositoryUpdateCapabilitiesIgnoresATrailingSlash(t *testing.T) { t.Fatalf("UpdateCapabilities with a normalized URL: %v", err) } } + +// Everything a capability report and a health sample describe belongs to the +// worker the URL addressed. Repointing the row at a different machine has to +// drop it: the caller publishes the returned row to the pools immediately, and +// these are the fields placement reads — the GPU identities behind +// physical_gpu_keys and the scratch fill behind admission. +func TestRepositoryUpdateClearsWorkerStateWhenTheURLMoves(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("repoint-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://repoint-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + note := "verified hardware backends lost: qsv" + payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:old", time.Now(), ¬e, []byte(`{"backends":["qsv"]}`)); err != nil { + t.Fatalf("store capabilities: %v", err) + } + if err := repo.UpdateHealth(ctx, node.ID, node.URL, true, 2, 0, []byte(`{"system":{"cpu_pct":41}}`)); err != nil { + t.Fatalf("store health: %v", err) + } + + moved := node.URL + "-elsewhere" + updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &moved}) + if err != nil { + t.Fatalf("repoint node: %v", err) + } + + if len(updated.Capabilities) != 0 || updated.CapabilitiesHash != nil || updated.CapabilitiesRefreshedAt != nil { + t.Fatalf("capabilities survived a repoint: %+v", updated) + } + if len(updated.LastStats) != 0 { + t.Fatalf("last_stats survived a repoint: %s", updated.LastStats) + } + if updated.CapabilityDrift != nil || len(updated.CapabilityDriftBaseline) != 0 { + t.Fatalf("drift survived a repoint: %v / %s", updated.CapabilityDrift, updated.CapabilityDriftBaseline) + } + if len(updated.PhysicalGPUKeys) != 0 { + t.Fatalf("GPU identities survived a repoint: %v", updated.PhysicalGPUKeys) + } +} + +// An edit that leaves the row on the same worker keeps its state — including +// when only a trailing slash differs, which the pools normalize away and the +// column does not. +func TestRepositoryUpdateKeepsWorkerStateWithoutAMove(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("stay-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://stay-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:keep", time.Now(), nil, nil); err != nil { + t.Fatalf("store capabilities: %v", err) + } + + renamed := "renamed-" + node.Name + if updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{Name: &renamed}); err != nil { + t.Fatalf("rename node: %v", err) + } else if len(updated.Capabilities) == 0 || updated.CapabilitiesHash == nil { + t.Fatalf("a rename dropped the worker's capabilities: %+v", updated) + } + + slashed := node.URL + "/" + if updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &slashed}); err != nil { + t.Fatalf("re-save url: %v", err) + } else if len(updated.Capabilities) == 0 { + t.Fatal("a trailing-slash change was treated as a different worker") + } +} From e4dc92485e2f9d126f709507e88587eda4e08a87 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:07:32 -0400 Subject: [PATCH 032/163] fix(nodes): re-query GPU identities on re-probe, omit unmeasured engine gauges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nvidia-smi identity listing was a sync.Once, so it was fixed for the process lifetime. An nvidia-smi that was missing or broken at first call was never asked again, leaving every NVIDIA card without its permanent id; and a card swapped into the same PCI slot kept answering to its predecessor's uuid. Both feed drift detection and shared-GPU placement, so a stale answer there is a wrong answer in both — including for the same-slot replacement case this branch added detection for, which cannot work if the uuid never changes. The listing is now cached until dropped, and the operator re-probe drops it alongside the ffmpeg and tone-map caches, which is exactly when either becomes true. Prometheus also exported engine busy percentages for a GPU whose source is `unavailable`, where the zeros mean "not measured". The JSON surfaces carry `source` alongside and render the difference; a sample does not, so those zeros read as an idle GPU on every dashboard and alert — for a card that may be busy and merely unobservable, which is what that source names. The engine gauges are now omitted for an unmeasured device. The session count still ships: it comes from this process's own workload accounting rather than from a driver, and a busy GPU with no engine reading is precisely when it is worth having. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- docs/wiki/admin/monitoring-nodes.md | 5 ++ internal/nodemetrics/collector.go | 24 ++++++++- internal/nodemetrics/collector_test.go | 47 ++++++++++++++++ internal/playback/gpudetect.go | 60 +++++++++++++++------ internal/playback/gpudetect_publish_test.go | 45 ++++++++++++++++ internal/playback/gpuidentity_test.go | 4 +- 6 files changed, 163 insertions(+), 22 deletions(-) diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index cbc1436df..fc94c2c8f 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -283,6 +283,11 @@ groups: summary: "Silo node CPU pegged on {{ $labels.instance }} — check the GPU column for a failed probe" ``` +A GPU nothing could measure exports no engine gauges at all rather than zeros — +a Prometheus sample carries no `source` to qualify them, and a zero would read as +idle on a card that may be busy and merely unobservable. Its session count still +ships, since that comes from Silo's own accounting rather than from a driver. + The GPU gauges (`streamapp_node_gpu_video_busy_percent`, `streamapp_node_gpu_render_busy_percent`, `streamapp_node_gpu_sessions`, `streamapp_node_gpu_vram_used_bytes`, `streamapp_node_gpu_vram_total_bytes`) are diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index 9728ed423..ce6e3890a 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -78,6 +78,13 @@ var ( // by. const gpuDeviceLabel = "device" +// isMeasuredGPUSource reports whether a source actually produced engine +// readings. SourceUnavailable means nothing could measure the device, so its +// zeros carry no measurement. +func isMeasuredGPUSource(source string) bool { + return source != "" && source != SourceUnavailable +} + // collector adapts a Sampler to prometheus.Collector. type collector struct{ sampler *Sampler } @@ -116,8 +123,21 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { } const bytesPerMBFloat = float64(1024 * 1024) for _, gpu := range snapshot.GPU { - gauge(descGPUVideoBusy, float64(gpu.VideoBusyPct), gpu.Device) - gauge(descGPURenderBusy, float64(gpu.RenderBusyPct), gpu.Device) + // Engine percentages are omitted rather than reported as zero when + // nothing measured them. The JSON surfaces carry `source` alongside and + // can render the difference; a Prometheus sample cannot, so an exported + // 0 would read as an idle GPU on every dashboard and alert — including + // for a card that is busy and merely unobservable, which is exactly the + // state `unavailable` names. An absent series is the honest shape for a + // number that was not taken. + if isMeasuredGPUSource(gpu.Source) { + gauge(descGPUVideoBusy, float64(gpu.VideoBusyPct), gpu.Device) + gauge(descGPURenderBusy, float64(gpu.RenderBusyPct), gpu.Device) + } + // Sessions always ships: it comes from this process's own workload + // accounting, not from a driver, so it is exact whatever the driver can + // or cannot tell us — and a busy GPU with no engine reading is precisely + // when an operator needs it. gauge(descGPUSessions, float64(gpu.Sessions), gpu.Device) if gpu.VRAMUsedMB != nil { gauge(descGPUVRAMUsed, float64(*gpu.VRAMUsedMB)*bytesPerMBFloat, gpu.Device) diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go index 6f1ce34c0..0ce345d7c 100644 --- a/internal/nodemetrics/collector_test.go +++ b/internal/nodemetrics/collector_test.go @@ -235,3 +235,50 @@ func diskSeriesByLabel(t *testing.T, sampler *Sampler) map[string]float64 { } return byLabel } + +// A GPU nothing could measure has zeros that mean "not taken", not "idle". The +// JSON surfaces carry `source` alongside and render the difference; a Prometheus +// sample cannot, so an exported 0 would show a busy-but-unobservable card as +// idle on every dashboard. The session count still ships: it comes from this +// process's own accounting rather than a driver. +func TestCollectorOmitsUnmeasuredGPUEngineGauges(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "cuda:0", Vendor: vendorNVIDIA, Sessions: 2, Source: SourceUnavailable, + }}, + }) + + values := gatherNames(t, sampler) + if _, present := values["streamapp_node_gpu_video_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a video busy percentage") + } + if _, present := values["streamapp_node_gpu_render_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a render busy percentage") + } + if got := values["streamapp_node_gpu_sessions"]; got != 2 { + t.Fatalf("gpu sessions = %v, want the workload count exported regardless", got) + } +} + +// A measured source exports both engines, including a genuine zero — which does +// mean idle. +func TestCollectorExportsMeasuredGPUEngineGauges(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "/dev/dri/renderD128", Sessions: 0, + VideoBusyPct: 0, RenderBusyPct: 0, Source: SourceFdinfo, + }}, + }) + + values := gatherNames(t, sampler) + if _, present := values["streamapp_node_gpu_video_busy_percent"]; !present { + t.Fatal("a measured idle GPU omitted its video busy percentage") + } + if _, present := values["streamapp_node_gpu_render_busy_percent"]; !present { + t.Fatal("a measured idle GPU omitted its render busy percentage") + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 5110dc2e0..1b99daa40 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -672,6 +672,11 @@ func InvalidateHWProbeCache() { hwProbeCache.generation++ hwProbeCache.entries = make(map[string]hwProbeCacheEntry) hwProbeCache.verifiedDevices = make(map[string]string) + // The GPU identity listing goes too. A re-probe is exactly when nvidia-smi + // may have become available, or a card in the same slot may have been + // replaced, and both of those change identities the drift comparison and + // shared-GPU placement read. + resetNVIDIAGPUUUIDs() } // hwProbeCacheKey separates results per invalidation generation, per backend, @@ -1082,12 +1087,29 @@ var nvidiaSMIQueryTimeout = 3 * time.Second // it rather than installing a fake binary on PATH. var nvidiaSMIQuery = runNVIDIASMIQuery -// nvidiaGPUUUIDs caches the one nvidia-smi listing this process makes. GPU -// identities cannot change without a reboot, so a second query could only cost -// a subprocess to learn the same answer. +// nvidiaGPUUUIDs caches the nvidia-smi listing. Within one generation of the +// probe caches a second query could only cost a subprocess to learn the same +// answer, since GPU identities do not change under a running kernel. +// +// It is not cached for the process lifetime, though, which a sync.Once would +// make it. Two things invalidate it and both are exactly what the operator +// re-probe exists for: an nvidia-smi that was missing or broken at first call +// would otherwise never be asked again, leaving every NVIDIA card without its +// permanent id; and a card swapped into the same PCI slot would keep answering +// to its predecessor's uuid. Both feed drift detection and shared-GPU +// placement, so a stale answer here is a wrong answer there. var nvidiaGPUUUIDs struct { - once sync.Once - byPCI map[string]string + mu sync.Mutex + loaded bool + byPCI map[string]string +} + +// resetNVIDIAGPUUUIDs drops the cached listing so the next lookup re-queries. +func resetNVIDIAGPUUUIDs() { + nvidiaGPUUUIDs.mu.Lock() + defer nvidiaGPUUUIDs.mu.Unlock() + nvidiaGPUUUIDs.loaded = false + nvidiaGPUUUIDs.byPCI = nil } func runNVIDIASMIQuery(ctx context.Context) ([]byte, error) { @@ -1099,18 +1121,22 @@ func runNVIDIASMIQuery(ctx context.Context) ([]byte, error) { } func nvidiaGPUUUIDsByPCIAddress() map[string]string { - nvidiaGPUUUIDs.once.Do(func() { - ctx, cancel := context.WithTimeout(context.Background(), nvidiaSMIQueryTimeout) - defer cancel() - output, err := nvidiaSMIQuery(ctx) - if err != nil { - // Expected on every host without the NVIDIA toolkit installed, so - // this stays at debug: the report is complete without it. - slog.Debug("nvidia-smi gpu identity query unavailable", "component", "playback", "error", err) - return - } - nvidiaGPUUUIDs.byPCI = parseNVIDIAGPUUUIDs(output) - }) + nvidiaGPUUUIDs.mu.Lock() + defer nvidiaGPUUUIDs.mu.Unlock() + if nvidiaGPUUUIDs.loaded { + return nvidiaGPUUUIDs.byPCI + } + nvidiaGPUUUIDs.loaded = true + ctx, cancel := context.WithTimeout(context.Background(), nvidiaSMIQueryTimeout) + defer cancel() + output, err := nvidiaSMIQuery(ctx) + if err != nil { + // Expected on every host without the NVIDIA toolkit installed, so this + // stays at debug: the report is complete without it. + slog.Debug("nvidia-smi gpu identity query unavailable", "component", "playback", "error", err) + return nil + } + nvidiaGPUUUIDs.byPCI = parseNVIDIAGPUUUIDs(output) return nvidiaGPUUUIDs.byPCI } diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 74806747c..a60c8fb54 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -251,3 +251,48 @@ func TestAcquireHWDeviceCountsTheAutoDetectedDeviceWithoutAProbe(t *testing.T) { t.Fatalf("active workloads after release = %d, want 0", got) } } + +// The identity listing used to be a sync.Once, so an nvidia-smi that was +// missing at first call was never asked again and a card swapped into the same +// slot kept answering to its predecessor's uuid — both of which feed drift +// detection and shared-GPU placement. A re-probe is exactly when either becomes +// true, so it drops the listing too. +func TestInvalidateHWProbeCacheRequeriesNVIDIAIdentities(t *testing.T) { + setupHWAccelTest(t) + + queries := 0 + answer := "" + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + queries++ + if answer == "" { + return nil, errors.New("nvidia-smi not installed") + } + return []byte(answer), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + + if got := nvidiaGPUUUIDsByPCIAddress(); len(got) != 0 { + t.Fatalf("identities = %v, want none while nvidia-smi is unavailable", got) + } + if nvidiaGPUUUIDsByPCIAddress(); queries != 1 { + t.Fatalf("queries = %d, want the failure cached within a generation", queries) + } + + // The toolkit is installed, or the card is replaced. Only a re-probe should + // make the process notice. + answer = "GPU-new, 00000000:03:00.0\n" + if got := nvidiaGPUUUIDsByPCIAddress(); len(got) != 0 { + t.Fatalf("identities = %v, want the cached answer until the caches are dropped", got) + } + + InvalidateHWProbeCache() + got := nvidiaGPUUUIDsByPCIAddress() + if got["0000:03:00.0"] != "GPU-new" { + t.Fatalf("identities = %v, want the re-probe to pick up the new uuid", got) + } +} diff --git a/internal/playback/gpuidentity_test.go b/internal/playback/gpuidentity_test.go index 4aaabaa2e..d0437ee45 100644 --- a/internal/playback/gpuidentity_test.go +++ b/internal/playback/gpuidentity_test.go @@ -5,7 +5,6 @@ import ( "errors" "os" "path/filepath" - "sync" "testing" ) @@ -63,8 +62,7 @@ func stubNVIDIASMI(t *testing.T, output string, err error) { } func resetNVIDIAUUIDCacheForTest() { - nvidiaGPUUUIDs.once = sync.Once{} - nvidiaGPUUUIDs.byPCI = nil + resetNVIDIAGPUUUIDs() } func renderDeviceDetail(t *testing.T, info HWAccelInfo, path string) RenderDeviceInfo { From 02291ddf790995168dc052220d707de9550d769f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:13:51 -0400 Subject: [PATCH 033/163] fix(nodemetrics): report known GPU workloads without a measurement source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVENC workloads are counted under a CUDA index or GPU uuid, and only the nvidia-smi step registers those aliases. With that step failed — a timeout, no toolkit, a tripped breaker — an NVENC node reported zero sessions while it was transcoding, and one exposing no readable render node dropped out of the GPU sample entirely. Session accounting comes from the playback allocator rather than from a driver, so it is true whether or not anything answered. The sample now includes an entry for every device a workload is counted against that nothing else named. Without nvidia-smi there is no way to say which card a bare "cuda:0" is, so it stands as its own unmeasured entry rather than being guessed onto another; a workload whose device an enrichment source did name is still joined onto that entry rather than duplicated. This matters more since the previous commit stopped exporting engine gauges for an unmeasured device: the session count is then the only signal that a GPU is doing anything at all. Found by review of #794 (Codex). Related issue: #780 Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/sampler.go | 21 ++++++++++ internal/nodemetrics/sampler_test.go | 61 ++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index bc61bb6b7..275f18ec4 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -460,6 +460,27 @@ func (s *Sampler) sampleGPU(ctx context.Context, now time.Time) []GPUStats { } } + // 4. Workloads this process knows it started, on devices nothing above + // named. Session accounting comes from the playback allocator rather than + // from a driver, so it is true whether or not any measurement source + // answered — and NVENC workloads are keyed by CUDA index or GPU uuid, which + // only the nvidia-smi step supplies. With that step failed or its breaker + // tripped, an NVENC node would otherwise report zero sessions while it + // transcodes, and one exposing no readable render node would vanish from the + // sample entirely. Nothing here can say *which* card a bare "cuda:0" is + // without nvidia-smi, so it stands as its own unmeasured entry: an honest + // "a workload is running on this device and nobody could measure it" beats a + // silent zero. + for name, count := range sessions { + if name == "" || count <= 0 { + continue + } + if slices.ContainsFunc(order, func(key string) bool { return slices.Contains(aliases[key], name) }) { + continue + } + upsert(name) + } + if len(order) == 0 { return nil } diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 7bc7e2ed9..a11453e60 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -3,6 +3,7 @@ package nodemetrics import ( "context" "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -742,3 +743,63 @@ func TestMemoryStatsUsesCgroupUsageBesideACgroupLimit(t *testing.T) { t.Fatalf("used = %d, want the cgroup working set", used) } } + +// NVENC workloads are keyed by CUDA index or GPU uuid, and only the nvidia-smi +// step supplies those aliases. With that step failed — a timeout, a missing +// toolkit, a tripped breaker — an NVENC node reported zero sessions while it +// transcoded, and one exposing no readable render node vanished from the sample +// altogether. Session accounting comes from the playback allocator, not a +// driver, so it is true regardless. +func TestSampleGPUReportsSessionsWhenNVIDIAEnrichmentFails(t *testing.T) { + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return nil, errors.New("nvidia-smi not installed") + } + s.identities = func() []DeviceIdentity { return nil } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + gpus := s.sampleGPU(context.Background(), s.now()) + + if len(gpus) != 1 { + t.Fatalf("gpu sample = %+v, want the known workload's device reported", gpus) + } + if gpus[0].Device != "cuda:0" { + t.Fatalf("device = %q, want the name the workload is counted under", gpus[0].Device) + } + if gpus[0].Sessions != 2 { + t.Fatalf("sessions = %d, want the allocator's count", gpus[0].Sessions) + } + if gpus[0].Source != SourceUnavailable { + t.Fatalf("source = %q, want it reported as unmeasured", gpus[0].Source) + } +} + +// A workload whose device an enrichment source did name is not duplicated: the +// alias set already covers it. +func TestSampleGPUDoesNotDuplicateAnAlreadyNamedDevice(t *testing.T) { + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.identities = func() []DeviceIdentity { return nil } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + gpus := s.sampleGPU(context.Background(), s.now()) + + if len(gpus) != 1 { + t.Fatalf("gpu sample = %+v, want one entry for the enriched device", gpus) + } + if gpus[0].Sessions != 2 { + t.Fatalf("sessions = %d, want the workload joined onto the enriched entry", gpus[0].Sessions) + } +} From 75f476283203f76c67721f5701637df2b83f5999 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:34:12 -0400 Subject: [PATCH 034/163] fix(nodemetrics): keep per-field availability for GPU readings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings, all "an unmeasured value was published as a measured zero" in different clothes. nvidia-smi prints "[N/A]" or "[Not Supported]" per column, not per row: a card can report memory and utilization while saying nothing about its video engines. Parsing each column to 0 published those as measurements under a source that claimed to have taken them — an unobservable encoder read as idle, unsupported VRAM read as 0 bytes, and /metrics exported both. Every measurement in GPUStats is now a pointer, the shape TotalBusyPct already used for the same reason, and the collector gates each series on its own value rather than on the device's source. An nvidia-smi-only card therefore stops exporting a render-engine series it never had, and fdinfo's first sample — which has no interval to divide by — stops reporting an unknown load as idle. NVENC was probed under a policy written for render devices: a mixed host configured with a render path for its Intel card marked NVENC *failed*, which sets capability drift, when the truth is that the path says nothing about CUDA. It is now skipped, which does not. The drift baseline recorded a device as a flat alias list, so recovery matched any single overlapping alias. A replacement card inherits the slot's PCI address and usually its render path, so swapping one in read as the lost card returning. Baseline devices now keep their uuid and match through sameDevice, where conflicting uuids are decisive — the same rule the loss comparison already applied, now applied to recovery. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 23 ++++--- .../api/handlers/system_resources_test.go | 3 +- internal/nodemetrics/collector.go | 28 +++----- internal/nodemetrics/collector_test.go | 10 ++- internal/nodemetrics/fdinfo_test.go | 30 +++++--- internal/nodemetrics/nvidia.go | 57 +++++++++++---- internal/nodemetrics/nvidia_test.go | 69 ++++++++++++++++--- internal/nodemetrics/sampler.go | 24 ++++--- internal/nodemetrics/sampler_test.go | 7 +- internal/nodemetrics/snapshot.go | 15 ++-- internal/nodepool/health.go | 49 ++++++++----- internal/nodepool/health_drift_test.go | 45 +++++++++++- internal/playback/gpudetect.go | 51 +++++++++++--- internal/playback/gpudetect_publish_test.go | 59 ++++++++++++++++ internal/proxy/metrics_test.go | 3 +- internal/transcodenode/metrics_test.go | 3 +- 16 files changed, 364 insertions(+), 112 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 9b45ae4f9..6426ea21e 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -189,16 +189,23 @@ Each entry in `last_stats.gpu`: | `vendor` | string | `intel`, `nvidia` or `amd`. Omitted when sysfs names a vendor we do not recognize. | | `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. With no `playback.hw_device` configured the workload is counted against the render device the transcode will actually open — the one auto-detection verified the backend on, or the first available render node when the backend was named explicitly and no detection walk ran. It goes uncounted only on a host with no render device at all. | | `video_busy_pct`, `render_busy_pct` | int | Engine busy percentages over the sampling interval. | -| `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. Present only with an enrichment source — absent is not zero, and must not be rendered as an idle GPU. | -| `vram_used_mb`, `vram_total_mb` | int | GPU memory, on the same terms as `total_busy_pct`. | +| `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. | +| `vram_used_mb`, `vram_total_mb` | int | GPU memory. | | `source` | string | What produced the numbers: `fdinfo`, `nvidia-smi`, `fdinfo+nvidia-smi`, or `unavailable`. | -`source` is what tells an operator how far to trust the busy percentages. -`fdinfo` is the unprivileged DRM baseline and covers **only this node's own -ffmpeg children** — a GPU shared with anything outside Silo reads as less busy -than it is. `nvidia-smi` is whole-GPU. `unavailable` means nothing could measure -the device this interval; its percentages are zeros with no measurement behind -them. +Every measurement field above is omitted when nothing measured it, and +availability is per field rather than per device: absent is not zero and must +not be rendered as an idle GPU. A card can answer for some columns and not +others — `nvidia-smi` prints `[N/A]` for an engine a GPU cannot report while +still giving real memory figures, and a device reached only through `nvidia-smi` +has no render-engine reading at all — so read each field's presence, not the +device's. + +`source` is what tells an operator how far to trust the busy percentages that +are present. `fdinfo` is the unprivileged DRM baseline and covers **only this +node's own ffmpeg children** — a GPU shared with anything outside Silo reads as +less busy than it is. `nvidia-smi` is whole-GPU. `unavailable` means nothing +could measure the device this interval, so it carries no percentages at all. A node reports these fields in its own `/health` and `/status`; the API stores them opaquely and parses only what it routes on. No GPU field is one of those — diff --git a/internal/api/handlers/system_resources_test.go b/internal/api/handlers/system_resources_test.go index 70c9c76db..1637e0eee 100644 --- a/internal/api/handlers/system_resources_test.go +++ b/internal/api/handlers/system_resources_test.go @@ -15,6 +15,7 @@ func TestSystemResourcesReportsLocalSample(t *testing.T) { sampledAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) total := 71 + video, render := 63, 12 handler := &SystemHandler{} handler.SetResourceSampler(nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ Available: true, @@ -27,7 +28,7 @@ func TestSystemResourcesReportsLocalSample(t *testing.T) { }, GPU: []nodemetrics.GPUStats{{ Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, - VideoBusyPct: 63, RenderBusyPct: 12, TotalBusyPct: &total, + VideoBusyPct: &video, RenderBusyPct: &render, TotalBusyPct: &total, Source: nodemetrics.SourceFdinfo, }}, })) diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index ce6e3890a..fce145204 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -78,13 +78,6 @@ var ( // by. const gpuDeviceLabel = "device" -// isMeasuredGPUSource reports whether a source actually produced engine -// readings. SourceUnavailable means nothing could measure the device, so its -// zeros carry no measurement. -func isMeasuredGPUSource(source string) bool { - return source != "" && source != SourceUnavailable -} - // collector adapts a Sampler to prometheus.Collector. type collector struct{ sampler *Sampler } @@ -123,16 +116,17 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { } const bytesPerMBFloat = float64(1024 * 1024) for _, gpu := range snapshot.GPU { - // Engine percentages are omitted rather than reported as zero when - // nothing measured them. The JSON surfaces carry `source` alongside and - // can render the difference; a Prometheus sample cannot, so an exported - // 0 would read as an idle GPU on every dashboard and alert — including - // for a card that is busy and merely unobservable, which is exactly the - // state `unavailable` names. An absent series is the honest shape for a - // number that was not taken. - if isMeasuredGPUSource(gpu.Source) { - gauge(descGPUVideoBusy, float64(gpu.VideoBusyPct), gpu.Device) - gauge(descGPURenderBusy, float64(gpu.RenderBusyPct), gpu.Device) + // Every measurement is exported only when the snapshot actually holds + // one. The JSON surfaces carry `source` alongside and can render the + // difference between unmeasured and idle; a Prometheus sample cannot, so + // an exported 0 would read as an idle GPU on every dashboard and alert — + // including for a card that is busy and merely unobservable. An absent + // series is the honest shape for a number that was not taken. + if gpu.VideoBusyPct != nil { + gauge(descGPUVideoBusy, float64(*gpu.VideoBusyPct), gpu.Device) + } + if gpu.RenderBusyPct != nil { + gauge(descGPURenderBusy, float64(*gpu.RenderBusyPct), gpu.Device) } // Sessions always ships: it comes from this process's own workload // accounting, not from a driver, so it is exact whatever the driver can diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go index 0ce345d7c..0c0d47006 100644 --- a/internal/nodemetrics/collector_test.go +++ b/internal/nodemetrics/collector_test.go @@ -58,7 +58,6 @@ func TestCollectorExposesSnapshot(t *testing.T) { "streamapp_node_network_rx_bps", "streamapp_node_network_tx_bps", "streamapp_node_gpu_video_busy_percent", - "streamapp_node_gpu_render_busy_percent", "streamapp_node_gpu_sessions", "streamapp_node_gpu_vram_used_bytes", "streamapp_node_gpu_vram_total_bytes", @@ -67,6 +66,13 @@ func TestCollectorExposesSnapshot(t *testing.T) { t.Fatalf("%s missing from the scrape (got %v)", name, values) } } + // nvidia-smi reports encoder and decoder but nothing for the render engine, + // and this card has no DRM counters to supply one. Exporting the missing + // column as 0 would draw an idle 3D engine for a GPU nobody measured. + if _, ok := values["streamapp_node_gpu_render_busy_percent"]; ok { + t.Fatalf("render busy exported for an nvidia-smi-only card: %v", values) + } + // Bytes on the wire, gibibytes in the JSON: the conversion has to survive. if got := values["streamapp_node_disk_used_bytes"]; got != float64(100<<30) { t.Fatalf("disk used = %v, want %v bytes", got, float64(100<<30)) @@ -270,7 +276,7 @@ func TestCollectorExportsMeasuredGPUEngineGauges(t *testing.T) { System: &SystemStats{}, GPU: []GPUStats{{ Device: "/dev/dri/renderD128", Sessions: 0, - VideoBusyPct: 0, RenderBusyPct: 0, Source: SourceFdinfo, + VideoBusyPct: ptr(0), RenderBusyPct: ptr(0), Source: SourceFdinfo, }}, }) diff --git a/internal/nodemetrics/fdinfo_test.go b/internal/nodemetrics/fdinfo_test.go index 639241a85..739d6191e 100644 --- a/internal/nodemetrics/fdinfo_test.go +++ b/internal/nodemetrics/fdinfo_test.go @@ -152,8 +152,8 @@ func TestSampleGPUMapsPdevToDevicePathAndComputesBusy(t *testing.T) { if first[0].Source != SourceFdinfo { t.Fatalf("Source = %q, want %q", first[0].Source, SourceFdinfo) } - if first[0].VideoBusyPct != 0 { - t.Fatalf("VideoBusyPct = %d on the first sample, want 0 (nothing to diff against)", first[0].VideoBusyPct) + if first[0].VideoBusyPct != nil { + t.Fatalf("VideoBusyPct = %d on the first sample, want it unset (nothing to diff against)", *first[0].VideoBusyPct) } // Over 4s of wall time: +2s of video engine, +1s of render engine. @@ -167,11 +167,11 @@ drm-engine-video: 4000000000 ns s.sample(context.Background()) second := s.Snapshot().GPU[0] - if second.VideoBusyPct != 50 { - t.Fatalf("VideoBusyPct = %d, want 50", second.VideoBusyPct) + if got := enginePct(t, second.VideoBusyPct); got != 50 { + t.Fatalf("VideoBusyPct = %d, want 50", got) } - if second.RenderBusyPct != 25 { - t.Fatalf("RenderBusyPct = %d, want 25", second.RenderBusyPct) + if got := enginePct(t, second.RenderBusyPct); got != 25 { + t.Fatalf("RenderBusyPct = %d, want 25", got) } if second.TotalBusyPct != nil { t.Fatal("TotalBusyPct set without an enrichment source") @@ -215,7 +215,7 @@ drm-engine-video: 7000000000 ns clock.advance(5 * time.Second) s.sample(context.Background()) - if got := s.Snapshot().GPU[0].VideoBusyPct; got != 100 { + if got := enginePct(t, s.Snapshot().GPU[0].VideoBusyPct); got != 100 { t.Fatalf("VideoBusyPct = %d while the surviving transcode ran the engine flat out, want 100", got) } } @@ -251,8 +251,8 @@ drm-engine-video: 5000000000 ns if len(gpu) != 1 { t.Fatalf("GPU = %+v, want one device", gpu) } - if gpu[0].VideoBusyPct != 0 || gpu[0].RenderBusyPct != 0 { - t.Fatalf("busy = %d/%d after a transcode exited, want 0/0", gpu[0].VideoBusyPct, gpu[0].RenderBusyPct) + if video, render := enginePct(t, gpu[0].VideoBusyPct), enginePct(t, gpu[0].RenderBusyPct); video != 0 || render != 0 { + t.Fatalf("busy = %d/%d after a transcode exited, want a measured 0/0", video, render) } // The reduced total is the new baseline: the surviving transcode's next @@ -264,7 +264,7 @@ drm-engine-video: 3000000000 ns `) clock.advance(10 * time.Second) s.sample(context.Background()) - if got := s.Snapshot().GPU[0].VideoBusyPct; got != 10 { + if got := enginePct(t, s.Snapshot().GPU[0].VideoBusyPct); got != 10 { t.Fatalf("VideoBusyPct after re-baselining = %d, want 10", got) } } @@ -337,3 +337,13 @@ func TestDeviceEngineDeltasIgnoresCounterRegressions(t *testing.T) { t.Fatalf("delta on a counter regression = %d, want 0", got) } } + +// enginePct reads an engine percentage a test requires to be present, so a +// missing measurement fails as itself rather than as a nil dereference. +func enginePct(t *testing.T, got *int) int { + t.Helper() + if got == nil { + t.Fatal("engine percentage is unset, want a measurement") + } + return *got +} diff --git a/internal/nodemetrics/nvidia.go b/internal/nodemetrics/nvidia.go index 139dff33e..dcd3759c8 100644 --- a/internal/nodemetrics/nvidia.go +++ b/internal/nodemetrics/nvidia.go @@ -45,17 +45,38 @@ var runNVIDIASMI = func(ctx context.Context) ([]byte, error) { } // nvidiaGPU is one parsed nvidia-smi row. +// +// The measurement columns are optional because a successful row can still be +// only partly measurable: a driver reports "[N/A]" or "[Not Supported]" per +// column for engines or memory it cannot see, and coercing those to zero would +// publish an unobservable video engine as idle and unsupported VRAM as 0 bytes +// under an "nvidia-smi" source that claims they were measured. type nvidiaGPU struct { Index int UUID string PCIAddress string - GPUUtil int - EncoderUtil int - DecoderUtil int - MemUsedMB int64 - MemTotalMB int64 + GPUUtil *int + EncoderUtil *int + DecoderUtil *int + MemUsedMB *int64 + MemTotalMB *int64 } +// videoUtil is the higher of the two fixed-function video engines, or nil when +// the driver reported neither. +func (g nvidiaGPU) videoUtil() *int { + switch { + case g.EncoderUtil != nil && g.DecoderUtil != nil: + return ptr(max(*g.EncoderUtil, *g.DecoderUtil)) + case g.EncoderUtil != nil: + return g.EncoderUtil + default: + return g.DecoderUtil + } +} + +func ptr[T any](value T) *T { return &value } + // sourceBreaker retires an enrichment source after repeated failure. type sourceBreaker struct { name string @@ -139,22 +160,28 @@ func parseNVIDIASMI(output []byte) []nvidiaGPU { GPUUtil: parseNVIDIAInt(fields[3]), EncoderUtil: parseNVIDIAInt(fields[4]), DecoderUtil: parseNVIDIAInt(fields[5]), - MemUsedMB: int64(parseNVIDIAInt(fields[6])), - MemTotalMB: int64(parseNVIDIAInt(fields[7])), + MemUsedMB: parseNVIDIAInt64(fields[6]), + MemTotalMB: parseNVIDIAInt64(fields[7]), }) } return gpus } -// parseNVIDIAInt reads one numeric column, treating the driver's "[N/A]" and -// "[Not Supported]" placeholders as zero. -func parseNVIDIAInt(field string) int { +// parseNVIDIAInt reads one numeric column, or nil for the driver's "[N/A]" and +// "[Not Supported]" placeholders — which say the value was not measured, not +// that it is zero. +func parseNVIDIAInt(field string) *int { value, err := strconv.Atoi(strings.TrimSpace(field)) - if err != nil { - return 0 + if err != nil || value < 0 { + return nil } - if value < 0 { - return 0 + return &value +} + +func parseNVIDIAInt64(field string) *int64 { + value := parseNVIDIAInt(field) + if value == nil { + return nil } - return value + return ptr(int64(*value)) } diff --git a/internal/nodemetrics/nvidia_test.go b/internal/nodemetrics/nvidia_test.go index 5812e3207..50f42059b 100644 --- a/internal/nodemetrics/nvidia_test.go +++ b/internal/nodemetrics/nvidia_test.go @@ -23,23 +23,72 @@ func TestParseNVIDIASMI(t *testing.T) { if first.PCIAddress != "0000:03:00.0" { t.Fatalf("PCIAddress = %q, want the normalized sysfs form", first.PCIAddress) } - if first.GPUUtil != 71 || first.EncoderUtil != 63 || first.DecoderUtil != 12 { - t.Fatalf("gpus[0] utilization = %+v", first) + if *first.GPUUtil != 71 || *first.EncoderUtil != 63 || *first.DecoderUtil != 12 { + t.Fatalf("gpus[0] utilization = %d/%d/%d", *first.GPUUtil, *first.EncoderUtil, *first.DecoderUtil) } - if first.MemUsedMB != 812 || first.MemTotalMB != 8192 { - t.Fatalf("gpus[0] memory = %+v", first) + if *first.MemUsedMB != 812 || *first.MemTotalMB != 8192 { + t.Fatalf("gpus[0] memory = %d/%d", *first.MemUsedMB, *first.MemTotalMB) } } // Drivers print "[N/A]" for a column a card does not support. One unsupported -// column must not discard the whole row. +// column must not discard the whole row — nor be read as a measured zero, which +// would show an engine nobody can see as idle. func TestParseNVIDIASMIToleratesPlaceholders(t *testing.T) { - gpus := parseNVIDIASMI([]byte("0, GPU-x, 00000000:03:00.0, [N/A], [Not Supported], 4, 100, 8192\n")) + gpus := parseNVIDIASMI([]byte("0, GPU-x, 00000000:03:00.0, [N/A], [Not Supported], 4, 100, [N/A]\n")) if len(gpus) != 1 { t.Fatalf("gpus = %+v, want the row kept", gpus) } - if gpus[0].GPUUtil != 0 || gpus[0].EncoderUtil != 0 || gpus[0].DecoderUtil != 4 { - t.Fatalf("gpus[0] = %+v, want placeholders read as zero and 4 preserved", gpus[0]) + got := gpus[0] + if got.GPUUtil != nil || got.EncoderUtil != nil || got.MemTotalMB != nil { + t.Fatalf("gpus[0] = %+v, want the placeholder columns unset", got) + } + if got.DecoderUtil == nil || *got.DecoderUtil != 4 { + t.Fatalf("DecoderUtil = %v, want the reported 4 preserved", got.DecoderUtil) + } + if got.MemUsedMB == nil || *got.MemUsedMB != 100 { + t.Fatalf("MemUsedMB = %v, want the reported 100 preserved", got.MemUsedMB) + } + // One reported engine still gives a video reading; both missing gives none. + if video := got.videoUtil(); video == nil || *video != 4 { + t.Fatalf("videoUtil() = %v, want the one engine the driver did report", video) + } + if video := (nvidiaGPU{}).videoUtil(); video != nil { + t.Fatalf("videoUtil() = %d with neither engine reported, want none", *video) + } +} + +// A card that reports memory but no video engines keeps the nvidia-smi source +// for the columns it did answer, and simply carries no engine reading. +func TestSampleGPUKeepsPartialNVIDIAReadings(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, [N/A], [N/A], 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if gpu[0].Source != SourceNVIDIASMI { + t.Fatalf("Source = %q, want %q for the columns it did measure", gpu[0].Source, SourceNVIDIASMI) + } + if gpu[0].VideoBusyPct != nil { + t.Fatalf("VideoBusyPct = %d, want no reading for engines the driver cannot see", *gpu[0].VideoBusyPct) + } + if gpu[0].TotalBusyPct == nil || *gpu[0].TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want the 71 nvidia-smi did report", gpu[0].TotalBusyPct) + } + if gpu[0].VRAMTotalMB == nil || *gpu[0].VRAMTotalMB != 8192 { + t.Fatalf("VRAMTotalMB = %v, want the 8192 nvidia-smi did report", gpu[0].VRAMTotalMB) } } @@ -76,8 +125,8 @@ func TestSampleGPUEnrichesWithNVIDIASMI(t *testing.T) { if first.TotalBusyPct == nil || *first.TotalBusyPct != 71 { t.Fatalf("TotalBusyPct = %v, want 71", first.TotalBusyPct) } - if first.VideoBusyPct != 63 { - t.Fatalf("VideoBusyPct = %d, want the busier of encoder/decoder", first.VideoBusyPct) + if got := enginePct(t, first.VideoBusyPct); got != 63 { + t.Fatalf("VideoBusyPct = %d, want the busier of encoder/decoder", got) } if first.VRAMUsedMB == nil || *first.VRAMUsedMB != 812 { t.Fatalf("VRAMUsedMB = %v, want 812", first.VRAMUsedMB) diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 275f18ec4..103e868f4 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -415,8 +415,11 @@ func (s *Sampler) sampleGPU(ctx context.Context, now time.Time) []GPUStats { alias(pdev, path) } if elapsedNS > 0 { - entry.VideoBusyPct = engineBusyPercent(delta.videoNS, elapsedNS) - entry.RenderBusyPct = engineBusyPercent(delta.renderNS, elapsedNS) + // Before the second sample there is no interval to divide by, so the + // engines stay unset rather than reporting a freshly started node's + // unknown load as idle. + entry.VideoBusyPct = ptr(engineBusyPercent(delta.videoNS, elapsedNS)) + entry.RenderBusyPct = ptr(engineBusyPercent(delta.renderNS, elapsedNS)) } entry.Source = SourceFdinfo } @@ -445,18 +448,21 @@ func (s *Sampler) sampleGPU(ctx context.Context, now time.Time) []GPUStats { // this entry. alias(key, cudaName, gpu.UUID) entry.Vendor = vendorNVIDIA - total := gpu.GPUUtil - entry.TotalBusyPct = &total - used, capacity := gpu.MemUsedMB, gpu.MemTotalMB - entry.VRAMUsedMB = &used - entry.VRAMTotalMB = &capacity + // Each column is carried only when the driver actually reported it: a + // successful row can still answer "[N/A]" for engines or memory it + // cannot see, and publishing those as zero would show an unobservable + // video engine as idle and unsupported VRAM as 0 bytes. + entry.TotalBusyPct = gpu.GPUUtil + entry.VRAMUsedMB = gpu.MemUsedMB + entry.VRAMTotalMB = gpu.MemTotalMB if entry.Source == SourceFdinfo { entry.Source = SourceFdinfoNVIDIASMI } else { entry.Source = SourceNVIDIASMI // fdinfo is unimplemented by the proprietary driver, so the video - // engines only have an nvidia-smi reading. - entry.VideoBusyPct = max(gpu.EncoderUtil, gpu.DecoderUtil) + // engines only have an nvidia-smi reading — which stays unset when + // the driver answered "[N/A]" for both of them. + entry.VideoBusyPct = gpu.videoUtil() } } diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index a11453e60..585215e49 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -450,6 +450,7 @@ func TestNonLinuxHostReportsUnavailable(t *testing.T) { // endpoint, and the persisted last_stats column all carry this shape. func TestSnapshotJSONShape(t *testing.T) { total := 71 + video, render := 63, 12 vramUsed := int64(812) vramTotal := int64(8192) snapshot := Snapshot{ @@ -472,8 +473,8 @@ func TestSnapshotJSONShape(t *testing.T) { Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, - VideoBusyPct: 63, - RenderBusyPct: 12, + VideoBusyPct: &video, + RenderBusyPct: &render, TotalBusyPct: &total, VRAMUsedMB: &vramUsed, VRAMTotalMB: &vramTotal, @@ -531,7 +532,7 @@ func TestSnapshotJSONShape(t *testing.T) { t.Fatalf("system emitted when absent: %s", bare) } bareGPU := bareDecoded["gpu"].([]any)[0].(map[string]any) - for _, key := range []string{"total_busy_pct", "vram_used_mb", "vram_total_mb", "vendor"} { + for _, key := range []string{"video_busy_pct", "render_busy_pct", "total_busy_pct", "vram_used_mb", "vram_total_mb", "vendor"} { if _, ok := bareGPU[key]; ok { t.Fatalf("gpu.%s emitted when absent: %s", key, bare) } diff --git a/internal/nodemetrics/snapshot.go b/internal/nodemetrics/snapshot.go index ca6b9a42c..1c08865e0 100644 --- a/internal/nodemetrics/snapshot.go +++ b/internal/nodemetrics/snapshot.go @@ -168,11 +168,16 @@ type GPUStats struct { Sessions int `json:"sessions"` // VideoBusyPct and RenderBusyPct are engine busy percentages over the last // sample interval. From fdinfo they cover only our own processes. - VideoBusyPct int `json:"video_busy_pct"` - RenderBusyPct int `json:"render_busy_pct"` - // TotalBusyPct is whole-GPU utilization including other tenants. It is a - // pointer because "no enrichment source" and "idle" are different facts and - // an operator must not read the first as the second. + VideoBusyPct *int `json:"video_busy_pct,omitempty"` + RenderBusyPct *int `json:"render_busy_pct,omitempty"` + // TotalBusyPct is whole-GPU utilization including other tenants. + // + // Every measurement above and below is a pointer because availability is per + // field, not per device: a source answers for some columns and not others — + // nvidia-smi prints "[N/A]" for an engine a card cannot report while still + // giving real memory figures, and fdinfo has no reading at all until it has + // two samples to diff. "Nothing measured this" and "measured, and idle" are + // different facts, and an operator must not read the first as the second. TotalBusyPct *int `json:"total_busy_pct,omitempty"` VRAMUsedMB *int64 `json:"vram_used_mb,omitempty"` VRAMTotalMB *int64 `json:"vram_total_mb,omitempty"` diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 3f30ce9c2..6ed3ae152 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -689,9 +689,26 @@ func resolveDriftNote(stored *string, storedBaseline []byte, drift capabilityDri type driftBaseline struct { // Backends must verify again. Backends []string `json:"backends,omitempty"` - // Devices are alias sets: any one member reappearing identifies the card, - // so a renumbered render node or a pass without nvidia-smi still matches. - Devices [][]string `json:"devices,omitempty"` + // Devices are the cards that must reappear, each carrying every name it + // answered to so a renumbered render node — or a pass where nvidia-smi did + // not answer — still matches it. + Devices []driftBaselineDevice `json:"devices,omitempty"` +} + +// driftBaselineDevice is one lost card's identity, in the same shape the loss +// comparison uses so both sides apply the same matching rule. +type driftBaselineDevice struct { + // UUID is the card's permanent identity where it published one. It is held + // apart from Aliases because it is the only name that can prove two devices + // are *different*: a replacement in the same slot inherits the slot and + // usually the render path, and matching on those would read as the lost + // card returning. + UUID string `json:"uuid,omitempty"` + Aliases []string `json:"aliases,omitempty"` +} + +func (d driftBaselineDevice) matches(candidate renderDeviceAliases) bool { + return renderDeviceAliases{uuid: d.UUID, aliases: d.Aliases}.sameDevice(candidate) } func (b driftBaseline) empty() bool { return len(b.Backends) == 0 && len(b.Devices) == 0 } @@ -718,14 +735,13 @@ func (b driftBaseline) recoveredBy(payload []byte) bool { return false } } - present := make(map[string]bool) - for _, device := range renderDeviceAliasSets(current) { - for _, alias := range device.aliases { - present[alias] = true - } - } - for _, aliases := range b.Devices { - if !slices.ContainsFunc(aliases, func(alias string) bool { return present[alias] }) { + // Matched with sameDevice, not by any shared alias: a replacement card in + // the same slot shares the PCI address and usually the render path, and + // treating that as the lost card returning is exactly the false recovery + // the loss side already refuses to call a match. + currentDevices := renderDeviceAliasSets(current) + for _, device := range b.Devices { + if !slices.ContainsFunc(currentDevices, device.matches) { return false } } @@ -747,14 +763,15 @@ func mergeDriftBaseline(stored []byte, drift capabilityDrift) driftBaseline { } } for _, device := range drift.lostDeviceAliases { - if slices.ContainsFunc(baseline.Devices, func(existing []string) bool { - return slices.ContainsFunc(existing, func(alias string) bool { - return slices.Contains(device.aliases, alias) - }) + if slices.ContainsFunc(baseline.Devices, func(existing driftBaselineDevice) bool { + return existing.matches(device) }) { continue } - baseline.Devices = append(baseline.Devices, slices.Clone(device.aliases)) + baseline.Devices = append(baseline.Devices, driftBaselineDevice{ + UUID: device.uuid, + Aliases: slices.Clone(device.aliases), + }) } slices.Sort(baseline.Backends) return baseline diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 93814a037..4f13ed575 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -196,7 +196,7 @@ func TestResolveDriftNoteKeepsNoteWhenAnUnrelatedGPUIsAdded(t *testing.T) { standing := "render devices gone: /dev/dri/renderD128" // The lost card, by every identity it answered to. - baseline := []byte(`{"devices":[["0000:03:00.0","/dev/dri/renderD128"]]}`) + baseline := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) payload := []byte(gained) drift, parsed := computeCapabilityDrift(payload, payload) @@ -221,7 +221,7 @@ func TestResolveDriftNoteClearsWhenTheLostCardReturnsRenumbered(t *testing.T) { `"detected_backends":[{"backend":"vaapi","verified":true}]}` standing := "render devices gone: /dev/dri/renderD128" - baseline := []byte(`{"devices":[["0000:03:00.0","/dev/dri/renderD128"]]}`) + baseline := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) payload := []byte(back) drift, parsed := computeCapabilityDrift(payload, payload) @@ -276,7 +276,7 @@ func TestResolveDriftNoteKeepsNoteWhileASiblingGPUIsStillMissing(t *testing.T) { `"detected_backends":[{"backend":"vaapi","verified":true}]}` standing := "render devices gone: /dev/dri/renderD129" - baseline := []byte(`{"devices":[["0000:04:00.0","/dev/dri/renderD129"]]}`) + baseline := []byte(`{"devices":[{"aliases":["0000:04:00.0","/dev/dri/renderD129"]}]}`) payload := []byte(degraded) // The next refetch is degraded-to-degraded: nothing newly lost, and every // probe that ran passed, because the survivor is fine. @@ -645,3 +645,42 @@ func TestResolveDriftNoteClearsALegacyNoteWithNoBaseline(t *testing.T) { t.Fatalf("capability_drift = %q, want a baseline-less note cleared by a clean report", *got) } } + +// A replacement card in the same slot inherits the PCI address and usually the +// render path. Clearing on that would report the lost card as returned when a +// different one arrived — the same false match the loss comparison already +// refuses, applied to the recovery side. +func TestResolveDriftNoteKeepsNoteWhenAReplacementCardTakesTheSlot(t *testing.T) { + const replaced = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-new"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"uuid":"GPU-old","aliases":["GPU-old","0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(replaced) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared because a different card took the slot") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } +} + +// The original card returning does clear it, even under a renumbered path. +func TestResolveDriftNoteClearsWhenTheSameCardReturnsByUUID(t *testing.T) { + const back = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD131"],` + + `"render_device_details":[{"path":"/dev/dri/renderD131","pci_address":"0000:09:00.0","gpu_uuid":"GPU-old"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"uuid":"GPU-old","aliases":["GPU-old","0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(back) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want the same uuid in a new slot to clear it", *got) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 1b99daa40..00e83f233 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -441,8 +441,8 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi complete = false break } - if !candidates.deviceProbeable(backend, device) { - reasons = append(reasons, hwProbeFailureReason(len(devices), device, "device not accessible on this node")) + if reason, unprobeable := candidates.unprobeableReason(backend, device); unprobeable { + reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) continue } probed = true @@ -518,17 +518,46 @@ func verifiedHWDeviceKey(generation uint64, backend string) string { return strconv.FormatUint(generation, 10) + "\x00" + backend } -// deviceProbeable reports whether a candidate device may be smoke-encoded on. +// unprobeableReason reports why a candidate cannot be smoke-encoded on, or +// false when it can be. // -// An empty device names no file, and a nil map means the candidate set came -// from discovery, which is openable by construction. NVENC is exempt whatever -// its device says: a CUDA index or GPU uuid is not a path, so failing to open it -// is meaningless, and the smoke encode is the only thing that can answer. -func (c hwCandidates) deviceProbeable(backend, device string) bool { - if device == "" || c.accessible == nil || backend == transcodeHWNVENC { - return true +// Both reasons describe the *configuration*, not the hardware, which is why the +// caller records them as skipped rather than failed: neither is evidence that a +// card stopped working. +func (c hwCandidates) unprobeableReason(backend, device string) (string, bool) { + if device == "" { + // NVENC's CUDA default, and the only shape a discovered candidate takes. + return "", false + } + if backend == transcodeHWNVENC { + // A render node path is a perfectly good hw_device for QSV or VAAPI and + // meaningless to CUDA. On a mixed host NVENC stays a candidate through + // hasNVIDIADevice even while the node is deliberately configured for + // QSV, and smoke-encoding CUDA against /dev/dri/renderD128 fails for a + // reason that has nothing to do with the NVIDIA card being fine. + if !isCUDADeviceIdentity(device) { + return "configured hw_device is a render node path, not a CUDA index or GPU uuid", true + } + // Otherwise unconditionally probeable: a CUDA index or uuid is not a + // file, so failing to open it is meaningless and only the smoke encode + // can answer. + return "", false } - return c.accessible[device] + if c.accessible == nil { + // Discovered candidates are openable by construction. + return "", false + } + if c.accessible[device] { + return "", false + } + return "device not accessible on this node", true +} + +// isCUDADeviceIdentity reports whether a configured device names a GPU the way +// CUDA does — an index, a "cuda:N", or a GPU uuid — rather than a DRM render +// node path. +func isCUDADeviceIdentity(device string) bool { + return !strings.ContainsRune(device, '/') } // deviceOpenable mirrors the accessibility filter listRenderDevices applies to diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index a60c8fb54..808c6dcab 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -296,3 +296,62 @@ func TestInvalidateHWProbeCacheRequeriesNVIDIAIdentities(t *testing.T) { t.Fatalf("identities = %v, want the re-probe to pick up the new uuid", got) } } + +// On a mixed NVIDIA/Intel host NVENC stays a candidate through hasNVIDIADevice +// even when the node is deliberately configured for QSV on a render node. Smoke +// encoding CUDA against /dev/dri/renderD128 fails for a reason that has nothing +// to do with the NVIDIA card, and a non-skipped failure latches a drift warning +// that cannot clear while the QSV policy stands. +func TestNVENCSkippedWhenTheConfiguredDeviceIsARenderPath(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addNVIDIADevice(t, "nvidia0") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult( + context.Background(), hwAccelAuto, ffmpeg.path, env.devicePath("renderD128")) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if !backend.Skipped { + t.Fatalf("nvenc = %+v, want it skipped rather than failed for a render-path device", backend) + } + if backend.Verified { + t.Fatalf("nvenc = %+v, want no verification claimed", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) +} + +// A CUDA identity is still probed: skipping is about the device being the wrong +// *kind* of name, not about avoiding NVENC. +func TestNVENCProbedWhenTheConfiguredDeviceIsACUDAIdentity(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + for _, device := range []string{"0", "cuda:1", "GPU-a1b2c3d4"} { + t.Run(device, func(t *testing.T) { + InvalidateHWProbeCache() + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, device) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if backend.Skipped || !backend.Verified { + t.Fatalf("nvenc = %+v, want a CUDA identity probed and verified", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) + }) + } +} diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index a609d68ba..8ed0b693c 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -25,6 +25,7 @@ func newMetricsProxyServer(t *testing.T) *Server { // newFakeSampler answers with a fixed reading so the handlers under test are // exercised without a Linux host beneath them. func newFakeSampler() *nodemetrics.Sampler { + video := 8 return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ Available: true, SampledAt: time.Now(), @@ -36,7 +37,7 @@ func newFakeSampler() *nodemetrics.Sampler { }, GPU: []nodemetrics.GPUStats{{ Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 1, - VideoBusyPct: 8, Source: nodemetrics.SourceFdinfo, + VideoBusyPct: &video, Source: nodemetrics.SourceFdinfo, }}, }) } diff --git a/internal/transcodenode/metrics_test.go b/internal/transcodenode/metrics_test.go index b52d39186..2d2c59fc8 100644 --- a/internal/transcodenode/metrics_test.go +++ b/internal/transcodenode/metrics_test.go @@ -14,6 +14,7 @@ import ( // newFakeSampler answers with a fixed reading so the handlers under test are // exercised without a Linux host beneath them. func newFakeSampler() *nodemetrics.Sampler { + video, render := 63, 12 return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ Available: true, SampledAt: time.Now(), @@ -25,7 +26,7 @@ func newFakeSampler() *nodemetrics.Sampler { }, GPU: []nodemetrics.GPUStats{{ Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, - VideoBusyPct: 63, RenderBusyPct: 12, Source: nodemetrics.SourceFdinfo, + VideoBusyPct: &video, RenderBusyPct: &render, Source: nodemetrics.SourceFdinfo, }}, }) } From 8e422ee8bc125a7b307eee46a96801f87e8eb374 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:51:16 -0400 Subject: [PATCH 035/163] fix(playback,nodemetrics): verify every allocatable GPU, measure writable disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three more review findings, each a case of a number or a guarantee that did not cover what it appeared to. A configured multi-device playback.hw_device is a set the balancer allocates *across*, but the detection walk stopped at the first device that passed. Every later entry went untested while the capability report said the backend was verified, and acquireHWDevice balanced onto them anyway — so a share of a node's transcodes started on a card that had already failed its smoke encode and died at ffmpeg init. A configured list is now probed in full, the verified set is per device rather than one device, and the balancer picks only from it. Discovered candidates are unchanged: they are alternatives, not an allocation pool, so probing the losers would cost FFmpeg launches and buy nothing. Disk capacity came from Blocks, which counts the blocks a filesystem reserves for root. A non-root process cannot write those, so on a default ext4 volume Bavail reaches zero at exactly used/total = 95% — the scratch admission threshold. The guard meant to keep five percent of headroom fired only once there was none left, which is the mid-stream write failure it exists to prevent. Capacity is now used plus available, so used/total is the ratio df prints as Use%. The disk probe budget is a global ceiling and a probe parked on a wedged mount holds its slot until the process exits. Offering the same paths in the same order every sample then spent the remaining budget on the same prefix, and the path at the end was never measured even once — reported unavailable indefinitely while being a disk that would have answered instantly. Scratch keeps its priority; the rest rotate. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 2 +- internal/nodemetrics/disk.go | 57 +++++++++- internal/nodemetrics/disk_test.go | 112 ++++++++++++++++++++ internal/nodemetrics/sampler.go | 4 + internal/nodemetrics/snapshot.go | 9 +- internal/nodemetrics/statfs_unix.go | 27 +++-- internal/playback/gpudetect.go | 92 +++++++++++++--- internal/playback/gpudetect_publish_test.go | 112 ++++++++++++++++++++ internal/playback/hwdevice.go | 38 ++++++- web/src/api/types.ts | 7 +- 10 files changed, 425 insertions(+), 35 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 6426ea21e..5c093837d 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -144,7 +144,7 @@ Each entry in `disks`: |---|---|---| | `path` | string | Where the mount is. Absent from a node's `last_stats`: the node reports it on its own bearer-authed `/status`, and the API host reports its own on `GET /admin/system/resources`, but a node's `/health` takes no credential and withholds it. Use `role`. | | `role` | string | What the mount is for: `scratch`, or `library-N` positionally per media root. Assigned when the sample is built, so it names the same mount on `/health`, `/status`, `/admin/system/resources` and the `streamapp_node_disk_*` series, and it stays with the mount even when a probe cannot measure it. | -| `used_gb`, `total_gb` | float | Capacity in GiB. Used counts filesystem-reserved blocks, matching `df`. | +| `used_gb`, `total_gb` | float | Capacity in GiB. `used_gb` counts filesystem-reserved blocks, as `df` does. `total_gb` is the capacity usable by the node process — used plus still-available — so it reads lower than the device's nameplate size on a volume that reserves blocks for root, and `used_gb`/`total_gb` is the ratio `df` prints as Use%. | | `stale` | bool | The numbers are real but carried over from an earlier pass because the current probe has not returned — the normal reading for a network mount whose server went away. Omitted when false. | | `unavailable` | bool | The path has never been measured on this node (it does not exist here, or the first probe is still hanging). `used_gb`/`total_gb` are meaningless. Omitted when false. | | `scratch` | bool | This is the node's transcode working directory. Set on at most one entry; a media root sharing that volume is deduplicated onto it. Omitted when false. | diff --git a/internal/nodemetrics/disk.go b/internal/nodemetrics/disk.go index de58255da..de52427bf 100644 --- a/internal/nodemetrics/disk.go +++ b/internal/nodemetrics/disk.go @@ -31,6 +31,21 @@ type fsStats struct { FSID string } +// fsCapacity converts raw statfs(2) block counts into the used/total shape this +// package reports. It is separate from the syscall so the arithmetic — which is +// where the reserved-block subtlety lives — can be exercised directly. +// +// A filesystem can only report more free blocks than it holds if the numbers +// are nonsense, so the subtraction is guarded rather than allowed to wrap an +// unsigned counter into a petabyte. +func fsCapacity(blocks, free, available, blockSize uint64) fsStats { + if free > blocks { + free = blocks + } + used := (blocks - free) * blockSize + return fsStats{UsedBytes: used, TotalBytes: used + available*blockSize} +} + // diskEntry is one path's probe state. Probes run detached from the sample // loop, so this holds the last good answer for readers to fall back to. type diskEntry struct { @@ -92,6 +107,7 @@ func (s *Sampler) refreshDisks(paths []string, now time.Time) { s.pruneDisksLocked(wanted) seen := make(map[string]bool, len(paths)) + candidates := make([]*diskEntry, 0, len(paths)) for _, path := range paths { if path == "" || seen[path] { continue @@ -101,14 +117,18 @@ func (s *Sampler) refreshDisks(paths []string, now time.Time) { if entry == nil { entry = &diskEntry{path: path} s.disks[path] = entry + // Reporting order is the order the caller offered, whatever order + // the probes below are started in. s.diskOrder = append(s.diskOrder, path) } + candidates = append(candidates, entry) + } + + for _, entry := range s.probeOrderLocked(candidates) { if entry.inFlight { continue } if s.probesInFlight >= maxOutstandingDiskProbes { - // Paths are offered scratch-first, so the mount admission control - // reads is the one that gets a freed slot first. s.noteProbeBudgetExhaustedLocked() break } @@ -119,6 +139,39 @@ func (s *Sampler) refreshDisks(paths []string, now time.Time) { } } +// probeOrderLocked is the order this sample offers entries to the probe budget. +// +// Scratch stays first: it is the mount transcode admission reads, so it is the +// one that must get a freed slot. The rest rotate, and that is not fairness for +// its own sake. The budget is a global ceiling, and a probe parked on a wedged +// mount holds its slot until the mount recovers or the process exits — so a +// deployment with one dead mount permanently runs one slot short. Offering the +// same list in the same order every sample would then spend the whole remaining +// budget on the same prefix and never reach the last path at all: it would be +// reported unavailable indefinitely, which is a lie about a disk that is fine +// and would have answered instantly. Advancing the start by one each sample +// costs a path its refresh roughly once per cycle and guarantees every mount is +// measured. +// +// Callers must hold diskMu. +func (s *Sampler) probeOrderLocked(candidates []*diskEntry) []*diskEntry { + rotatable := candidates + ordered := make([]*diskEntry, 0, len(candidates)) + if s.scratchDir != "" && len(candidates) > 0 && candidates[0].path == s.scratchDir { + ordered = append(ordered, candidates[0]) + rotatable = candidates[1:] + } + if len(rotatable) == 0 { + return ordered + } + start := s.diskProbeCursor % len(rotatable) + s.diskProbeCursor = (start + 1) % len(rotatable) + for i := range rotatable { + ordered = append(ordered, rotatable[(start+i)%len(rotatable)]) + } + return ordered +} + // noteProbeBudgetExhaustedLocked logs the first sample in which the probe // ceiling stopped new work, and nothing further until it clears. Every entry it // skips keeps reporting its last good numbers marked stale, so without a line diff --git a/internal/nodemetrics/disk_test.go b/internal/nodemetrics/disk_test.go index c8bde58f9..abb3e7419 100644 --- a/internal/nodemetrics/disk_test.go +++ b/internal/nodemetrics/disk_test.go @@ -493,3 +493,115 @@ func TestDiskProbesAreBoundedAcrossReconfiguration(t *testing.T) { t.Fatalf("outstanding probes = %d, want at most %d", outstanding, maxOutstandingDiskProbes) } } + +// A probe parked on a mount that has since been retired holds one global probe +// slot until the process exits. Offering the same paths in the same order every +// sample then spends the remaining budget on the same prefix, and the path at +// the end of the list is never measured even once — reported unavailable for as +// long as the dead mount stays dead, which is a lie about a healthy disk. +func TestRefreshDisksRotatesProbesPastAWedgedRetiredMount(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{ScratchDir: "/transcode"}) + done := make(chan string, 64) + s.diskProbeDone = done + + gate := make(chan struct{}) + t.Cleanup(func() { close(gate) }) + var mu sync.Mutex + probed := map[string]bool{} + s.statfs = func(path string) (fsStats, error) { + if path == "/dead" { + <-gate + return fsStats{}, os.ErrNotExist + } + mu.Lock() + probed[path] = true + mu.Unlock() + return fsStats{UsedBytes: 1 << 30, TotalBytes: 2 << 30, FSID: path}, nil + } + + now := clock.now() + // One sample parks a probe on a mount that the next configuration drops. + s.refreshDisks([]string{"/dead"}, now) + + live := []string{"/transcode"} + for i := range maxOutstandingDiskProbes - 1 { + live = append(live, "/media/"+strconv.Itoa(i)) + } + // live fills the ceiling exactly and one slot is gone for good, so each pass + // can start one fewer probe than there are paths. + perPass := len(live) - 1 + for range len(live) { + s.refreshDisks(live, now) + for range perPass { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a disk probe to complete") + } + } + } + + mu.Lock() + defer mu.Unlock() + for _, path := range live { + if !probed[path] { + t.Fatalf("%s was never probed across %d passes; the retired mount starved it", path, len(live)) + } + } +} + +// Scratch keeps its priority through the rotation: it is the mount transcode +// admission reads, so it must be the one that gets a freed slot. +func TestRefreshDisksAlwaysOffersScratchFirst(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{ScratchDir: "/transcode"}) + + candidates := []*diskEntry{{path: "/transcode"}, {path: "/a"}, {path: "/b"}, {path: "/c"}} + for pass := range 6 { + ordered := s.probeOrderLocked(candidates) + if ordered[0].path != "/transcode" { + t.Fatalf("pass %d offered %q first, want the scratch dir", pass, ordered[0].path) + } + if len(ordered) != len(candidates) { + t.Fatalf("pass %d offered %d entries, want all %d", pass, len(ordered), len(candidates)) + } + } +} + +// Capacity is measured against what this process can write, not against the +// device: a filesystem that reserves blocks for root hands them to nobody else, +// and counting them as headroom is what makes a volume with nothing left read +// as 95% full — exactly where the scratch admission guard sits. +func TestFSCapacityExcludesBlocksReservedFromThisProcess(t *testing.T) { + const block = 4096 + // 100 GiB with a 5% root reserve, filled to the point an unprivileged + // process can write nothing more. + total := uint64(100<<30) / block + reserved := total / 20 + stats := fsCapacity(total, reserved, 0, block) + + if stats.UsedBytes != (total-reserved)*block { + t.Fatalf("UsedBytes = %d, want the reserve counted as used, as df does", stats.UsedBytes) + } + if stats.TotalBytes != stats.UsedBytes { + t.Fatalf("TotalBytes = %d, want %d: no writable bytes remain", stats.TotalBytes, stats.UsedBytes) + } + + // An empty volume reports the whole writable capacity, which is the device + // size less the reserve rather than the device size. + empty := fsCapacity(total, total, total-reserved, block) + if empty.UsedBytes != 0 { + t.Fatalf("UsedBytes = %d on an empty volume, want 0", empty.UsedBytes) + } + if empty.TotalBytes != (total-reserved)*block { + t.Fatalf("TotalBytes = %d, want the reserve excluded from capacity", empty.TotalBytes) + } + + // Nonsense counters must not wrap an unsigned subtraction into a petabyte. + if got := fsCapacity(10, 20, 5, block); got.UsedBytes != 0 { + t.Fatalf("UsedBytes = %d for free > total, want 0", got.UsedBytes) + } +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 103e868f4..ccbd785a1 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -106,6 +106,10 @@ type Sampler struct { // probesInFlight is how many statfs goroutines are outstanding right now, // bounded by maxOutstandingDiskProbes. Guarded by diskMu. probesInFlight int + // diskProbeCursor rotates which non-scratch mount is offered a probe slot + // first, so a slot parked forever on a wedged mount cannot starve the path + // at the end of the list. See probeOrderLocked. + diskProbeCursor int // probeBudgetExhausted latches the ceiling warning to one line per episode. probeBudgetExhausted bool // diskProbeDone, when non-nil, receives each completed probe's path. Tests diff --git a/internal/nodemetrics/snapshot.go b/internal/nodemetrics/snapshot.go index 1c08865e0..890007724 100644 --- a/internal/nodemetrics/snapshot.go +++ b/internal/nodemetrics/snapshot.go @@ -80,8 +80,13 @@ type DiskStats struct { // for each media root. It is assigned when the sample is built, so every // surface — health, metrics, the admin API — names a mount identically, and // the index stays with the mount even when a probe cannot measure it. - Role string `json:"role,omitempty"` - UsedGB float64 `json:"used_gb"` + Role string `json:"role,omitempty"` + UsedGB float64 `json:"used_gb"` + // TotalGB is the capacity usable by this process: bytes in use plus bytes + // still available to it. Blocks a filesystem reserves for root are in + // neither, so this reads lower than the device's nameplate size on a default + // ext4 volume — and UsedGB/TotalGB is the ratio `df` prints as Use%, which + // is what makes a full volume read as full rather than as 95%. TotalGB float64 `json:"total_gb"` // Stale marks numbers carried over from an earlier pass because the current // probe has not returned — the normal reading for a network mount whose diff --git a/internal/nodemetrics/statfs_unix.go b/internal/nodemetrics/statfs_unix.go index aa4b94651..eea12b77e 100644 --- a/internal/nodemetrics/statfs_unix.go +++ b/internal/nodemetrics/statfs_unix.go @@ -6,12 +6,20 @@ import ( "golang.org/x/sys/unix" ) -// osStatfs reports one path's filesystem capacity. +// osStatfs reports one path's filesystem capacity, in the terms that matter to +// a process deciding whether it can keep writing. // -// Used space is computed from blocks the filesystem considers free, not from -// the free space available to an unprivileged user (Bavail): the reserved -// margin is genuinely occupied capacity, and reporting it as used is what makes -// the number match what an operator sees in `df`. +// Used counts every block the filesystem considers taken (Blocks-Bfree), which +// is what `df` puts in its Used column. Total is that plus the blocks still +// available to *this* process (Bavail) — deliberately not the raw device size. +// A filesystem holds blocks back from unprivileged users (ext4 reserves 5% by +// default), and those are writable by root alone: counting them as capacity +// makes a volume with nothing left for Silo to write read as 95% full, exactly +// where the scratch admission guard is set. A node would then admit sessions +// until the moment it has zero bytes of headroom and start failing transcodes +// mid-stream, which is the failure that guard exists to prevent. +// +// The resulting Used/Total is the same ratio `df` prints as Use%. // // This call can block indefinitely on an unresponsive network mount. Callers // must treat it as such — see probeDisk, which runs it on a goroutine nothing @@ -21,10 +29,7 @@ func osStatfs(path string) (fsStats, error) { if err := unix.Statfs(path, &st); err != nil { return fsStats{}, err } - blockSize := uint64(st.Bsize) - return fsStats{ - UsedBytes: (st.Blocks - st.Bfree) * blockSize, - TotalBytes: st.Blocks * blockSize, - FSID: formatFSID(int64(st.Fsid.Val[0]), int64(st.Fsid.Val[1])), - }, nil + stats := fsCapacity(st.Blocks, st.Bfree, st.Bavail, uint64(st.Bsize)) + stats.FSID = formatFSID(int64(st.Fsid.Val[0]), int64(st.Fsid.Val[1])) + return stats, nil } diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 00e83f233..f64f7076c 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "sort" "strconv" "strings" @@ -68,13 +69,15 @@ var hwProbeCache = struct { // a caller arriving afterwards asks a different key and therefore starts a // fresh probe instead of joining the stale one. generation uint64 - // verifiedDevices records, per generation and backend, the candidate device - // whose smoke encode passed. Execution reads it so a backend verified on - // one render node is not then run on another; see VerifiedHWDevice. - verifiedDevices map[string]string + // verifiedDevices records, per generation and backend, every candidate + // device whose smoke encode passed, in probe order. Execution reads it so a + // backend verified on one render node is not then run on another, and so + // balancing across a configured multi-device list cannot land a workload on + // a card no probe ever passed; see VerifiedHWDevice and VerifiedHWDevices. + verifiedDevices map[string][]string }{ entries: make(map[string]hwProbeCacheEntry), - verifiedDevices: make(map[string]string), + verifiedDevices: make(map[string][]string), } // DetectedBackend reports one hardware backend that has candidate devices on @@ -420,9 +423,18 @@ func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCa return resolved, detected, complete } -// verifyHWAccelBackend probes a backend's candidate devices in order and stops -// at the first one that passes, so a broken GPU sorting ahead of a working one -// does not disable the backend for the whole host. +// verifyHWAccelBackend probes a backend's candidate devices in order. A broken +// GPU sorting ahead of a working one does not disable the backend for the whole +// host: the first device that passes decides the backend's verdict. +// +// Whether the walk continues past that device depends on what the rest of the +// list is for. Discovered candidates are alternatives — execution adopts the one +// device detection verified — so probing the losers costs FFmpeg launches and +// buys nothing. A configured multi-device playback.hw_device is different: it is +// a set the device balancer allocates *across*, so every entry is a device a +// transcode can be handed, and stopping early would leave the inventory +// vouching for cards nothing ever tested while acquireHWDevice happily balanced +// onto them. // // complete reports whether every candidate was reached. A device left unprobed // because the budget ran out is not a device that failed, and the difference is @@ -433,6 +445,7 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi reasons := make([]string, 0, len(devices)) probed := false complete = true + probeEveryDevice := candidates.allocatesAcross(backend) // Captured before any probe runs: a verdict earned under this generation is // only worth recording if no invalidation has landed by the time it lands. generation := hwProbeGeneration() @@ -448,16 +461,27 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi probed = true available, reason := ffmpegSupportsBackendContext(ctx, backend, ffmpegPath, device) if available { - entry.Verified = true - entry.Device = device - // Execution has to land on this device and not on whatever sorts - // first under /dev/dri, or a report saying "qsv verified" is paired - // with a transcode initializing a GPU the probe never touched. + // Execution has to land on a device a probe passed and not on + // whatever sorts first under /dev/dri, or a report saying "qsv + // verified" is paired with a transcode initializing a GPU the probe + // never touched. recordVerifiedHWDevice(generation, backend, device) - return entry, complete + if !entry.Verified { + entry.Verified = true + entry.Device = device + } + if !probeEveryDevice { + return entry, complete + } + continue } reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) } + if entry.Verified { + // The reasons collected past the first pass belong to devices the + // balancer will now skip, not to a backend that failed. + return entry, complete + } if len(reasons) == 0 { reasons = append(reasons, "hardware detection budget exhausted before probing "+backend) } @@ -494,7 +518,11 @@ func recordVerifiedHWDevice(generation uint64, backend, device string) { if hwProbeCache.generation != generation { return } - hwProbeCache.verifiedDevices[verifiedHWDeviceKey(generation, backend)] = device + key := verifiedHWDeviceKey(generation, backend) + if slices.Contains(hwProbeCache.verifiedDevices[key], device) { + return + } + hwProbeCache.verifiedDevices[key] = append(hwProbeCache.verifiedDevices[key], device) } // VerifiedHWDevice returns the render device this process most recently @@ -509,15 +537,45 @@ func recordVerifiedHWDevice(generation uint64, backend, device string) { // never verified. Reading the verified device closes that gap without making // every caller of resolution carry a second return value. func VerifiedHWDevice(backend string) string { + devices := VerifiedHWDevices(backend) + if len(devices) == 0 { + return "" + } + return devices[0] +} + +// VerifiedHWDevices returns every device this process has verified the given +// backend on, in probe order, or nil when no probe has passed for it. +// +// It has more than one entry only for a configured multi-device +// playback.hw_device: detection stops at the first pass when it is choosing a +// backend, but a configured list is *allocated* across, so every entry in it +// has to be probed or the inventory would vouch for cards nothing tested. The +// device balancer intersects its candidates with this set, which is what keeps +// a workload off a card that is present but broken. +func VerifiedHWDevices(backend string) []string { hwProbeCache.Lock() defer hwProbeCache.Unlock() - return hwProbeCache.verifiedDevices[verifiedHWDeviceKey(hwProbeCache.generation, backend)] + return slices.Clone(hwProbeCache.verifiedDevices[verifiedHWDeviceKey(hwProbeCache.generation, backend)]) } func verifiedHWDeviceKey(generation uint64, backend string) string { return strconv.FormatUint(generation, 10) + "\x00" + backend } +// allocatesAcross reports whether every one of a backend's candidate devices is +// one execution may be handed, rather than an alternative detection chooses +// between. +// +// That is true only for a configured multi-device playback.hw_device, which is +// what accessible being non-nil marks. NVENC is never balanced across a list — +// acquireHWDevice hands it the first configured entry — and discovered +// candidates are alternatives, resolved to the single device VerifiedHWDevice +// reports. +func (c hwCandidates) allocatesAcross(backend string) bool { + return backend != transcodeHWNVENC && c.accessible != nil && len(c.devicesFor(backend)) > 1 +} + // unprobeableReason reports why a candidate cannot be smoke-encoded on, or // false when it can be. // @@ -700,7 +758,7 @@ func InvalidateHWProbeCache() { defer hwProbeCache.Unlock() hwProbeCache.generation++ hwProbeCache.entries = make(map[string]hwProbeCacheEntry) - hwProbeCache.verifiedDevices = make(map[string]string) + hwProbeCache.verifiedDevices = make(map[string][]string) // The GPU identity listing goes too. A re-probe is exactly when nvidia-smi // may have become available, or a card in the same slot may have been // replaced, and both of those change identities the drift comparison and diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 808c6dcab..4ccc7b5fd 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -355,3 +356,114 @@ func TestNVENCProbedWhenTheConfiguredDeviceIsACUDAIdentity(t *testing.T) { }) } } + +// A configured multi-device hw_device is a set the balancer allocates *across*, +// not a list of alternatives detection picks from. Stopping the walk at the +// first pass left every later entry untested while the report said the backend +// was verified, and acquireHWDevice balanced onto them regardless — so a share +// of the node's transcodes started on a card that had already failed its smoke +// encode, and each of them died at ffmpeg init. +func TestConfiguredDeviceListIsProbedInFullAndBalancedOnlyAcrossPasses(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + probe := successfulQSVProbe() + probe.smokeDeviceFailures = []string{"renderD129"} + ffmpeg := writeFakeFFmpeg(t, probe) + + working, broken := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := working + "," + broken + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, configured) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWQSV { + t.Fatalf("resolved = %q, want qsv from the device that passed", info.Resolved) + } + + if got := VerifiedHWDevices(transcodeHWQSV); !slices.Equal(got, []string{working}) { + t.Fatalf("verified devices = %v, want only the card whose probe passed", got) + } + + // Ten acquisitions is far more than the balancer needs to reach a second + // device: with both present it alternates on the very next one. + releases := make([]func(), 0, 10) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for i := range 10 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + if device != working { + t.Fatalf("acquisition %d selected %q, want the verified device %q", i, device, working) + } + } +} + +// With nothing verified — a cold process, or hw_accel named explicitly so the +// walk never ran — the balancer must not narrow to an empty set and must keep +// using every present device exactly as before. +func TestBalancingIsUnchangedWhenNoDeviceHasBeenVerified(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + first, second := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := first + "," + second + + selected := map[string]bool{} + releases := make([]func(), 0, 2) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range 2 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + selected[device] = true + } + if len(selected) != 2 { + t.Fatalf("selected %v, want both devices used when no probe has ruled either out", selected) + } +} + +// The narrowing above is only safe because the whole configured list is probed. +// If the walk stopped at the first pass, every other card in the set would be +// unverified and the balancer would collapse a multi-GPU node onto one device — +// trading a correctness bug for a capacity one. +func TestEveryConfiguredDeviceThatPassesStaysInTheBalancer(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + + first, second := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := first + "," + second + + if _, err := DetectHWAccelWithFFmpegContextResult( + context.Background(), hwAccelAuto, ffmpeg.path, configured); err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if got := VerifiedHWDevices(transcodeHWQSV); !slices.Equal(got, []string{first, second}) { + t.Fatalf("verified devices = %v, want both cards in the configured set", got) + } + + selected := map[string]bool{} + releases := make([]func(), 0, 2) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range 2 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + selected[device] = true + } + if len(selected) != 2 { + t.Fatalf("selected %v, want the workload spread across both verified cards", selected) + } +} diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index 0e1ec8e68..e81473d50 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -3,6 +3,7 @@ package playback import ( "log/slog" "os" + "slices" "strconv" "strings" "sync" @@ -283,7 +284,7 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, w } // Select and reserve in one critical section so concurrent workload starts // observe each other's reservations instead of piling onto one device. - present := presentHWDevices(set.List()) + present := verifiedHWDevices(resolvedHWAccel, presentHWDevices(set.List())) if len(present) > 1 && avoidDevice != "" { eligible := make([]string, 0, len(present)-1) for _, device := range present { @@ -305,6 +306,41 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, w return selected, selected, newHWDeviceRelease(selected) } +// verifiedHWDevices narrows a configured device list to the entries whose smoke +// encode actually passed. +// +// Presence is not fitness. A device that exists and can be opened can still fail +// to initialize the backend — a card in a bad state, a driver mismatch, a +// container that mapped the node without the matching libraries — and detection +// already found that out. Balancing across every present entry would hand a +// share of the node's workloads to that card and fail each of them at startup, +// while the capability report shows the backend verified. +// +// It narrows only when there is something to narrow to. An empty verified set +// means detection never ran for this backend (a cold process, or hw_accel named +// explicitly so resolution short-circuited), which is no evidence against any +// device, so the full present list stands. +func verifiedHWDevices(resolvedHWAccel string, present []string) []string { + verified := VerifiedHWDevices(resolvedHWAccel) + if len(verified) == 0 { + return present + } + eligible := make([]string, 0, len(present)) + for _, device := range present { + if slices.Contains(verified, device) { + eligible = append(eligible, device) + } + } + if len(eligible) == 0 { + // Every present device failed its probe, or the probe set and the + // configured set have drifted apart. Excluding everything would leave + // the balancer with nothing to pick, which is worse than letting ffmpeg + // try and report a real error. + return present + } + return eligible +} + // hwDeviceActiveCount reports the active workload count for one device; test // helper for asserting release boundaries. func hwDeviceActiveCount(device string) int { diff --git a/web/src/api/types.ts b/web/src/api/types.ts index aa56a1451..5d7b07245 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3848,8 +3848,13 @@ export interface HostDiskStats { * sample is built, so it names the same mount on every surface. */ role?: string; - /** Capacity in GiB. Used counts filesystem-reserved blocks, matching `df`. */ + /** Capacity in GiB. `used_gb` counts filesystem-reserved blocks, as `df` does. */ used_gb?: number; + /** + * Capacity usable by the node process — used plus still-available. Blocks a + * filesystem reserves for root are in neither, so this reads lower than the + * device's nameplate size, and `used_gb`/`total_gb` is `df`'s Use%. + */ total_gb?: number; /** Real numbers carried over from an earlier pass because the probe has not returned. */ stale?: boolean; From b134fa295703449b37d81b2810f9aadbfb278450 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:11:52 -0400 Subject: [PATCH 036/163] fix(nodes): key CUDA-only GPUs, survive request cancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings; a third in the same batch was wrong. A card is not always reachable through a DRM render node. An NVIDIA container is routinely given /dev/nvidia* and the toolkit with no /dev/dri at all: NVENC works, render_device_details is empty, and the whole host contributed no physical GPU key. Two such containers sharing one card read as two independent GPUs, so the shared-GPU tie-break kept piling work onto the same hardware — in the deployment shape where GPU sharing is most common. The report now carries nvidia_gpu_uuids independent of render devices, physicalGPUKeys consumes them, and the capability hash covers them so a card appearing or disappearing still forces a refetch. A uuid is host-independent, so a card reported both ways yields one key and a container with /dev/dri recognizes the same GPU as one without. reloadPools ran on the request context after the response was already written. An admin whose browser gave up on a slow save cancelled it, the database reads then failed, and EventNodePoolChanged was never published — leaving this instance and every replica dispatching under the old acceleration policy indefinitely, since nothing else re-reads the column. The row was committed, so the write and its publication were two outcomes where they had to be one. Both it and the worker nudge now run detached from the request, bounded. Fixed inside reloadPools rather than at its three call sites, all of which are post-response. Not changed: Codex reported a P1 nil dereference on deps.NodeHealthChecker.SetCapabilitiesChangedCallback in router.go. The method guards its own nil receiver, and internal/api passes — the finding also cites two file paths that do not exist in this repository. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 14 ++++- internal/api/handlers/nodes.go | 23 +++++++- internal/api/handlers/nodes_test.go | 62 +++++++++++++++++++++ internal/nodepool/gpuidentity.go | 34 ++++++++--- internal/nodepool/gpuidentity_test.go | 25 +++++++++ internal/playback/capabilityhash.go | 2 + internal/playback/gpudetect.go | 28 ++++++++++ internal/playback/gpudetect_publish_test.go | 40 +++++++++++++ web/src/api/types.ts | 6 ++ 9 files changed, 222 insertions(+), 12 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 5c093837d..f886a2d4d 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -292,15 +292,22 @@ probe results are cached for its process lifetime. `POST ### `physical_gpu_keys` -One key per render device in the stored report, deduplicated and sorted: +One key per GPU in the stored report, deduplicated and sorted. From each render +device: - the device's `gpu_uuid` when present (NVIDIA's permanent GPU identity, which follows the card between slots and hosts), otherwise - `|`, because a PCI slot only means the same hardware within one boot of one kernel. -A device with neither contributes no key rather than a synthetic one, and so -does a slot on a host that reported no `boot_id`: `boot_id` detection is +Plus every entry in `nvidia_gpu_uuids`, which is what covers a card with no +readable DRM node — the ordinary NVIDIA container, where NVENC works and +`render_device_details` is empty. A uuid is host-independent, so a card reported +both ways yields one key, and a container that sees only `/dev/nvidia*` and one +that also sees `/dev/dri` recognize the same physical GPU. + +A device with neither identity contributes no key rather than a synthetic one, +and so does a slot on a host that reported no `boot_id`: `boot_id` detection is best-effort, and an unscoped slot is not an identity, since every host with an Intel iGPU has one at `0000:00:02.0`. Two nodes sharing a key are backed by the same physical GPU — the case that makes per-node capacity accounting wrong, and @@ -477,6 +484,7 @@ Top-level (and each node's own report): | `intel_detected` | bool | An Intel GPU is present in the inventory. | | `detected_backends` | object[] | One entry per backend that had candidate hardware, with the outcome of its FFmpeg verification (see below). | | `boot_id` | string | The host's kernel boot identity (Linux only). Pairs with a device's `pci_address` to distinguish the same GPU from the same slot after a reboot. | +| `nvidia_gpu_uuids` | string[] | Every GPU `nvidia-smi` reports on this host, sorted. Independent of `render_device_details`, because a card is not always reachable through a DRM node — an NVIDIA container is routinely given `/dev/nvidia*` and the toolkit with no `/dev/dri` at all. Omitted where `nvidia-smi` is absent. | | `capability_hash` | string | `sha256:` over this report's hardware identity and capability fields — not over `source`, `node_url`, the probe budget, or itself. Two reports of unchanged hardware hash identically regardless of probe order. | | `source` | string | `local` for a probe of this host. | | `node_url` | string | Set on a node's report. | diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index c9e792593..32c749f79 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -211,7 +211,10 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { // (QSV on a render node to NVENC on a CUDA index, say) gets up to a minute // of requests pairing the new backend with the old device. if nodeAccelerationChanged(previous, node) { - if !h.reloadNodeConfig(r.Context(), node) { + // Detached for the same reason reloadPools is: the response is already + // written, so a client that has gone away must not decide whether the + // worker hears about its new policy. + if !h.reloadNodeConfig(context.WithoutCancel(r.Context()), node) { // The node did not confirm. Its backend now comes from this // server's pool while its device still comes from its own // configuration, so until its poll catches up (within 60s) a start @@ -740,11 +743,27 @@ func (h *NodeHandler) HandleListSessions(w http.ResponseWriter, r *http.Request) writeJSON(w, http.StatusOK, sessionsResponse{Sessions: sessions}) } -// reloadPools refreshes the in-memory proxy and transcode pools from the database. +// nodePostCommitTimeout bounds work that runs after the response is written and +// therefore no longer has a client waiting on it. +const nodePostCommitTimeout = 15 * time.Second + +// reloadPools refreshes the in-memory proxy and transcode pools from the +// database and tells every replica to do the same. +// +// Every caller reaches here after its response is written, so the request +// context is the wrong lifetime: a client that disconnects — or an admin who +// navigates away from a slow save — cancels it, and this would then fail both +// database reads and return without publishing EventNodePoolChanged. The row is +// already committed at that point, so this instance and every replica would go +// on dispatching under the old policy indefinitely; nothing else re-reads the +// column. Detaching from that cancellation, bounded, is what makes the write and +// its publication one outcome instead of two. func (h *NodeHandler) reloadPools(ctx context.Context) { if h.lister == nil { return } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), nodePostCommitTimeout) + defer cancel() proxyNodes, proxyErr := h.lister.ListEnabled(ctx, nodepool.NodeTypeProxy) transcodeNodes, tcErr := h.lister.ListEnabled(ctx, nodepool.NodeTypeTranscode) if proxyErr != nil || tcErr != nil { diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 83640a92f..ce7a39ef5 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -6,10 +6,12 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" + "github.com/Silo-Server/silo-server/internal/cache" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/go-chi/chi/v5" ) @@ -462,3 +464,63 @@ func TestHandleUpdateNodeDoesNotReloadWithoutAnOverrideChange(t *testing.T) { default: } } + +// recordingEventBus captures publications so a test can assert the pool change +// actually reached the other replicas. +type recordingEventBus struct { + mu sync.Mutex + events []cache.Event +} + +func (b *recordingEventBus) Publish(_ context.Context, _ string, event cache.Event) error { + b.mu.Lock() + defer b.mu.Unlock() + b.events = append(b.events, event) + return nil +} + +func (b *recordingEventBus) Subscribe(context.Context, string, cache.EventHandler) error { return nil } +func (b *recordingEventBus) Close() error { return nil } + +func (b *recordingEventBus) types() []string { + b.mu.Lock() + defer b.mu.Unlock() + types := make([]string, 0, len(b.events)) + for _, event := range b.events { + types = append(types, event.Type) + } + return types +} + +// ctxAwareLister answers only for a live context, the way a database read does. +type ctxAwareLister struct{ nodes []*nodepool.Node } + +func (l *ctxAwareLister) ListEnabled(ctx context.Context, _ string) ([]*nodepool.Node, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return l.nodes, nil +} + +// The pool reload runs after the response is written, so the request context is +// no longer the right lifetime: an admin whose browser gives up on a slow save +// cancels it, and the reload would then fail its reads and never publish. The +// row is already committed at that point, so this instance and every replica +// would keep dispatching under the old acceleration policy indefinitely — +// nothing else re-reads the column. +func TestReloadPoolsSurvivesRequestCancellation(t *testing.T) { + bus := &recordingEventBus{} + lister := &ctxAwareLister{nodes: []*nodepool.Node{{ID: 1, URL: "http://node", Enabled: true}}} + handler := NewNodeHandler(&stubNodeRepository{}, nodepool.NewProxyPool(), nodepool.NewTranscodePool(), lister, bus, nil, "secret") + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + handler.reloadPools(canceled) + + if got := bus.types(); len(got) != 1 || got[0] != string(cache.EventNodePoolChanged) { + t.Fatalf("published %v, want a single %q after a canceled request", got, cache.EventNodePoolChanged) + } + if got := handler.transcodePool.Nodes(); len(got) != 1 { + t.Fatalf("transcode pool holds %d nodes, want the reload to have landed", len(got)) + } +} diff --git a/internal/nodepool/gpuidentity.go b/internal/nodepool/gpuidentity.go index 20aee8843..d212e1e8f 100644 --- a/internal/nodepool/gpuidentity.go +++ b/internal/nodepool/gpuidentity.go @@ -15,6 +15,12 @@ type gpuIdentityView struct { PCIAddress string `json:"pci_address"` GPUUUID string `json:"gpu_uuid"` } `json:"render_device_details"` + // NVIDIAGPUUUIDs covers cards with no readable DRM node, which is the + // ordinary shape of an NVIDIA container: /dev/nvidia* and the toolkit, no + // /dev/dri. NVENC works there and render_device_details is empty, so + // without this the host contributes no identity at all and two containers + // on one card read as two GPUs. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` } // physicalGPUKeys derives one stable key per GPU a node can see, deduplicated @@ -45,8 +51,19 @@ func physicalGPUKeys(capabilities []byte) []string { if err := json.Unmarshal(capabilities, &identity); err != nil { return nil } - seen := make(map[string]struct{}, len(identity.RenderDeviceDetails)) - keys := make([]string, 0, len(identity.RenderDeviceDetails)) + total := len(identity.RenderDeviceDetails) + len(identity.NVIDIAGPUUUIDs) + seen := make(map[string]struct{}, total) + keys := make([]string, 0, total) + add := func(key string) { + if key == "" { + return + } + if _, duplicate := seen[key]; duplicate { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } for _, device := range identity.RenderDeviceDetails { key := device.GPUUUID if key == "" { @@ -55,11 +72,14 @@ func physicalGPUKeys(capabilities []byte) []string { } key = identity.BootID + "|" + device.PCIAddress } - if _, duplicate := seen[key]; duplicate { - continue - } - seen[key] = struct{}{} - keys = append(keys, key) + add(key) + } + // A uuid is host-independent, so a card reported only through nvidia-smi + // keys the same way whether or not it also has a render node — which is + // what lets a container with /dev/dri and one without recognize the same + // physical GPU. + for _, uuid := range identity.NVIDIAGPUUUIDs { + add(uuid) } if len(keys) == 0 { return nil diff --git a/internal/nodepool/gpuidentity_test.go b/internal/nodepool/gpuidentity_test.go index 0933d2416..382c29375 100644 --- a/internal/nodepool/gpuidentity_test.go +++ b/internal/nodepool/gpuidentity_test.go @@ -58,6 +58,31 @@ func TestPhysicalGPUKeys(t *testing.T) { capabilities: `{"render_device_details":[{"pci_address":"0000:03:00.0","gpu_uuid":"GPU-ddd"}]}`, want: []string{"GPU-ddd"}, }, + { + // The ordinary NVIDIA container: /dev/nvidia* and the toolkit, no + // /dev/dri at all. NVENC works, render_device_details is empty, and + // without the uuid list the whole host contributes no identity — so + // two containers on one card read as two independent GPUs and the + // shared-GPU tie-break keeps piling work onto the same hardware. + name: "cuda-only host keys by uuid with no render device", + capabilities: `{"boot_id":"boot-1","render_device_details":[],"nvidia_gpu_uuids":["GPU-eee","GPU-fff"]}`, + want: []string{"GPU-eee", "GPU-fff"}, + }, + { + // A uuid is host-independent, so the card a container reaches only + // through nvidia-smi keys identically to the one its neighbor also + // sees through a render node. That identity is the whole point. + name: "a card reported both ways contributes one key", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-ggg"}], + "nvidia_gpu_uuids":["GPU-ggg"]}`, + want: []string{"GPU-ggg"}, + }, + { + name: "unparseable uuid list yields no keys", + capabilities: `{"nvidia_gpu_uuids":"nope"}`, + want: nil, + }, {name: "no capabilities stored", capabilities: "", want: nil}, {name: "unparseable payload", capabilities: `not json`, want: nil}, {name: "payload of the wrong shape", capabilities: `{"render_device_details":"nope"}`, want: nil}, diff --git a/internal/playback/capabilityhash.go b/internal/playback/capabilityhash.go index a10313183..181114200 100644 --- a/internal/playback/capabilityhash.go +++ b/internal/playback/capabilityhash.go @@ -41,6 +41,7 @@ type canonicalCapability struct { BootID string `json:"boot_id"` RenderDevices []string `json:"render_devices"` RenderDeviceDetails []canonicalRenderDevice `json:"render_device_details"` + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` DetectedBackends []canonicalDetectedBackend `json:"detected_backends"` Transformations []canonicalTransformation `json:"transformations"` ToneMapCapabilities []canonicalToneMap `json:"tone_map_capabilities"` @@ -85,6 +86,7 @@ func canonicalCapabilities(info HWAccelInfo) canonicalCapability { BootID: info.BootID, RenderDevices: sortedStrings(info.RenderDevices), RenderDeviceDetails: canonicalRenderDevices(info.RenderDeviceDetails), + NVIDIAGPUUUIDs: sortedStrings(info.NVIDIAGPUUUIDs), DetectedBackends: canonicalDetectedBackends(info.DetectedBackends), Transformations: canonicalTransformations(info.Transformations), ToneMapCapabilities: canonicalToneMaps(info.ToneMapCapabilities), diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index f64f7076c..772f1deaa 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -115,6 +115,16 @@ type HWAccelInfo struct { // render device's PCI address it distinguishes "same GPU, same boot" from // "same device path on a host that rebooted or was replaced". BootID string `json:"boot_id,omitempty"` + // NVIDIAGPUUUIDs lists every GPU nvidia-smi reports on this host, sorted. + // + // It exists because a card is not always reachable through a DRM render + // node. An NVIDIA container is routinely given /dev/nvidia* and the toolkit + // with no /dev/dri at all: NVENC works, RenderDeviceDetails is empty, and + // the whole host would otherwise contribute no hardware identity. Two such + // containers sharing one card would then look like two independent GPUs to + // the planner — which is precisely the deployment where GPU sharing is most + // common, and the placement mistake most expensive. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids,omitempty"` // CapabilityHash summarizes every hardware-identity and capability field // below, so a reader can detect change without diffing the whole report. // Set by the node that serves the report; see ComputeCapabilityHash. @@ -199,6 +209,7 @@ func DetectHWAccelWithFFmpegContextResult(ctx context.Context, hwAccel, ffmpegPa IntelDetected: candidates.intelPresent, DetectedBackends: detected, BootID: detectBootID(), + NVIDIAGPUUUIDs: nvidiaGPUUUIDList(), Source: "local", } if !complete { @@ -1227,6 +1238,23 @@ func nvidiaGPUUUIDsByPCIAddress() map[string]string { return nvidiaGPUUUIDs.byPCI } +// nvidiaGPUUUIDList returns every uuid nvidia-smi reports, sorted and +// deduplicated, independent of whether the card has a readable render node. +func nvidiaGPUUUIDList() []string { + byPCI := nvidiaGPUUUIDsByPCIAddress() + if len(byPCI) == 0 { + return nil + } + uuids := make([]string, 0, len(byPCI)) + for _, uuid := range byPCI { + if uuid != "" && !slices.Contains(uuids, uuid) { + uuids = append(uuids, uuid) + } + } + slices.Sort(uuids) + return uuids +} + // parseNVIDIAGPUUUIDs reads "csv,noheader" rows of ", " and // keys them by normalized PCI address. Malformed rows are skipped. func parseNVIDIAGPUUUIDs(output []byte) map[string]string { diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 4ccc7b5fd..593ce4de6 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -467,3 +467,43 @@ func TestEveryConfiguredDeviceThatPassesStaysInTheBalancer(t *testing.T) { t.Fatalf("selected %v, want the workload spread across both verified cards", selected) } } + +// An NVIDIA container is routinely given /dev/nvidia* and the toolkit with no +// /dev/dri at all: NVENC works and render_device_details is empty. Without a +// standalone uuid list the whole host contributes no hardware identity, so two +// such containers on one card look like two independent GPUs to the planner — +// which is the deployment where GPU sharing is most common and the placement +// mistake most expensive. +func TestCUDAOnlyHostPublishesItsGPUIdentities(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + return []byte("GPU-aaa, 00000000:03:00.0\nGPU-bbb, 00000000:04:00.0\n"), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if len(info.RenderDeviceDetails) != 0 { + t.Fatalf("render devices = %+v, want none on a container with no /dev/dri", info.RenderDeviceDetails) + } + if !slices.Equal(info.NVIDIAGPUUUIDs, []string{"GPU-aaa", "GPU-bbb"}) { + t.Fatalf("nvidia gpu uuids = %v, want both cards nvidia-smi reported", info.NVIDIAGPUUUIDs) + } + + // A card appearing or disappearing has to move the hash, or a node that + // gained or lost one is never refetched. + withOne := info + withOne.NVIDIAGPUUUIDs = []string{"GPU-aaa"} + if ComputeCapabilityHash(info) == ComputeCapabilityHash(withOne) { + t.Fatal("capability hash ignores the GPU identity list; a lost card would never trigger a refetch") + } +} diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 5d7b07245..e2ecbc3ca 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3827,6 +3827,12 @@ export interface NodeCapabilities { detected_backends?: NodeDetectedBackend[]; /** Kernel boot identity (Linux only); scopes pci_address to one boot. */ boot_id?: string; + /** + * Every GPU nvidia-smi reports, sorted. Independent of + * render_device_details: an NVIDIA container often has /dev/nvidia* and the + * toolkit but no /dev/dri, so this is the only identity such a host has. + */ + nvidia_gpu_uuids?: string[]; /** "sha256:" over the report's hardware identity and capabilities. */ capability_hash?: string; source?: string; From f7b6d30c3b621acd817ae3240689f493ac6bf604 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:18:53 -0400 Subject: [PATCH 037/163] fix(nodes): store the node's own capability bytes, not a re-marshal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stored payload was this server's re-marshaling of the decoded HWAccelInfo, which drops every field the reading build has no struct member for. That is not hypothetical: it is every rolling upgrade where a node is newer than the API server. The truncation was then filed under the node's own hash, so once the API was upgraded the sweep saw the hashes agree and never refetched — the durable inventory stayed missing fields the new code reads, until an unrelated capability change or a manual re-probe moved the hash. The nvidia_gpu_uuids field added in the previous commit is a concrete instance: an API one commit older would have stored a payload without it and kept the node's hash, and the shared-GPU tie-break would still be blind after that API was upgraded. FetchHWCapabilitiesPayload returns the response bytes alongside the decoded report, for callers that persist rather than read. They are already bounded by maxHWCapabilitiesResponseBytes and parsed before return, so storing them verbatim is safe. The other three callers, which only read, keep the existing signature. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 23 ++++++----- cmd/silo/main_test.go | 43 +++++++++++++++++++++ internal/transcodenode/capability_client.go | 31 +++++++++++---- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 35163c334..7a90c9797 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -141,26 +141,31 @@ func resolveNodeIdentity() string { } // nodeCapabilityFetcher adapts the authenticated node capability client to the -// node health sweep, which stores capability reports opaquely. The stored -// payload is this server's re-marshaling of the decoded report rather than the -// node's bytes, so what is persisted is exactly what the API understood; the -// hash comes out of the payload itself, because only the node knows what it +// node health sweep, which stores capability reports opaquely. +// +// What is persisted is the node's own response bytes, not this server's +// re-marshaling of the decoded struct. The two differ exactly when the node is +// newer than the API server reading it, which is every rolling upgrade: a +// re-marshal drops the fields this build has no struct member for, and stores +// the truncation under the node's hash. After the API is upgraded the sweep +// then sees the hashes agree and never refetches, leaving the durable inventory +// permanently missing fields the new code reads. The bytes are bounded and +// already parsed by the client, so storing them verbatim costs nothing but +// keeps the payload honest about the hash filed with it. +// +// The hash comes out of the payload itself, because only the node knows what it // hashed. A report without one is refused rather than given a synthetic hash, // which would make an old node look like it had capability tracking. func nodeCapabilityFetcher(jwtSecret string) nodepool.CapabilityFetcher { client := &http.Client{Timeout: nodeCapabilityRequestTimeout} return func(ctx context.Context, nodeURL string) ([]byte, string, error) { - info, status, err := transcodenode.FetchHWCapabilities(ctx, client, nodeURL, jwtSecret) + info, payload, status, err := transcodenode.FetchHWCapabilitiesPayload(ctx, client, nodeURL, jwtSecret) if err != nil { return nil, "", err } if status != http.StatusOK { return nil, "", fmt.Errorf("node capability request returned status %d", status) } - payload, err := json.Marshal(info) - if err != nil { - return nil, "", err - } return payload, info.CapabilityHash, nil } } diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index 54c76c7ab..d433d7d4b 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "io" "net/http" @@ -374,3 +375,45 @@ func TestCachedLibraryPathsDoesNotShareItsCachedSlice(t *testing.T) { t.Fatalf("fallback = %v, want the cache untouched by a mutation of an earlier fallback", got) } } + +// The stored capability payload has to be the node's own bytes. +// +// Re-marshaling the decoded struct drops every field this build has no member +// for, which is exactly what happens during a rolling upgrade where a node is +// newer than the API reading it — and the truncation is then stored under the +// node's own hash. After the API is upgraded, the sweep sees the hashes agree +// and never refetches, so the durable inventory stays missing fields the new +// code reads until something unrelated moves the hash. +func TestNodeCapabilityFetcherStoresTheNodesOwnBytes(t *testing.T) { + const body = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"capability_hash":"sha256:abc","a_field_this_build_has_never_heard_of":{"nested":[1,2,3]}}` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/hw-capabilities" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + payload, hash, err := nodeCapabilityFetcher("secret")(context.Background(), server.URL) + if err != nil { + t.Fatalf("nodeCapabilityFetcher: %v", err) + } + if hash != "sha256:abc" { + t.Fatalf("hash = %q, want the node's own", hash) + } + + var stored map[string]any + if err := json.Unmarshal(payload, &stored); err != nil { + t.Fatalf("stored payload is not valid JSON: %v (%s)", err, payload) + } + if _, ok := stored["a_field_this_build_has_never_heard_of"]; !ok { + t.Fatalf("a field this build does not know was dropped, but its hash was kept: %s", payload) + } + if stored["resolved"] != "nvenc" { + t.Fatalf("resolved = %v, want the report's own value", stored["resolved"]) + } +} diff --git a/internal/transcodenode/capability_client.go b/internal/transcodenode/capability_client.go index 6b7aff155..e7ed1049f 100644 --- a/internal/transcodenode/capability_client.go +++ b/internal/transcodenode/capability_client.go @@ -19,9 +19,26 @@ const maxHWCapabilitiesResponseBytes = 1 << 20 // cloned so its transport, timeout, and cookie jar remain available without // mutating its redirect policy. func FetchHWCapabilities(ctx context.Context, baseClient *http.Client, nodeURL, jwtSecret string) (playback.HWAccelInfo, int, error) { + info, _, status, err := FetchHWCapabilitiesPayload(ctx, baseClient, nodeURL, jwtSecret) + return info, status, err +} + +// FetchHWCapabilitiesPayload is FetchHWCapabilities with the node's own +// response bytes returned alongside the decoded report. +// +// A caller that *persists* the report must use this form. Re-marshaling the +// decoded struct instead drops every field this build does not know about, and +// during a rolling upgrade a node is routinely newer than the API server that +// reads it. The truncated payload would then be stored under the node's own +// hash, so after the API is upgraded the sweep still sees the hashes agree and +// never refetches: the durable inventory stays missing fields the new code +// reads, until some unrelated capability change or a manual re-probe moves the +// hash. The bytes are already bounded and parsed by the time they are returned, +// which is what makes storing them verbatim safe. +func FetchHWCapabilitiesPayload(ctx context.Context, baseClient *http.Client, nodeURL, jwtSecret string) (playback.HWAccelInfo, []byte, int, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(nodeURL, "/")+"/hw-capabilities", nil) if err != nil { - return playback.HWAccelInfo{}, 0, logredact.SanitizeURLError(err) + return playback.HWAccelInfo{}, nil, 0, logredact.SanitizeURLError(err) } request.Header.Set("Authorization", "Bearer "+jwtSecret) @@ -34,26 +51,26 @@ func FetchHWCapabilities(ctx context.Context, baseClient *http.Client, nodeURL, } response, err := client.Do(request) if err != nil { - return playback.HWAccelInfo{}, 0, logredact.SanitizeURLError(err) + return playback.HWAccelInfo{}, nil, 0, logredact.SanitizeURLError(err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { // Drain the (small) error body so the transport can reuse the // connection instead of tearing it down on every failed probe. _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) - return playback.HWAccelInfo{}, response.StatusCode, nil + return playback.HWAccelInfo{}, nil, response.StatusCode, nil } data, err := io.ReadAll(io.LimitReader(response.Body, maxHWCapabilitiesResponseBytes+1)) if err != nil { - return playback.HWAccelInfo{}, response.StatusCode, err + return playback.HWAccelInfo{}, nil, response.StatusCode, err } if len(data) > maxHWCapabilitiesResponseBytes { - return playback.HWAccelInfo{}, response.StatusCode, fmt.Errorf("node capability response exceeds %d bytes", maxHWCapabilitiesResponseBytes) + return playback.HWAccelInfo{}, nil, response.StatusCode, fmt.Errorf("node capability response exceeds %d bytes", maxHWCapabilitiesResponseBytes) } var info playback.HWAccelInfo if err := json.Unmarshal(data, &info); err != nil { - return playback.HWAccelInfo{}, response.StatusCode, err + return playback.HWAccelInfo{}, nil, response.StatusCode, err } - return info, response.StatusCode, nil + return info, data, response.StatusCode, nil } From 379776fe41b749fe9ce51129a5dc600e7d14a22c Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:31:55 -0400 Subject: [PATCH 038/163] fix(nodes): close the teardown and repoint gaps in the GPU gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both windows where a counter said "idle" while the hardware was not. Every teardown path — stop, idle reap, quality switch, force reload — drops activeJobs before calling TranscodeSession.Close, so a stop is reflected immediately. But Close waits for ffmpeg to exit, so the encoder holds its GPU session for the whole call, counted by neither activeJobs nor the gate. A re-probe landing in that gap saw an idle node, smoke- encoded beside a live encoder, and published the false hardware failure the gate exists to prevent. The four teardowns now go through retireGPUSession, which holds the gate across the close — the same overlap the start path already has between the gate and activeJobs. holdWork rather than beginWork, because a stop must never be refused. nodeAccelerationChanged compared only the override values, so repointing a row at a different worker with identical overrides skipped the reload nudge. reloadPools publishes the new URL at once, so until the replacement's own 60s poll this server dispatched the row's overridden backend to a worker still running on what it inherited: the same backend/device mismatch, reached by a different edit. Renamed to nodePolicyTargetChanged, which is what the condition actually is — the policy moved, or the worker it applies to did. Trailing slashes are normalized so an unrelated save does not nudge. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 50 +++++++++++++++-------- internal/api/handlers/nodes_test.go | 55 ++++++++++++++++++++++++++ internal/transcodenode/gpugate.go | 18 +++++++++ internal/transcodenode/gpugate_test.go | 25 ++++++++++++ internal/transcodenode/reprobe_test.go | 46 +++++++++++++++++++++ internal/transcodenode/server.go | 36 +++++++++++++---- 6 files changed, 206 insertions(+), 24 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 32c749f79..50a04dcd0 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -203,14 +203,15 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, node) - // Order matters: the node has to adopt its new device before this server - // starts dispatching its new backend. reloadPools publishes the updated - // policy, and EffectiveHWAccel then names the new backend on every start - // request; the node itself only re-reads its row on the config watcher's - // 60s poll. Without the nudge, an operator switching both overlays at once - // (QSV on a render node to NVENC on a CUDA index, say) gets up to a minute - // of requests pairing the new backend with the old device. - if nodeAccelerationChanged(previous, node) { + // Order matters: the node has to adopt its policy before this server starts + // dispatching under it. reloadPools publishes the updated row, and + // EffectiveHWAccel then names its backend on every start request; the node + // itself only re-reads its row on the config watcher's 60s poll. Without the + // nudge, an operator switching both overlays at once (QSV on a render node + // to NVENC on a CUDA index, say) — or repointing the row at a worker that + // has been running on what it inherited — gets up to a minute of requests + // pairing one side's backend with the other side's device. + if nodePolicyTargetChanged(previous, node) { // Detached for the same reason reloadPools is: the response is already // written, so a client that has gone away must not decide whether the // worker hears about its new policy. @@ -246,22 +247,39 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { h.reloadPools(r.Context()) } -// nodeAccelerationChanged reports whether this update actually moved either -// acceleration override. +// nodePolicyTargetChanged reports whether this update changed which effective +// acceleration policy applies, or to which worker. // -// The admin form submits both fields on every transcode-node save, so their -// presence in the body is not evidence of a change — without this, renaming a -// node or editing its capacity would nudge it too. An unreadable previous row -// reports no change: the nudge is an optimization over the node's own config -// poll, and skipping it costs at most that interval. -func nodeAccelerationChanged(before, after *nodepool.Node) bool { +// Two ways that happens, and both open the same window. The obvious one is an +// override moving. The other is the row being repointed: the overrides may be +// byte-identical, but they now describe a *different* worker, one that has been +// running under whatever it inherited and will not learn otherwise until its own +// 60s poll. reloadPools publishes the new URL immediately, so between those two +// moments this server dispatches the row's overridden backend to a worker still +// holding its inherited device — the same mismatch, reached by a different edit. +// +// The admin form submits every field on each save, so their presence in the body +// is not evidence of a change; without this, renaming a node or editing its +// capacity would nudge it too. An unreadable previous row reports no change: the +// nudge is an optimization over the node's own config poll, and skipping it +// costs at most that interval. +func nodePolicyTargetChanged(before, after *nodepool.Node) bool { if before == nil || after == nil { return false } + if !sameNodeURL(before.URL, after.URL) { + return true + } return !sameOptionalString(before.HWAccelOverride, after.HWAccelOverride) || !sameOptionalString(before.HWDeviceOverride, after.HWDeviceOverride) } +// sameNodeURL compares two stored node URLs the way the pools and the database +// fences do, so a trailing slash is not a repoint. +func sameNodeURL(a, b string) bool { + return strings.TrimRight(a, "/") == strings.TrimRight(b, "/") +} + func sameOptionalString(a, b *string) bool { if a == nil || b == nil { return a == nil && b == nil diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index ce7a39ef5..6c5f56a9e 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -524,3 +524,58 @@ func TestReloadPoolsSurvivesRequestCancellation(t *testing.T) { t.Fatalf("transcode pool holds %d nodes, want the reload to have landed", len(got)) } } + +// Repointing a row at a different worker changes which machine those overrides +// apply to, even when the values are byte-identical. reloadPools publishes the +// new URL at once, so between that and the replacement's own 60s config poll +// this server dispatches the row's overridden backend to a worker still running +// on whatever it inherited — the same backend/device mismatch an override edit +// causes, reached by a different edit. +func TestHandleUpdateNodeReloadsTheReplacementWhenAURLMoves(t *testing.T) { + reloaded := make(chan string, 4) + replacement := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/reload") { + reloaded <- r.URL.Path + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(replacement.Close) + + qsv, device := "qsv", "/dev/dri/renderD128" + repo := &stubNodeRepository{ + node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://retired-worker", + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + updateResult: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: replacement.URL, + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + } + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + body := `{"name":"gpu-1","url":"` + replacement.URL + `","hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}` + handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, body)) + + select { + case path := <-reloaded: + if path != "/admin/reload-config" { + t.Fatalf("nudged %q, want the non-destructive reload", path) + } + default: + t.Fatal("a repointed row left the replacement worker on its inherited policy") + } +} + +// A trailing slash is not a repoint: the pools normalize URLs and the database +// column does not, so treating it as one would nudge on every unrelated save. +func TestNodePolicyTargetChangeIgnoresATrailingSlash(t *testing.T) { + qsv := "qsv" + before := &nodepool.Node{ID: 1, URL: "http://node/", HWAccelOverride: &qsv} + sameAccel := qsv + after := &nodepool.Node{ID: 1, URL: "http://node", HWAccelOverride: &sameAccel} + + if nodePolicyTargetChanged(before, after) { + t.Fatal("a trailing-slash difference read as repointing the row") + } +} diff --git a/internal/transcodenode/gpugate.go b/internal/transcodenode/gpugate.go index 969f84c4a..dbda8b809 100644 --- a/internal/transcodenode/gpugate.go +++ b/internal/transcodenode/gpugate.go @@ -47,6 +47,24 @@ func (g *gpuGate) beginWork() bool { return true } +// holdWork registers GPU work that is already running, and so cannot be +// refused. +// +// The gate admits work at its start and the node counts it in activeJobs once +// ffmpeg is up, which between them cover a session from admission to teardown — +// except for teardown itself. TranscodeSession.Close waits for ffmpeg to exit, +// so the encoder holds its GPU session for the whole call, while every teardown +// path drops activeJobs first so a stop is reflected immediately. That leaves a +// live encoder counted by neither, and a re-probe landing in the gap sees an +// idle node and smoke-encodes beside it — publishing exactly the false hardware +// failure this gate exists to prevent. Refusing here is not an option: a stop +// must always proceed. Counting it is. +func (g *gpuGate) holdWork() { + g.mu.Lock() + g.workers++ + g.mu.Unlock() +} + // endWork releases one unit of admitted GPU work. func (g *gpuGate) endWork() { g.mu.Lock() diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go index 9bbcdda29..91127ad98 100644 --- a/internal/transcodenode/gpugate_test.go +++ b/internal/transcodenode/gpugate_test.go @@ -78,3 +78,28 @@ func TestGPUGateEndWorkDoesNotUnderflow(t *testing.T) { t.Fatal("re-probe admitted while one unit of work was outstanding") } } + +// Teardown is GPU work too, and it cannot be refused: a stop must always +// proceed. TranscodeSession.Close waits for ffmpeg to exit, so the encoder holds +// its GPU session for the whole call while activeJobs has already dropped — +// counted by neither unless the gate holds it. +func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { + var gate gpuGate + + gate.holdWork() + if busy, ok := gate.beginReprobe(0); ok { + t.Fatal("re-probe admitted while a session was still closing") + } else if busy != 1 { + t.Fatalf("busy = %d, want the closing session counted", busy) + } + + gate.endWork() + if _, ok := gate.beginReprobe(0); !ok { + t.Fatal("re-probe refused after the teardown finished") + } + + // A re-probe already holding the encoder must not turn a stop into a hang or + // a refusal; the teardown is registered regardless. + gate.holdWork() + gate.endWork() +} diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go index 9f65157e7..3d8395513 100644 --- a/internal/transcodenode/reprobe_test.go +++ b/internal/transcodenode/reprobe_test.go @@ -319,3 +319,49 @@ func TestCapabilitySnapshotRefusedWhileReprobing(t *testing.T) { t.Fatalf("stored hash = %q, want the previous report untouched", got) } } + +// Every teardown path drops activeJobs before closing the session, so a stop is +// reflected immediately — but Close waits for ffmpeg to exit, so the encoder +// keeps its GPU session for the whole call. Without the gate holding it, that +// live encoder is counted by neither activeJobs nor the gate, and a re-probe +// landing in the gap smoke-encodes beside it and publishes the false hardware +// failure the gate exists to prevent. +func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + server.activeJobs.Store(1) + + var ( + jobsDuringClose int32 + reprobeAdmitted bool + busyDuringClose int + ) + // Observed from inside the teardown, which is the only instant that matters. + err := server.retireGPUSession(func() error { + jobsDuringClose = server.activeJobs.Load() + busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(int(jobsDuringClose)) + if reprobeAdmitted { + server.gpu.endReprobe() + } + return nil + }) + if err != nil { + t.Fatalf("retireGPUSession: %v", err) + } + + if jobsDuringClose != 0 { + t.Fatalf("active jobs = %d during Close, want the 0 that made this a gap", jobsDuringClose) + } + if reprobeAdmitted { + t.Fatal("re-probe admitted while a session was still closing its encoder") + } + if busyDuringClose != 1 { + t.Fatalf("busy = %d, want the closing session counted as GPU work", busyDuringClose) + } + + // The hold is released with the teardown, so an idle node re-probes again. + if _, ok := server.gpu.beginReprobe(int(server.activeJobs.Load())); !ok { + t.Fatal("re-probe refused after the teardown completed") + } + server.gpu.endReprobe() +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 59c77ac5f..640eaa99c 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -557,8 +557,7 @@ func (s *Server) reapSession(sessionID string, session *playback.TranscodeSessio delete(s.lastAccess, sessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - if err := session.Close(); err != nil { + if err := s.closeSessionOffGPU(session); err != nil { slog.Error("close idle transcode session", "component", "transcodenode", "error", err, "session", sessionID, "playback_session_id", sessionID) } if s.tracker != nil { @@ -568,6 +567,30 @@ func (s *Server) reapSession(sessionID string, session *playback.TranscodeSessio "session", sessionID, "playback_session_id", sessionID, "idle_ms", time.Since(last).Milliseconds()) } +// closeSessionOffGPU retires one session: it drops the node's job count and +// closes the encoder, with the GPU gate holding the session as work for the +// whole teardown. +// +// The two counters have to overlap here, the same way the gate and activeJobs +// overlap when a session starts. Close waits for ffmpeg to exit, so the encoder +// keeps its GPU session for the length of the call, while activeJobs has to drop +// first for a stop to be reflected immediately. Without the hold, that live +// encoder is counted by nothing and a re-probe admitted in the gap smoke-encodes +// beside it. +func (s *Server) closeSessionOffGPU(session *playback.TranscodeSession) error { + return s.retireGPUSession(session.Close) +} + +// retireGPUSession drops the node's job count and runs one session's teardown +// with the gate holding it as work throughout. The hold discipline is the same +// whatever the teardown is, which is why it is separate from what it closes. +func (s *Server) retireGPUSession(close func() error) error { + s.gpu.holdWork() + defer s.gpu.endWork() + s.activeJobs.Add(-1) + return close() +} + // recipeStore reads a remote transcode's reconstruction recipe written by central // at transcode start, keyed by the transport id this node serves the job under. // It is the reconstruct source for every flow whose request cannot carry a @@ -1324,8 +1347,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { delete(s.sessions, req.SessionID) delete(s.lastAccess, req.SessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - _ = old.Close() + _ = s.closeSessionOffGPU(old) // Move the old segment directory aside and delete it in the // background: removing a long session's segments can take seconds // on slow disks, and the playback start that triggered this switch @@ -1706,9 +1728,8 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { delete(s.sessions, sessionID) delete(s.lastAccess, sessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - if err := session.Close(); err != nil { + if err := s.closeSessionOffGPU(session); err != nil { slog.ErrorContext(r.Context(), "close transcode session", "component", "transcodenode", "error", err, "session", sessionID, "playback_session_id", sessionID) } @@ -1970,9 +1991,8 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { delete(s.sessions, victim.id) delete(s.lastAccess, victim.id) s.mu.Unlock() - s.activeJobs.Add(-1) - victim.session.Close() + _ = s.closeSessionOffGPU(victim.session) if err := os.RemoveAll(s.sessionOutputDir(victim.id)); err != nil { slog.WarnContext(r.Context(), "remove transcode session directory during reload", "component", "transcodenode", "session", victim.id, "error", err) } From 3a9462cc23fc219a3c3165531a10ef90ecd7e887 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:50:03 -0400 Subject: [PATCH 039/163] fix(nodes): count detached probes, catch deadlines inside one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings. The previous commit taught nodePolicyTargetChanged about URL moves, but its caller only loaded the previous row when the body carried an acceleration field. A partial PUT changing just the url therefore left previous nil and the new clause never fired — the fix was dead for exactly the request shape that motivated it. The old row is now loaded for a URL change too. verifyHWAccelBackend checked ctx.Err() only at the top of its loop, so a walk budget that expired *inside* a probe was invisible. On the last candidate of the last backend nothing downstream caught it either, and the node published a timeout as a hardware regression: new hash, recorded drift, resolved to software, for a GPU that was merely slow to answer. A probe runs on a background context so an abandoned caller cannot kill work another request is waiting on — which means buildCapabilitySnapshot releases its gate claim while ffmpeg may still be encoding. A re-probe counting only this node's own bookkeeping then claimed an encoder that was not free and raced the smoke encode already running, producing the false verdict the gate exists to prevent, one layer down. playback now exports HWProbesInFlight and beginReprobe adds it to busy. Only the re-probe consults it: refusing ordinary transcodes whenever a background capability fetch is mid-probe would cost playback far more than it saves, and the gate's stated asymmetry is that work is never made to wait. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 12 +++- internal/api/handlers/nodes_test.go | 40 +++++++++++ internal/playback/gpudetect.go | 43 ++++++++++++ internal/playback/gpudetect_publish_test.go | 74 +++++++++++++++++++++ internal/transcodenode/gpugate.go | 21 ++++-- internal/transcodenode/gpugate_test.go | 38 ++++++++--- internal/transcodenode/reprobe.go | 2 +- internal/transcodenode/reprobe_test.go | 10 +-- 8 files changed, 218 insertions(+), 22 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 50a04dcd0..23c38d05b 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -180,10 +180,16 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { } // Read the row before the write so the nudge below can tell a real policy - // change from a resubmit: the admin form posts both override fields on - // every save, so the fields being present says nothing about them moving. + // change from a resubmit: the admin form posts every field on each save, so + // a field being present says nothing about it moving. + // + // The URL counts for the same reason the overrides do. A partial PUT that + // carries only a new url still repoints the row's existing overrides at a + // different worker, and without the old row to compare against + // nodePolicyTargetChanged has nothing to see — the replacement would keep + // running on what it inherited until its own 60s poll. var previous *nodepool.Node - if input.HWAccelOverride != nil || input.HWDeviceOverride != nil { + if input.URL != nil || input.HWAccelOverride != nil || input.HWDeviceOverride != nil { previous, _ = h.repo.GetByID(r.Context(), id) } diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 6c5f56a9e..082df5904 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -579,3 +579,43 @@ func TestNodePolicyTargetChangeIgnoresATrailingSlash(t *testing.T) { t.Fatal("a trailing-slash difference read as repointing the row") } } + +// A partial PUT that carries only a new url still repoints the row's existing +// overrides at a different worker. Loading the previous row only when an +// override field is present left nodePolicyTargetChanged with nothing to compare +// against, so the URL clause it grew for exactly this case never fired. +func TestHandleUpdateNodeReloadsOnAURLOnlyRepoint(t *testing.T) { + reloaded := make(chan string, 4) + replacement := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/reload") { + reloaded <- r.URL.Path + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(replacement.Close) + + qsv, device := "qsv", "/dev/dri/renderD128" + repo := &stubNodeRepository{ + node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://retired-worker", + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + updateResult: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: replacement.URL, + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + } + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + // Only the url: no acceleration field in the body at all. + handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, `{"url":"`+replacement.URL+`"}`)) + + select { + case path := <-reloaded: + if path != "/admin/reload-config" { + t.Fatalf("nudged %q, want the non-destructive reload", path) + } + default: + t.Fatal("a url-only repoint left the replacement worker on its inherited policy") + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 772f1deaa..a4c8ce4db 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/Silo-Server/silo-server/internal/nodemetrics" @@ -471,6 +472,16 @@ func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candi } probed = true available, reason := ffmpegSupportsBackendContext(ctx, backend, ffmpegPath, device) + if !available && ctx.Err() != nil { + // The walk's own budget ran out while this probe was in flight, so + // the failure describes the deadline and not the hardware. Checking + // only at the top of the loop misses it entirely on the last + // candidate of the last backend, and the report would then publish + // a timeout as a real regression: a new hash, recorded drift, and a + // node resolved to software with nothing wrong with its GPU. + complete = false + break + } if available { // Execution has to land on a device a probe passed and not on // whatever sorts first under /dev/dri, or a report saying "qsv @@ -712,6 +723,15 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi if ok && hwProbeCacheEntryCurrent(cached, now()) { return cached.result, nil } + // Counted for the life of the smoke encode, not the life of the caller. + // probeCtx is deliberately rooted at Background so an abandoned caller + // does not kill work another request is waiting on, which means ffmpeg + // can still be on the GPU after every caller has returned. Anything that + // needs the encoder to itself has to see that; see HWProbesInFlight. + // Raised before the test seam below, so an observer parked in it sees + // the state this counter exists to report. + hwProbesInFlight.Add(1) + defer hwProbesInFlight.Add(-1) if flightStarted != nil { flightStarted() } @@ -742,6 +762,29 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi } } +// hwProbesInFlight counts hardware smoke encodes running right now, including +// ones whose caller has already given up on them. +var hwProbesInFlight atomic.Int64 + +// HWProbesInFlight reports how many hardware smoke encodes this process is +// running. +// +// A probe outlives its caller by design: the singleflight task runs on a +// background context so that a canceled request cannot kill work another +// request is waiting on. The consequence is that a component which released its +// own claim on the GPU when its call returned — the transcode node's capability +// build is exactly this — can leave ffmpeg on the card with nothing accounting +// for it. Anything that needs the encoder exclusively must add this to whatever +// else it counts as busy, or it will claim an encoder that is not free and +// publish the collision as a hardware failure. +func HWProbesInFlight() int { + count := hwProbesInFlight.Load() + if count < 0 { + return 0 + } + return int(count) +} + func hwProbeCacheEntryCurrent(entry hwProbeCacheEntry, now time.Time) bool { return entry.result.available || now.Before(entry.expiresAt) } diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 593ce4de6..9d3c4fe8e 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -10,6 +10,7 @@ import ( "sync" "sync/atomic" "testing" + "time" ) // A detection walk that ran out of budget marks backends it never reached @@ -507,3 +508,76 @@ func TestCUDAOnlyHostPublishesItsGPUIdentities(t *testing.T) { t.Fatal("capability hash ignores the GPU identity list; a lost card would never trigger a refetch") } } + +// The walk's budget can also run out *inside* a probe rather than between two of +// them. Checking the context only at the top of the loop misses that entirely on +// the last candidate of the last backend: the probe returns a context error, the +// loop ends normally, and the report goes out claiming a verified regression — +// a new hash, a recorded drift note, and a node routed to software, for a GPU +// that is fine and merely slow to answer. +func TestDetectHWAccelReportsADeadlineInsideTheFinalProbe(t *testing.T) { + env := setupHWAccelTest(t) + // An AMD card so VAAPI is the only backend with candidates: with QSV also in + // play, the walk's between-backends check would mask the gap. + env.addRenderDevice(t, "renderD128", "0x1002") + probe := successfulVAAPIProbe() + probe.hang = true + ffmpeg := writeFakeFFmpeg(t, probe) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + t.Cleanup(cancel) + + info, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, ffmpeg.path, "") + if !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete for a deadline inside the probe", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend == transcodeHWVAAPI && backend.Verified { + t.Fatalf("vaapi = %+v, want no verification claimed", backend) + } + } +} + +// A probe outlives its caller by design, so a component that released its own +// claim on the GPU when its call returned can leave ffmpeg on the card with +// nothing accounting for it. The count is what lets the transcode node's +// re-probe gate see that. +func TestHWProbesInFlightCountsADetachedProbe(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + device := env.devicePath("renderD128") + + // Measured as a delta rather than against zero: a detached probe outliving + // its caller is the very thing under test, so an earlier one in this process + // may still be running. + baseline := HWProbesInFlight() + + // Decided by channel receipts, not a sleep: the flight parks inside the + // probe until this test has read the count. + started := make(chan struct{}) + release := make(chan struct{}) + hwProbeFlightStarted = func() { + close(started) + <-release + } + t.Cleanup(func() { hwProbeFlightStarted = nil }) + + var wg sync.WaitGroup + wg.Go(func() { + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Errorf("probe failed: %s", reason) + } + }) + + <-started + if got := HWProbesInFlight(); got != baseline+1 { + t.Fatalf("HWProbesInFlight() = %d while a smoke encode is running, want %d", got, baseline+1) + } + close(release) + wg.Wait() + + if got := HWProbesInFlight(); got != baseline { + t.Fatalf("HWProbesInFlight() = %d after the probe finished, want %d", got, baseline) + } +} diff --git a/internal/transcodenode/gpugate.go b/internal/transcodenode/gpugate.go index dbda8b809..2f1370aa0 100644 --- a/internal/transcodenode/gpugate.go +++ b/internal/transcodenode/gpugate.go @@ -75,13 +75,24 @@ func (g *gpuGate) endWork() { } // beginReprobe claims the encoder exclusively, or reports the work in progress -// that stopped it. activeJobs is the node's own running-session count, passed in -// so both halves of "is this node busy" are read under one lock rather than -// sampled at two different instants. -func (g *gpuGate) beginReprobe(activeJobs int) (busy int, ok bool) { +// that stopped it. +// +// activeJobs is the node's own running-session count and detachedProbes is +// playback.HWProbesInFlight. Both are passed in rather than read here so the +// gate stays a lock over its own state, and so the count that refuses a caller +// is the same one reported back to it. +// +// detachedProbes is the piece that is not this node's own bookkeeping. A +// hardware probe runs on a background context so an abandoned caller cannot +// kill work another request is waiting on, which means the capability build +// releases its gate claim while ffmpeg may still be encoding. Without counting +// those, a re-probe claims an encoder that is not free and its smoke matrix +// races the one already running — publishing the false hardware verdict this +// gate exists to prevent, by the same mechanism, one layer down. +func (g *gpuGate) beginReprobe(activeJobs, detachedProbes int) (busy int, ok bool) { g.mu.Lock() defer g.mu.Unlock() - busy = g.workers + activeJobs + busy = g.workers + activeJobs + detachedProbes if g.reprobing || busy > 0 { return busy, false } diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go index 91127ad98..16e3976b1 100644 --- a/internal/transcodenode/gpugate_test.go +++ b/internal/transcodenode/gpugate_test.go @@ -15,14 +15,14 @@ func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { } // activeJobs is still 0 here: the counter only moves once ffmpeg is running, // which is exactly the window a point-in-time check missed. - if busy, ok := gate.beginReprobe(0); ok { + if busy, ok := gate.beginReprobe(0, 0); ok { t.Fatal("re-probe admitted while a transcode was starting") } else if busy != 1 { t.Fatalf("busy = %d, want the admitted work counted", busy) } gate.endWork() - if _, ok := gate.beginReprobe(0); !ok { + if _, ok := gate.beginReprobe(0, 0); !ok { t.Fatal("re-probe refused after the work finished") } } @@ -32,7 +32,7 @@ func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { var gate gpuGate - busy, ok := gate.beginReprobe(2) + busy, ok := gate.beginReprobe(2, 0) if ok { t.Fatal("re-probe admitted on a node running transcodes") } @@ -47,13 +47,13 @@ func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { func TestGPUGateRefusesWorkWhileReprobing(t *testing.T) { var gate gpuGate - if _, ok := gate.beginReprobe(0); !ok { + if _, ok := gate.beginReprobe(0, 0); !ok { t.Fatal("re-probe refused on an idle gate") } if gate.beginWork() { t.Fatal("GPU work admitted while a re-probe held the encoder") } - if _, ok := gate.beginReprobe(0); ok { + if _, ok := gate.beginReprobe(0, 0); ok { t.Fatal("a second concurrent re-probe was admitted") } @@ -74,7 +74,7 @@ func TestGPUGateEndWorkDoesNotUnderflow(t *testing.T) { if !gate.beginWork() { t.Fatal("beginWork refused after unbalanced releases") } - if _, ok := gate.beginReprobe(0); ok { + if _, ok := gate.beginReprobe(0, 0); ok { t.Fatal("re-probe admitted while one unit of work was outstanding") } } @@ -87,14 +87,14 @@ func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { var gate gpuGate gate.holdWork() - if busy, ok := gate.beginReprobe(0); ok { + if busy, ok := gate.beginReprobe(0, 0); ok { t.Fatal("re-probe admitted while a session was still closing") } else if busy != 1 { t.Fatalf("busy = %d, want the closing session counted", busy) } gate.endWork() - if _, ok := gate.beginReprobe(0); !ok { + if _, ok := gate.beginReprobe(0, 0); !ok { t.Fatal("re-probe refused after the teardown finished") } @@ -103,3 +103,25 @@ func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { gate.holdWork() gate.endWork() } + +// A hardware probe runs on a background context so an abandoned caller cannot +// kill work another request is waiting on. The capability build therefore +// releases its gate claim while ffmpeg may still be encoding, and a re-probe +// that counted only this node's own bookkeeping would claim an encoder that is +// not free — its smoke matrix racing the one already running, publishing the +// false hardware verdict this gate exists to prevent. +func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(0, 1) + if ok { + t.Fatal("re-probe admitted while a detached smoke encode was still running") + } + if busy != 1 { + t.Fatalf("busy = %d, want the detached probe counted", busy) + } + + if _, ok := gate.beginReprobe(0, 0); !ok { + t.Fatal("re-probe refused once no probe was in flight") + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index 4f986cb71..8d86a5b46 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -60,7 +60,7 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques // collide with the smoke encode minutes into the probe. The gate is the // same exclusion the transcode-start path consults, so from here until the // deferred release no new GPU work is admitted. - busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load())) + busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load()), playback.HWProbesInFlight()) if !ok { slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", "component", "transcodenode", "active_jobs", busy) diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go index 3d8395513..becd895db 100644 --- a/internal/transcodenode/reprobe_test.go +++ b/internal/transcodenode/reprobe_test.go @@ -160,7 +160,7 @@ func TestReprobeCapabilitiesRefusesWhileWorkIsStarting(t *testing.T) { // and the API retries on another node. func TestTranscodeStartRefusedWhileReprobing(t *testing.T) { server := newTestServer(t) - if _, ok := server.gpu.beginReprobe(0); !ok { + if _, ok := server.gpu.beginReprobe(0, 0); !ok { t.Fatal("re-probe refused on an idle node") } t.Cleanup(server.gpu.endReprobe) @@ -289,7 +289,7 @@ func TestCapabilitySnapshotRegistersAsGPUWork(t *testing.T) { case <-time.After(30 * time.Second): t.Fatal("the capability snapshot was never admitted as GPU work") } - if _, ok := server.gpu.beginReprobe(0); ok { + if _, ok := server.gpu.beginReprobe(0, 0); ok { server.gpu.endReprobe() t.Fatal("a re-probe was admitted while a capability snapshot held the encoder") } @@ -307,7 +307,7 @@ func TestCapabilitySnapshotRegistersAsGPUWork(t *testing.T) { func TestCapabilitySnapshotRefusedWhileReprobing(t *testing.T) { server := newTestServer(t) server.storeCapabilityHash("sha256:previous") - if _, ok := server.gpu.beginReprobe(0); !ok { + if _, ok := server.gpu.beginReprobe(0, 0); !ok { t.Fatal("re-probe refused on an idle node") } t.Cleanup(server.gpu.endReprobe) @@ -339,7 +339,7 @@ func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { // Observed from inside the teardown, which is the only instant that matters. err := server.retireGPUSession(func() error { jobsDuringClose = server.activeJobs.Load() - busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(int(jobsDuringClose)) + busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(int(jobsDuringClose), 0) if reprobeAdmitted { server.gpu.endReprobe() } @@ -360,7 +360,7 @@ func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { } // The hold is released with the teardown, so an idle node re-probes again. - if _, ok := server.gpu.beginReprobe(int(server.activeJobs.Load())); !ok { + if _, ok := server.gpu.beginReprobe(int(server.activeJobs.Load()), 0); !ok { t.Fatal("re-probe refused after the teardown completed") } server.gpu.endReprobe() From 19e41d8795784eb0a4eb50267d45be0889c5f796 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:17:21 -0400 Subject: [PATCH 040/163] fix(playback): claim the encoder before dispatching a probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit singleflight.DoChan schedules its function on a new goroutine and returns without waiting for it, so raising the in-flight count inside that function left a registration gap: a caller whose context was already done took the ctx.Done() branch and returned before the probe reached its first line. Anything that released its own claim when the call returned — buildCapabilitySnapshot does exactly that — could hand a re-probe an encoder that was about to be busy, which is the same false-verdict race the counter was added to close, one scheduling hop earlier. The claim is now taken on the calling goroutine before DoChan, and released when the result lands. An abandoned caller hands its claim to a goroutine that waits out the flight rather than dropping it, so the count covers the probe rather than the caller. It counts claims, not processes, and errs high: a brief overcount costs an operator a 409 and a retry, an undercount costs a false hardware regression. Also, separately: enqueueRouteEventV3's background writer re-read h.PlanStoreV3 on every event. In production the router wires that field once before serving, but the goroutine outlives the request that started it, and a caller replacing the store afterwards raced it for real — which is what internal/api/handlers reported under -race for this branch. The queue now captures the store it was created for. With that and the in-flight tests made independent of process-global drain order, the -race suite is clean for every package this branch touches; the failures that remain are in jellycompat/web_component_test.go, metadata, and taskmanager, none of which this branch changes. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/playback_v3.go | 10 ++- internal/playback/gpudetect.go | 40 ++++++++--- internal/playback/gpudetect_publish_test.go | 75 ++++++++++++++++++--- 3 files changed, 104 insertions(+), 21 deletions(-) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 21e3f182d..62879be8f 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -4678,10 +4678,18 @@ func (h *PlaybackHandler) enqueueRouteEventV3(event playback.RouteEventRecordV3) } h.v3EventOnce.Do(func() { h.v3EventQueue = make(chan playback.RouteEventRecordV3, 512) + // The store is captured with the queue rather than read per event. The + // goroutine below outlives the request that started it, so re-reading + // the field would be an unsynchronized read of handler state — harmless + // in production, where the router wires PlanStoreV3 once before serving, + // and a real data race against any caller that replaces it afterwards. + // Capturing also matches what the queue is: work batched for the store + // that existed when it was created. + store := h.PlanStoreV3 go func() { for value := range h.v3EventQueue { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - if err := h.PlanStoreV3.RecordRouteEvent(ctx, value); err != nil { + if err := store.RecordRouteEvent(ctx, value); err != nil { slog.Warn("playback route event write failed", "error", err, "event", value.Event) } cancel() diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index a4c8ce4db..1900c16c4 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -716,6 +716,17 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi } hwProbeCache.Unlock() + // Raised here, on the calling goroutine, and not inside the function below. + // DoChan schedules that function on a new goroutine and returns without + // waiting for it to run, so a caller whose context is already done takes the + // ctx.Done() branch and returns while the probe has not reached its first + // line. Anything that released its own claim on the encoder when this call + // returned — the transcode node's capability build does exactly that — would + // then hand a re-probe an encoder that is about to be busy. Registering + // before the call closes the window: from here on the flight is counted + // whether or not it has started, and whether or not anyone still waits for + // it. See HWProbesInFlight. + hwProbesInFlight.Add(1) resultCh := hwProbeCache.group.DoChan(cacheKey, func() (any, error) { hwProbeCache.Lock() cached, ok := hwProbeCache.entries[cacheKey] @@ -723,15 +734,6 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi if ok && hwProbeCacheEntryCurrent(cached, now()) { return cached.result, nil } - // Counted for the life of the smoke encode, not the life of the caller. - // probeCtx is deliberately rooted at Background so an abandoned caller - // does not kill work another request is waiting on, which means ffmpeg - // can still be on the GPU after every caller has returned. Anything that - // needs the encoder to itself has to see that; see HWProbesInFlight. - // Raised before the test seam below, so an observer parked in it sees - // the state this counter exists to report. - hwProbesInFlight.Add(1) - defer hwProbesInFlight.Add(-1) if flightStarted != nil { flightStarted() } @@ -749,8 +751,17 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi }) select { case <-ctx.Done(): + // The flight outlives this caller by design, so the claim goes with it + // rather than with us. DoChan's channel is buffered and every registered + // waiter is served, so this receive always lands and the count always + // comes back down. + go func() { + <-resultCh + hwProbesInFlight.Add(-1) + }() return false, ctx.Err().Error() case shared := <-resultCh: + hwProbesInFlight.Add(-1) if shared.Err != nil { return false, shared.Err.Error() } @@ -766,8 +777,8 @@ func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, devi // ones whose caller has already given up on them. var hwProbesInFlight atomic.Int64 -// HWProbesInFlight reports how many hardware smoke encodes this process is -// running. +// HWProbesInFlight reports how many hardware smoke encodes this process has +// claimed the encoder for. // // A probe outlives its caller by design: the singleflight task runs on a // background context so that a canceled request cannot kill work another @@ -777,6 +788,13 @@ var hwProbesInFlight atomic.Int64 // for it. Anything that needs the encoder exclusively must add this to whatever // else it counts as busy, or it will claim an encoder that is not free and // publish the collision as a hardware failure. +// +// It counts claims rather than running processes, and deliberately errs high: +// the claim is taken before the probe is dispatched (so a caller can never +// return while its flight is unaccounted for) and released when the result +// lands, and callers that share one flight each hold one. A brief overcount +// costs an operator a 409 and a retry; an undercount costs a false hardware +// regression on a GPU that is fine. func HWProbesInFlight() int { count := hwProbesInFlight.Load() if count < 0 { diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 9d3c4fe8e..846161eb7 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -548,11 +549,11 @@ func TestHWProbesInFlightCountsADetachedProbe(t *testing.T) { ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) device := env.devicePath("renderD128") - // Measured as a delta rather than against zero: a detached probe outliving - // its caller is the very thing under test, so an earlier one in this process - // may still be running. - baseline := HWProbesInFlight() - + // Asserted as a floor rather than an exact count: the process-global counter + // is shared with any probe an earlier test detached, and those drain on + // their own schedule. What this test owns is that its own flight is counted + // while it runs and released when it lands. + // // Decided by channel receipts, not a sleep: the flight parks inside the // probe until this test has read the count. started := make(chan struct{}) @@ -571,13 +572,69 @@ func TestHWProbesInFlightCountsADetachedProbe(t *testing.T) { }) <-started - if got := HWProbesInFlight(); got != baseline+1 { - t.Fatalf("HWProbesInFlight() = %d while a smoke encode is running, want %d", got, baseline+1) + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d while a smoke encode is running, want at least 1", got) } close(release) wg.Wait() + awaitNoProbesInFlight(t) +} + +// The claim has to be taken on the calling goroutine, not inside the function +// singleflight schedules. DoChan returns without waiting for that goroutine to +// run, so a caller whose context is already done returns immediately — and a +// component that released its own claim on the encoder when this call returned +// would hand a re-probe a card that is about to be busy. +// +// Checked the instant the call returns, which is the only instant that matters: +// that is when the caller's own claim goes away. +func TestHWProbeClaimIsHeldBeforeAnAbandonedCallerReturns(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + device := env.devicePath("renderD128") + + // The flight parks so it cannot finish and decrement before the assertion. + release := make(chan struct{}) + var released sync.Once + hwProbeFlightStarted = func() { <-release } + t.Cleanup(func() { + hwProbeFlightStarted = nil + released.Do(func() { close(release) }) + }) - if got := HWProbesInFlight(); got != baseline { - t.Fatalf("HWProbesInFlight() = %d after the probe finished, want %d", got, baseline) + awaitNoProbesInFlight(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if ok, _ := ffmpegSupportsBackendContext(ctx, transcodeHWVAAPI, ffmpeg.path, device); ok { + t.Fatal("an already-canceled probe reported success") + } + + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d the moment the caller returned, want at least 1: "+ + "the detached flight was unaccounted for", got) + } + + released.Do(func() { close(release) }) + // The claim comes back down once the flight lands, with no caller left to + // receive it. + awaitNoProbesInFlight(t) +} + +// awaitNoProbesInFlight waits for every claim on the encoder to be released, +// including ones detached from a caller that has already returned. Waiting on +// the counter itself rather than on a fixed delay is what keeps these tests from +// depending on how loaded the machine is. +func awaitNoProbesInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if got := HWProbesInFlight(); got == 0 { + return + } else if time.Now().After(deadline) { + t.Fatalf("HWProbesInFlight() = %d, want every detached flight released", got) + } + runtime.Gosched() } } From d7e417647944d8ad912a5668e4998f7cb11adbb6 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:35:24 -0400 Subject: [PATCH 041/163] fix(nodemetrics): mark stale disk gauges, half-open the nvidia breaker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mount whose probe stops returning kept exporting its last used and total bytes on every scrape with a fresh timestamp and nothing to qualify them, so a fill alert sat green forever on a volume that stopped answering at 40% and had been filling since. Dropping the series is the wrong answer — `stale` is set as soon as a probe outlives its five second budget, which a network mount does routinely, and blanking the panel for a disk that is merely slow trades one blind spot for another. The values ship with streamapp_node_disk_stale beside them instead: the same pairing the JSON surfaces already carry, and something an alert can read. A mount Silo has never measured still exports nothing, staleness included, since there would be no numbers for it to qualify. nvidia-smi failing five samples running retired it for the process lifetime. That is right for a host with no toolkit and wrong for a driver reset, and the two are indistinguishable over five samples — an NVENC node that hit a transient outage reported no utilization and no VRAM for as long as it stayed up. The breaker now half-opens: one probationary call every ten minutes, which still spares a toolkit-less host almost the whole cost, and closes on the first success. Sampler.RetrySources returns it immediately, and the hardware re-probe calls it — an operator saying "something changed underneath this node" should not then wait out an interval to find out. The breaker's fields are now mutex-guarded, since reset arrives on whichever goroutine serves the re-probe. Co-Authored-By: Claude Opus 5 (1M context) --- docs/wiki/admin/monitoring-nodes.md | 25 +++++++ internal/nodemetrics/collector.go | 36 ++++++++-- internal/nodemetrics/collector_test.go | 66 +++++++++++++++++ internal/nodemetrics/nvidia.go | 99 ++++++++++++++++++++++---- internal/nodemetrics/nvidia_test.go | 89 +++++++++++++++++++++++ internal/transcodenode/reprobe.go | 5 ++ 6 files changed, 303 insertions(+), 17 deletions(-) diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index fc94c2c8f..1de9b8e2e 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -281,8 +281,25 @@ groups: for: 15m annotations: summary: "Silo node CPU pegged on {{ $labels.instance }} — check the GPU column for a failed probe" + + - alert: SiloDiskMeasurementStale + expr: streamapp_node_disk_stale == 1 + for: 15m + annotations: + summary: "Silo has not measured {{ $labels.mount }} on {{ $labels.instance }} for a while — its used/total figures are carried over" ``` +That last one matters more than it looks. A mount whose probe stops returning +keeps exporting its last real used and total bytes, because dropping the series +would blank the panel for a network mount that is merely slow to answer — which +happens routinely. The numbers are genuine, only old, so `SiloScratchVolumeFilling` +above still fires on a volume that was nearly full when measurement stopped. What +it cannot do is notice a volume that stopped answering at 40% and has been +filling since. `streamapp_node_disk_stale` is what closes that gap: it is `1` +whenever the used and total beside it are carried over, and `0` when they are +current. A mount Silo has never measured at all exports nothing — not even a +staleness series, since there would be no numbers for it to qualify. + A GPU nothing could measure exports no engine gauges at all rather than zeros — a Prometheus sample carries no `source` to qualify them, and a zero would read as idle on a card that may be busy and merely unobservable. Its session count still @@ -296,6 +313,14 @@ only, so a card shared with anything outside Silo reads as less busy than it is; on NVIDIA they come from `nvidia-smi` and are whole-GPU. A device that nothing could measure this interval publishes no VRAM series rather than a zero. +If `nvidia-smi` fails five samples running, Silo stops calling it — a host +without the NVIDIA toolkit would otherwise spawn a doomed subprocess every few +seconds forever. It is not retired for good: one probationary call goes out every +ten minutes, so a driver reset or a toolkit installed after startup is picked up +on its own. `POST /api/v1/admin/nodes/{id}/reprobe` puts it back in service +immediately, which is the faster path when you have just fixed the driver +yourself. + ## Source References - `internal/nodepool` — health sweep, capability refetch, drift detection, and diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index fce145204..597e0751a 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -39,11 +39,15 @@ var ( descDiskUsed = prometheus.NewDesc( "streamapp_node_disk_used_bytes", "Used bytes on a sampled mount, labeled by role rather than by path.", - []string{"mount"}, nil) + []string{diskMountLabel}, nil) descDiskTotal = prometheus.NewDesc( "streamapp_node_disk_total_bytes", "Total bytes on a sampled mount, labeled by role rather than by path.", - []string{"mount"}, nil) + []string{diskMountLabel}, nil) + descDiskStale = prometheus.NewDesc( + "streamapp_node_disk_stale", + "1 when a mount's used/total bytes are carried over from an earlier pass because its probe has not returned, 0 when they are current.", + []string{diskMountLabel}, nil) descNetworkRx = prometheus.NewDesc( "streamapp_node_network_rx_bps", "Aggregate received bits per second, loopback excluded.", @@ -74,9 +78,13 @@ var ( []string{gpuDeviceLabel}, nil) ) -// gpuDeviceLabel is the Prometheus label name every per-GPU series is keyed -// by. -const gpuDeviceLabel = "device" +// gpuDeviceLabel is the Prometheus label name every per-GPU series is keyed by, +// and diskMountLabel the one every per-mount series is keyed by. Its value is +// DiskStats.Role, not a path; see the comment on the disk descriptors. +const ( + gpuDeviceLabel = "device" + diskMountLabel = "mount" +) // collector adapts a Sampler to prometheus.Collector. type collector struct{ sampler *Sampler } @@ -110,8 +118,26 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { // zero would read as an empty disk in every alert rule. continue } + // A stale mount keeps exporting, with a series that says so. + // + // Unlike an unavailable one it has a real measurement, only an old + // one — and `stale` is set as soon as a probe outlives its five + // second budget, which a network mount does routinely without + // anything being wrong. Dropping the series there would blank a + // dashboard for a disk that is fine and merely slow to answer. + // Re-exporting old numbers under a fresh scrape timestamp with + // nothing to qualify them is the other half of the problem, though: + // a fill alert would sit green forever on a mount that stopped + // answering at 40% and has been filling since. So the values ship + // with their staleness beside them, the same pairing the JSON + // surfaces carry, and an alert can read both. + stale := 0.0 + if disk.Stale { + stale = 1 + } gauge(descDiskUsed, disk.UsedGB*bytesPerGB, disk.Role) gauge(descDiskTotal, disk.TotalGB*bytesPerGB, disk.Role) + gauge(descDiskStale, stale, disk.Role) } } const bytesPerMBFloat = float64(1024 * 1024) diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go index 0c0d47006..2d60333aa 100644 --- a/internal/nodemetrics/collector_test.go +++ b/internal/nodemetrics/collector_test.go @@ -288,3 +288,69 @@ func TestCollectorExportsMeasuredGPUEngineGauges(t *testing.T) { t.Fatal("a measured idle GPU omitted its render busy percentage") } } + +// gatherLabeled reads one metric family keyed by its single label value, so a +// test can assert per-mount or per-device rather than on whichever series the +// registry happened to order first. +func gatherLabeled(t *testing.T, sampler *Sampler, name string) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + values := map[string]float64{} + for _, family := range families { + if family.GetName() != name { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + values[label.GetValue()] = metric.GetGauge().GetValue() + } + } + } + return values +} + +// A mount whose probe has not come back keeps exporting its last real numbers — +// dropping them would blank a dashboard for a network mount that is merely slow, +// which sets Stale routinely — but a scrape carries no `stale` field the way the +// JSON surfaces do. Without a series saying so, a fill alert sits green forever +// on a volume that stopped answering at 40% and has been filling since. +func TestCollectorMarksStaleDiskReadings(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{Disks: []DiskStats{ + {Role: ScratchDiskRole, UsedGB: 100, TotalGB: 500}, + {Role: "library-1", UsedGB: 400, TotalGB: 500, Stale: true}, + {Role: "library-2", Unavailable: true}, + }}, + }) + + stale := gatherLabeled(t, sampler, "streamapp_node_disk_stale") + if got, ok := stale[ScratchDiskRole]; !ok || got != 0 { + t.Fatalf("scratch stale = %v (present=%v), want a measured 0", got, ok) + } + if got, ok := stale["library-1"]; !ok || got != 1 { + t.Fatalf("library-1 stale = %v (present=%v), want 1", got, ok) + } + // An unavailable mount has no measurement at all, so it exports nothing — + // including no staleness, which would imply there were numbers to qualify. + if _, ok := stale["library-2"]; ok { + t.Fatalf("unavailable mount exported a staleness series: %v", stale) + } + + // The values themselves still ship for the stale mount: they are real, only + // old, and an operator needs to see a volume that was nearly full. + used := gatherLabeled(t, sampler, "streamapp_node_disk_used_bytes") + if got, ok := used["library-1"]; !ok || got != 400*float64(1024*1024*1024) { + t.Fatalf("library-1 used = %v (present=%v), want the carried-over reading kept", got, ok) + } + if _, ok := used["library-2"]; ok { + t.Fatalf("unavailable mount exported a used-bytes series: %v", used) + } +} diff --git a/internal/nodemetrics/nvidia.go b/internal/nodemetrics/nvidia.go index dcd3759c8..457ffe61c 100644 --- a/internal/nodemetrics/nvidia.go +++ b/internal/nodemetrics/nvidia.go @@ -77,23 +77,68 @@ func (g nvidiaGPU) videoUtil() *int { func ptr[T any](value T) *T { return &value } -// sourceBreaker retires an enrichment source after repeated failure. +// sourceRetryInterval is how long a retired source waits before one +// probationary query. +// +// The breaker exists so a host without the NVIDIA toolkit stops spawning a +// doomed subprocess every five seconds, and at this interval it still does: +// one exec per ten minutes instead of a hundred and twenty. What it must not do +// is confuse "this host has no toolkit" with "this driver is resetting", which +// look identical for the handful of samples the limit counts. Retiring the +// source until the process restarts turns a recoverable outage into a node that +// reports no GPU utilization or VRAM for as long as it stays up. +const sourceRetryInterval = 10 * time.Minute + +// sourceBreaker retires an enrichment source after repeated failure, and lets +// it back in on probation. type sourceBreaker struct { + // mu guards the fields below. The sampling goroutine is their only regular + // writer, but reset comes from whatever goroutine serves a re-probe. + mu sync.Mutex name string failures int tripped bool - logOnce sync.Once + // retryAt is when a tripped source may next be tried. It advances on every + // attempt, so one probationary query runs per interval whether or not it + // succeeds. + retryAt time.Time + logOnce sync.Once } -// allow reports whether the source may be queried. -func (b *sourceBreaker) allow() bool { return !b.tripped } +// allow reports whether the source may be queried, admitting one probationary +// query per sourceRetryInterval once the breaker has tripped. +func (b *sourceBreaker) allow(now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + if !b.tripped { + return true + } + if now.Before(b.retryAt) { + return false + } + b.retryAt = now.Add(sourceRetryInterval) + return true +} -func (b *sourceBreaker) succeeded() { b.failures = 0 } +// succeeded clears the failure count, and closes the breaker when the query +// that succeeded was a probationary one. +func (b *sourceBreaker) succeeded() { + b.mu.Lock() + defer b.mu.Unlock() + b.failures = 0 + if b.tripped { + b.tripped = false + slog.Info("node metrics source answered again", "component", "nodemetrics", "source", b.name) + } +} // failed records one failure and trips the breaker at the limit, logging the // retirement exactly once so a node without the toolkit does not narrate it -// every interval. -func (b *sourceBreaker) failed(err error) { +// every interval. A probationary query that fails costs nothing further: allow +// has already pushed the next attempt out by a full interval. +func (b *sourceBreaker) failed(now time.Time, err error) { + b.mu.Lock() + defer b.mu.Unlock() if b.tripped { return } @@ -102,35 +147,65 @@ func (b *sourceBreaker) failed(err error) { return } b.tripped = true + b.retryAt = now.Add(sourceRetryInterval) b.logOnce.Do(func() { - slog.Info("node metrics source unavailable; not retrying until restart", - "component", "nodemetrics", "source", b.name, "failures", b.failures, "error", err) + slog.Info("node metrics source unavailable; retrying occasionally", + "component", "nodemetrics", "source", b.name, "failures", b.failures, + "retry_interval", sourceRetryInterval, "error", err) }) } +// reset returns the source to service immediately. +// +// It is what an operator's hardware re-probe calls: a re-probe is the explicit +// statement that something changed underneath this node, which is exactly the +// event the retry interval is otherwise waiting to discover on its own. +func (b *sourceBreaker) reset() { + b.mu.Lock() + defer b.mu.Unlock() + b.failures = 0 + b.tripped = false + b.retryAt = time.Time{} +} + // queryNVIDIA runs one bounded nvidia-smi query, honoring the breaker. func (s *Sampler) queryNVIDIA(ctx context.Context) []nvidiaGPU { - if !s.nvidiaBreaker.allow() { + now := s.now() + if !s.nvidiaBreaker.allow(now) { return nil } queryCtx, cancel := context.WithTimeout(ctx, nvidiaSMITimeout) defer cancel() output, err := s.runNVIDIASMI(queryCtx) if err != nil { - s.nvidiaBreaker.failed(err) + s.nvidiaBreaker.failed(now, err) return nil } gpus := parseNVIDIASMI(output) if len(gpus) == 0 { // A successful command that says nothing is as useless as a failure and // is how a stale query syntax presents. - s.nvidiaBreaker.failed(errNoNVIDIARows) + s.nvidiaBreaker.failed(now, errNoNVIDIARows) return nil } s.nvidiaBreaker.succeeded() return gpus } +// RetrySources returns every retired enrichment source to service. +// +// A node's hardware re-probe calls it: the operator is saying that something +// changed underneath this process, which is the same event the breaker's retry +// interval would otherwise take up to sourceRetryInterval to notice on its own. +// A driver reinstalled or a toolkit added should show in the next sample, not in +// ten minutes. +func (s *Sampler) RetrySources() { + if s == nil { + return + } + s.nvidiaBreaker.reset() +} + // errNoNVIDIARows marks a query that succeeded but produced nothing parseable. var errNoNVIDIARows = errors.New("nvidia-smi returned no parseable rows") diff --git a/internal/nodemetrics/nvidia_test.go b/internal/nodemetrics/nvidia_test.go index 50f42059b..49df6d4a3 100644 --- a/internal/nodemetrics/nvidia_test.go +++ b/internal/nodemetrics/nvidia_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" ) const nvidiaSMIOutput = "0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n" + @@ -323,3 +324,91 @@ func TestNVIDIACircuitBreakerResetsOnSuccess(t *testing.T) { t.Fatalf("nvidia-smi invoked %d times, want the failure count reset by the success", calls) } } + +// Retiring a source until the process restarts confuses "this host has no +// toolkit" with "this driver is resetting" — they look identical for the five +// samples the limit counts. An NVENC node that hit a transient driver outage +// would then report no utilization and no VRAM for as long as it stayed up. +func TestNVIDIACircuitBreakerHalfOpensAfterItsRetryInterval(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + fail := true + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("nvidia-smi: driver/library version mismatch") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit + 5 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d failures", calls, sourceFailureLimit) + } + + // Still retired just short of the interval: the point of the breaker is that + // a toolkit-less host stops paying for a subprocess every few seconds. + clock.advance(sourceRetryInterval - time.Second) + s.queryNVIDIA(context.Background()) + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times before the retry interval elapsed, want %d", calls, sourceFailureLimit) + } + + // One probationary query per interval, and a failure buys another interval + // rather than a burst. + clock.advance(2 * time.Second) + s.queryNVIDIA(context.Background()) + s.queryNVIDIA(context.Background()) + if calls != sourceFailureLimit+1 { + t.Fatalf("nvidia-smi invoked %d times, want exactly one probationary query", calls) + } + + // The driver comes back. The probationary query that succeeds closes the + // breaker, and sampling resumes at its normal rate. + fail = false + clock.advance(sourceRetryInterval) + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("recovered query returned %d gpus, want 2", len(gpus)) + } + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("query after recovery returned %d gpus, want the breaker closed", len(gpus)) + } + if calls != sourceFailureLimit+3 { + t.Fatalf("nvidia-smi invoked %d times, want the breaker to stop gating once it answered", calls) + } +} + +// A re-probe is an operator saying something changed underneath this node, so +// it must not wait out the retry interval to find out. +func TestRetrySourcesReturnsARetiredSourceImmediately(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + fail := true + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("nvidia-smi: command not found") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit + 2 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired", calls) + } + + // The toolkit is installed; the operator re-probes rather than waiting. + fail = false + s.RetrySources() + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("query after RetrySources returned %d gpus, want the source back in service", len(gpus)) + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index 8d86a5b46..4f324581d 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -80,6 +80,11 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques playback.InvalidateHWProbeCache() tonemap.InvalidateProbeCache() + // The resource sampler retires nvidia-smi after repeated failure, and a + // driver that was broken at start is exactly what a re-probe is called for. + // Without this the node re-verifies its encoders here and still reports no + // GPU utilization until the breaker's own retry interval comes round. + s.metrics.RetrySources() // buildCapabilitySnapshotLocked owns the probe deadline, so a re-probe can // never cost more than a cold capability fetch already may. From 7f2259f9ef2347ccd59379316bb4cbd0bf93dcb7 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:56:01 -0400 Subject: [PATCH 042/163] fix(nodemetrics): read this process's cgroup; size capability fetches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every cgroup file was named from the mount root. Inside a container that is right — the cgroup namespace makes the container's own cgroup appear as the root — but a process limited without being namespaced sees something else. A systemd unit with CPUQuota= or MemoryMax= lives at /sys/fs/cgroup/system.slice/silo.service/, and the root files there describe the whole machine, so a service pegged at its quota reported host-wide busyness and the host's memory total: mostly idle with plenty of memory left, which is exactly the failure the cgroup correction was written to prevent, reintroduced for anyone running Silo as a plain service. Each path is now tried at this process's own cgroup first and at the root second. The fallback is what keeps every container case identical: a namespaced container reports "/" and rewrites to the root anyway, and one without a cgroup namespace reports a host path that does not exist under its own mount, so the rewrite fails to open and the root read happens as before. CPU layouts move all their files together or none — this process's usage over the root's quota would divide a service's own CPU time by the whole machine's budget. Separately, capability fetches were bounded at a flat two minutes while the probe matrix they wait on grows with the configured device count: a node with two render devices advertises 136 seconds and was cut off at 120, so its first fetch failed and its inventory waited for a later warmed retry. The bound now comes from tonemap.ProbeRequestTimeout over the live configuration, floored at the old constant, and the health sweep's own bound is documented and raised to what it actually is — a backstop against a fetcher that never returns, not a budget. A backstop that trips during ordinary operation cannot be told from the bug it was meant to catch. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 47 +++++- cmd/silo/main_test.go | 52 +++++- internal/nodemetrics/cgrouppath.go | 151 +++++++++++++++++ internal/nodemetrics/cgrouppath_test.go | 205 ++++++++++++++++++++++++ internal/nodemetrics/meminfo.go | 9 +- internal/nodemetrics/sampler.go | 33 ++-- internal/nodemetrics/sampler_test.go | 2 +- internal/nodepool/health.go | 19 ++- 8 files changed, 488 insertions(+), 30 deletions(-) create mode 100644 internal/nodemetrics/cgrouppath.go create mode 100644 internal/nodemetrics/cgrouppath_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 7a90c9797..45fa4f7d3 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -110,6 +110,7 @@ import ( "github.com/Silo-Server/silo-server/internal/taskmanager/tasks" "github.com/Silo-Server/silo-server/internal/taskmanager/triggers" "github.com/Silo-Server/silo-server/internal/telemetry" + "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodenode" "github.com/Silo-Server/silo-server/internal/usercollections" "github.com/Silo-Server/silo-server/internal/userdb" @@ -156,9 +157,18 @@ func resolveNodeIdentity() string { // The hash comes out of the payload itself, because only the node knows what it // hashed. A report without one is refused rather than given a synthetic hash, // which would make an old node look like it had capability tracking. -func nodeCapabilityFetcher(jwtSecret string) nodepool.CapabilityFetcher { - client := &http.Client{Timeout: nodeCapabilityRequestTimeout} +// The request bound comes from budget rather than a constant. A cold node's +// answer runs the whole FFmpeg probe matrix, and the size of that matrix is a +// function of how many hardware devices playback is configured with — a node +// with two render devices legitimately advertises a budget past two minutes. +// Bounding every fetch at a fixed two minutes abandons such a node mid-probe +// and reports a fetch failure for a node operating inside its published +// contract. +func nodeCapabilityFetcher(jwtSecret string, budget func() time.Duration) nodepool.CapabilityFetcher { + client := &http.Client{} return func(ctx context.Context, nodeURL string) ([]byte, string, error) { + ctx, cancel := context.WithTimeout(ctx, budget()) + defer cancel() info, payload, status, err := transcodenode.FetchHWCapabilitiesPayload(ctx, client, nodeURL, jwtSecret) if err != nil { return nil, "", err @@ -223,12 +233,34 @@ func cachedLibraryPaths(query func(context.Context) ([]string, error)) func(cont // libraryPathQueryTimeout bounds the per-sample library root lookup. const libraryPathQueryTimeout = 2 * time.Second -// nodeCapabilityRequestTimeout bounds one capability request. A cold node runs -// ffmpeg probes to answer and advertises a probe budget of up to ~2 minutes; -// the fetch runs detached from the health sweep, so matching that budget is -// safe and lets a cold node's first report land instead of timing out. +// nodeCapabilityRequestTimeout is the floor under one capability request. +// +// The real bound is the node's own advertised probe budget, which +// nodeCapabilityProbeBudget computes from the configured hardware — that grows +// with the device count and passes two minutes at two devices. This floor covers +// the rest of what a fetch does (transport, a node answering from a warm cache +// but under load) and keeps a misconfigured or unreadable setting from +// producing an absurdly small bound. The fetch runs detached from the health +// sweep, so a generous bound costs the sweep nothing and lets a cold node's +// first report land instead of timing out. const nodeCapabilityRequestTimeout = 2 * time.Minute +// nodeCapabilityProbeBudget reports how long to allow one capability fetch, +// read from the live configuration on every call so an operator adding a second +// render device does not have to restart the API for its fetches to stop being +// cut short. +func nodeCapabilityProbeBudget(live func() *config.Config) func() time.Duration { + return func() time.Duration { + hwAccel, hwDevice := "", "" + if live != nil { + if current := live(); current != nil { + hwAccel, hwDevice = current.Playback.HWAccel, current.Playback.HWDevice + } + } + return max(nodeCapabilityRequestTimeout, tonemap.ProbeRequestTimeout(hwAccel, hwDevice)) + } +} + func clientIPResolverFromConfig(cfg *config.Config) (*clientip.Resolver, error) { if cfg == nil { return nil, fmt.Errorf("config is not loaded") @@ -1181,7 +1213,8 @@ func main() { deps.NodePlanner = nodepool.NewPlanner(proxyPool, transcodePool) healthChecker := nodepool.NewHealthChecker(proxyPool, transcodePool, nodeRepo) - healthChecker.SetCapabilityFetcher(nodeCapabilityFetcher(cfg.Auth.JWTSecret)) + healthChecker.SetCapabilityFetcher( + nodeCapabilityFetcher(cfg.Auth.JWTSecret, nodeCapabilityProbeBudget(configWatcher.Config))) deps.NodeHealthChecker = healthChecker healthChecker.Start(appCtx) slog.Info("node pools initialized", "proxy_nodes", len(proxyNodes), "transcode_nodes", len(transcodeNodes)) diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index d433d7d4b..a792d38bc 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -15,6 +15,7 @@ import ( pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" "github.com/Silo-Server/silo-server/internal/api" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/plugins" "github.com/Silo-Server/silo-server/internal/tonemap" @@ -398,7 +399,7 @@ func TestNodeCapabilityFetcherStoresTheNodesOwnBytes(t *testing.T) { })) t.Cleanup(server.Close) - payload, hash, err := nodeCapabilityFetcher("secret")(context.Background(), server.URL) + payload, hash, err := nodeCapabilityFetcher("secret", nodeCapabilityProbeBudget(nil))(context.Background(), server.URL) if err != nil { t.Fatalf("nodeCapabilityFetcher: %v", err) } @@ -417,3 +418,52 @@ func TestNodeCapabilityFetcherStoresTheNodesOwnBytes(t *testing.T) { t.Fatalf("resolved = %v, want the report's own value", stored["resolved"]) } } + +// A cold node's answer runs the whole FFmpeg probe matrix, and that matrix grows +// with the configured device count: a node with two render devices legitimately +// advertises a request budget past two minutes. Bounding every fetch at a fixed +// two minutes abandons such a node mid-probe and reports a failure for a node +// operating inside its published contract. +func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { + single := &config.Config{} + single.Playback.HWAccel, single.Playback.HWDevice = "qsv", "/dev/dri/renderD128" + pair := &config.Config{} + pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" + + oneDevice := nodeCapabilityProbeBudget(func() *config.Config { return single })() + twoDevices := nodeCapabilityProbeBudget(func() *config.Config { return pair })() + + if want := tonemap.ProbeRequestTimeout("qsv", pair.Playback.HWDevice); twoDevices != want { + t.Fatalf("two-device budget = %v, want the node's own advertised %v", twoDevices, want) + } + if twoDevices <= nodeCapabilityRequestTimeout { + t.Fatalf("two-device budget = %v, want more than the %v floor — that is the case that was being cut short", + twoDevices, nodeCapabilityRequestTimeout) + } + if twoDevices <= oneDevice { + t.Fatalf("two-device budget %v is not above the one-device %v; the budget does not track the matrix", + twoDevices, oneDevice) + } + + // Nothing configured, or no live config at all, still gets a usable bound + // rather than zero. + if got := nodeCapabilityProbeBudget(nil)(); got < nodeCapabilityRequestTimeout { + t.Fatalf("budget with no configuration = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) + } + if got := nodeCapabilityProbeBudget(func() *config.Config { return nil })(); got < nodeCapabilityRequestTimeout { + t.Fatalf("budget with a nil config = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) + } +} + +// The backstop the health sweep puts around the same fetch must never be the +// thing that cuts it short, or the budget above is decorative. +func TestCapabilityFetchBackstopExceedsTheAdvertisedBudget(t *testing.T) { + pair := &config.Config{} + pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" + budget := nodeCapabilityProbeBudget(func() *config.Config { return pair })() + + if nodepool.CapabilityRefreshTimeout <= budget { + t.Fatalf("health sweep backstop %v does not exceed the %v a two-device node advertises", + nodepool.CapabilityRefreshTimeout, budget) + } +} diff --git a/internal/nodemetrics/cgrouppath.go b/internal/nodemetrics/cgrouppath.go new file mode 100644 index 000000000..3a9f0ab34 --- /dev/null +++ b/internal/nodemetrics/cgrouppath.go @@ -0,0 +1,151 @@ +package nodemetrics + +import ( + "os" + "path" + "strings" +) + +// Resolving this process's own cgroup. +// +// Every cgroup file this package reads is named from the mount root — +// /sys/fs/cgroup/cpu.max and friends. Inside a container that is exactly right: +// the cgroup namespace makes the container's own cgroup appear as the root, so +// the root files describe the container. +// +// A process that is limited *without* being namespaced sees something else. A +// systemd unit with CPUQuota= or MemoryMax= lives at +// /sys/fs/cgroup/system.slice/silo.service/, and the root files there describe +// the whole machine. Reading them reports host-wide CPU busyness and the host's +// memory total, so a service pegged at its quota looks mostly idle with plenty +// of memory left — the exact failure the cgroup correction exists to prevent, +// reintroduced for anyone who runs Silo as a plain service rather than a +// container. +// +// So each path is tried at this process's own cgroup first and at the root +// second. The fallback is what keeps every container case working unchanged: +// a namespaced container reports "/" and rewrites to the root anyway, and a +// container without a cgroup namespace reports a host path +// (/docker/) that does not exist under its own mount, so the rewritten +// path simply fails to open and the root read happens as before. + +// cgroupMountRoot is where the cgroup hierarchy is mounted on Linux. +const cgroupMountRoot = "/sys/fs/cgroup" + +// cgroupRelativePaths reports this process's path within each cgroup hierarchy, +// read from /self/cgroup. +// +// The v2 unified hierarchy has an empty controller field and is keyed by "". +// A v1 line names one or more controllers, and is keyed both by each controller +// individually and by the whole comma-joined field, because that joined form is +// what the mount directory is named ("cpu,cpuacct"). +// +// A missing or unreadable file yields no entries, which leaves every path at +// the root — the behavior this process had before. +func cgroupRelativePaths(procDir string) map[string]string { + raw, err := os.ReadFile(path.Join(procDir, "self", "cgroup")) + if err != nil { + return nil + } + paths := map[string]string{} + for line := range strings.Lines(string(raw)) { + // "::", and the path may itself + // contain colons, so only the first two separators are structural. + fields := strings.SplitN(strings.TrimSpace(line), ":", 3) + if len(fields) != 3 { + continue + } + controllers, relative := fields[1], strings.TrimPrefix(fields[2], "/") + if relative == "" { + // Already at the root of its hierarchy: a namespaced container, or + // an unconstrained process. Recording it would rewrite to the same + // place at extra cost. + continue + } + paths[controllers] = relative + for _, controller := range strings.Split(controllers, ",") { + if controller != "" { + paths[controller] = relative + } + } + } + if len(paths) == 0 { + return nil + } + return paths +} + +// cgroupSelfFile rewrites a root-relative cgroup file path to this process's own +// cgroup, or returns "" when it already reads the right file. +// +// The controller is taken from the path itself: a v1 file sits under a +// controller directory (/sys/fs/cgroup/memory/memory.stat), a v2 file sits +// directly at the root (/sys/fs/cgroup/memory.stat) and belongs to the unified +// hierarchy, which cgroupRelativePaths keys by "". +func cgroupSelfFile(relative map[string]string, file string) string { + if len(relative) == 0 { + return "" + } + rest, ok := strings.CutPrefix(file, cgroupMountRoot+"/") + if !ok { + return "" + } + controller, name := "", rest + if dir, base, found := strings.Cut(rest, "/"); found { + controller, name = dir, base + } + own, ok := relative[controller] + if !ok || name == "" { + return "" + } + return path.Join(cgroupMountRoot, controller, own, name) +} + +// withCgroupSelfPaths returns files preceded by their this-process equivalents, +// so a read tries the process's own cgroup before falling back to the root. +func withCgroupSelfPaths(relative map[string]string, files []string) []string { + out := make([]string, 0, len(files)*2) + for _, file := range files { + if own := cgroupSelfFile(relative, file); own != "" { + out = append(out, own) + } + out = append(out, file) + } + return out +} + +// withCgroupSelfCPUPaths is withCgroupSelfPaths for the CPU layouts, where one +// entry names several files that must be rewritten together — a usage file from +// this process's cgroup paired with the root's quota would normalize the +// service's own CPU time against the whole machine's budget. +func withCgroupSelfCPUPaths(relative map[string]string, layouts []cgroupCPUPath) []cgroupCPUPath { + out := make([]cgroupCPUPath, 0, len(layouts)*2) + for _, layout := range layouts { + own := layout + own.usage = cgroupSelfFile(relative, layout.usage) + own.quota = cgroupSelfFile(relative, layout.quota) + if layout.period != "" { + own.period = cgroupSelfFile(relative, layout.period) + } + if own.usage != "" && own.quota != "" && (layout.period == "" || own.period != "") { + out = append(out, own) + } + out = append(out, layout) + } + return out +} + +// withCgroupSelfUsagePaths is withCgroupSelfPaths for the memory usage layouts. +func withCgroupSelfUsagePaths(relative map[string]string, layouts []cgroupUsagePath) []cgroupUsagePath { + out := make([]cgroupUsagePath, 0, len(layouts)*2) + for _, layout := range layouts { + own := layout + own.usage = cgroupSelfFile(relative, layout.usage) + own.stat = cgroupSelfFile(relative, layout.stat) + if own.usage != "" && own.stat != "" { + out = append(out, own) + } + out = append(out, layout) + } + return out +} diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go new file mode 100644 index 000000000..b5252ec9c --- /dev/null +++ b/internal/nodemetrics/cgrouppath_test.go @@ -0,0 +1,205 @@ +package nodemetrics + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +// writeSelfCgroup lays down a /self/cgroup with the given body. +func writeSelfCgroup(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "self"), 0o755); err != nil { + t.Fatalf("create self dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "self", "cgroup"), []byte(body), 0o644); err != nil { + t.Fatalf("write self/cgroup: %v", err) + } + return dir +} + +func TestCgroupRelativePaths(t *testing.T) { + tests := []struct { + name string + body string + want map[string]string + }{ + { + // A systemd unit with CPUQuota= or MemoryMax=. Its limits live + // below the mount root, so the root files describe the machine. + name: "systemd unit on cgroup v2", + body: "0::/system.slice/silo.service\n", + want: map[string]string{"": "system.slice/silo.service"}, + }, + { + // v1 names its controllers, and the mount directory is named with + // the whole comma-joined field, so both forms have to resolve. + name: "cgroup v1 controllers", + body: "9:cpu,cpuacct:/system.slice/silo.service\n5:memory:/system.slice/silo.service\n", + want: map[string]string{ + "cpu,cpuacct": "system.slice/silo.service", + "cpu": "system.slice/silo.service", + "cpuacct": "system.slice/silo.service", + "memory": "system.slice/silo.service", + }, + }, + { + // A namespaced container is already at its own root, which is + // exactly what the unrewritten paths read. + name: "namespaced container", + body: "0::/\n", + want: nil, + }, + { + name: "unreadable lines are skipped", + body: "garbage\n0::/system.slice/silo.service\n", + want: map[string]string{"": "system.slice/silo.service"}, + }, + { + // The path field may contain colons; only the first two separators + // are structural. + name: "path containing a colon", + body: "0::/system.slice/silo:one.service\n", + want: map[string]string{"": "system.slice/silo:one.service"}, + }, + {name: "empty file", body: "", want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cgroupRelativePaths(writeSelfCgroup(t, tt.body)) + if len(got) != len(tt.want) { + t.Fatalf("cgroupRelativePaths() = %v, want %v", got, tt.want) + } + for key, value := range tt.want { + if got[key] != value { + t.Fatalf("cgroupRelativePaths()[%q] = %q, want %q", key, got[key], value) + } + } + }) + } +} + +// A missing file is the ordinary case on a host that is not Linux, or one whose +// /proc is not where we looked. It must read as "no rewrite", never as an error +// that costs the root reading too. +func TestCgroupRelativePathsWithoutTheFile(t *testing.T) { + if got := cgroupRelativePaths(t.TempDir()); got != nil { + t.Fatalf("cgroupRelativePaths() = %v, want none when /self/cgroup is absent", got) + } +} + +func TestCgroupSelfFile(t *testing.T) { + v2 := map[string]string{"": "system.slice/silo.service"} + v1 := map[string]string{"cpu,cpuacct": "system.slice/silo.service", "memory": "system.slice/silo.service"} + + tests := []struct { + name string + relative map[string]string + file string + want string + }{ + { + name: "v2 file sits at the mount root", relative: v2, + file: cgroupCPUPaths[0].quota, + want: "/sys/fs/cgroup/system.slice/silo.service/cpu.max", + }, + { + name: "v1 file sits under its controller", relative: v1, + file: "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage", + want: "/sys/fs/cgroup/cpu,cpuacct/system.slice/silo.service/cpuacct.usage", + }, + { + // v1 memory under v2 membership: no unified entry names it, so the + // root path stands rather than being rewritten into nonsense. + name: "controller this process has no membership for", relative: v2, + file: "/sys/fs/cgroup/memory/memory.stat", + want: "", + }, + {name: "no membership at all", relative: nil, file: cgroupCPUPaths[0].quota, want: ""}, + {name: "path outside the cgroup mount", relative: v2, file: "/proc/stat", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := cgroupSelfFile(tt.relative, tt.file); got != tt.want { + t.Fatalf("cgroupSelfFile() = %q, want %q", got, tt.want) + } + }) + } +} + +// The rewritten path goes first and the root stays behind it, so a container +// whose /proc names a host path it cannot open still falls through to the read +// that has always worked. +func TestWithCgroupSelfPathsKeepsTheRootFallback(t *testing.T) { + relative := map[string]string{"": "system.slice/silo.service", "memory": "system.slice/silo.service"} + got := withCgroupSelfPaths(relative, CgroupMemoryLimitPaths()) + want := []string{ + "/sys/fs/cgroup/system.slice/silo.service/memory.max", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/system.slice/silo.service/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + } + if !slices.Equal(got, want) { + t.Fatalf("withCgroupSelfPaths() =\n%v\nwant\n%v", got, want) + } + + // With no membership the list is exactly what it was. + if got := withCgroupSelfPaths(nil, CgroupMemoryLimitPaths()); !slices.Equal(got, CgroupMemoryLimitPaths()) { + t.Fatalf("withCgroupSelfPaths(nil) = %v, want the original list", got) + } +} + +// A CPU layout names several files that have to move together: this process's +// usage normalized against the root's quota would divide the service's own CPU +// time by the whole machine's budget. +func TestWithCgroupSelfCPUPathsRewritesEveryFileOrNone(t *testing.T) { + relative := map[string]string{"": "system.slice/silo.service"} + got := withCgroupSelfCPUPaths(relative, cgroupCPUPaths) + + if len(got) != len(cgroupCPUPaths)+1 { + t.Fatalf("got %d layouts, want only the v2 one rewritten alongside the originals", len(got)) + } + own := got[0] + if own.usage != "/sys/fs/cgroup/system.slice/silo.service/cpu.stat" { + t.Fatalf("usage = %q, want this process's own cpu.stat", own.usage) + } + if own.quota != "/sys/fs/cgroup/system.slice/silo.service/cpu.max" { + t.Fatalf("quota = %q, want this process's own cpu.max", own.quota) + } + if got[1].usage != cgroupCPUPaths[0].usage { + t.Fatalf("got[1].usage = %q, want the root layout kept behind it", got[1].usage) + } + // The v1 layouts have no unified membership to resolve, so they are carried + // through unrewritten rather than half-rewritten. + for _, layout := range got[1:] { + if layout.usage != "" && layout.quota == "" { + t.Fatalf("layout %+v has a usage file with no quota file", layout) + } + } +} + +func TestWithCgroupSelfUsagePathsRewritesEveryFileOrNone(t *testing.T) { + relative := map[string]string{"memory": "system.slice/silo.service"} + got := withCgroupSelfUsagePaths(relative, cgroupMemoryUsagePaths) + + if len(got) != len(cgroupMemoryUsagePaths)+1 { + t.Fatalf("got %d layouts, want only the v1 one rewritten alongside the originals", len(got)) + } + var rewritten *cgroupUsagePath + for i, layout := range got { + if layout.usage == "/sys/fs/cgroup/memory/system.slice/silo.service/memory.usage_in_bytes" { + rewritten = &got[i] + } + } + if rewritten == nil { + t.Fatalf("no rewritten v1 layout in %+v", got) + } + if rewritten.stat != "/sys/fs/cgroup/memory/system.slice/silo.service/memory.stat" { + t.Fatalf("stat = %q, want it rewritten beside its usage file", rewritten.stat) + } + if rewritten.inactiveFile != cgroupInactiveFileKeyV1 { + t.Fatalf("inactiveFile = %q, want the layout's own key preserved", rewritten.inactiveFile) + } +} diff --git a/internal/nodemetrics/meminfo.go b/internal/nodemetrics/meminfo.go index bb8795819..efb511a2b 100644 --- a/internal/nodemetrics/meminfo.go +++ b/internal/nodemetrics/meminfo.go @@ -14,7 +14,12 @@ const bytesPerKB = int64(1024) // cgroupInactiveFileKeyV2 is the cgroup v2 memory.stat key for reclaimable // page cache. cgroup v1's memory.stat spells the same figure // "total_inactive_file". -const cgroupInactiveFileKeyV2 = "inactive_file" +const ( + cgroupInactiveFileKeyV2 = "inactive_file" + // cgroupInactiveFileKeyV1 is the same quantity under cgroup v1, which + // prefixes its memory.stat totals. + cgroupInactiveFileKeyV1 = "total_inactive_file" +) // CgroupMemoryLimitPaths returns the memory-limit files to consult, cgroup v2 // first. A container that has a limit publishes it in exactly one of these; a @@ -52,7 +57,7 @@ var cgroupMemoryUsagePaths = []cgroupUsagePath{ { usage: "/sys/fs/cgroup/memory/memory.usage_in_bytes", stat: "/sys/fs/cgroup/memory/memory.stat", - inactiveFile: "total_inactive_file", + inactiveFile: cgroupInactiveFileKeyV1, }, } diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index ccbd785a1..178bc3463 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -133,25 +133,32 @@ func NewSampler(opts Options) *Sampler { } procDir := "/proc" hostProcDir := "/host/proc" + // Read from this process's own /proc, never the lxcfs-mounted host view: + // the question is which cgroup *this* process belongs to. + cgroupSelf := cgroupRelativePaths(procDir) ffmpegPIDs := opts.FFmpegChildren if ffmpegPIDs == nil { pid := os.Getpid() ffmpegPIDs = func() []int { return defaultFFmpegChildren(procDir, pid) } } s := &Sampler{ - interval: interval, - now: now, - goos: runtime.GOOS, - scratchDir: opts.ScratchDir, - mediaRoots: opts.MediaRoots, - sessions: opts.DeviceSessions, - identities: opts.DeviceIdentities, - ffmpegPIDs: ffmpegPIDs, - procDir: procDir, - hostProcDir: hostProcDir, - cgroupLimitPaths: CgroupMemoryLimitPaths(), - cgroupUsagePaths: slices.Clone(cgroupMemoryUsagePaths), - cgroupCPUPaths: slices.Clone(cgroupCPUPaths), + interval: interval, + now: now, + goos: runtime.GOOS, + scratchDir: opts.ScratchDir, + mediaRoots: opts.MediaRoots, + sessions: opts.DeviceSessions, + identities: opts.DeviceIdentities, + ffmpegPIDs: ffmpegPIDs, + procDir: procDir, + hostProcDir: hostProcDir, + // Each cgroup read is tried at this process's own cgroup before the + // mount root, so a systemd unit with CPUQuota= or MemoryMax= is + // measured against its limit rather than against the whole machine. + // See cgrouppath.go; a container is unaffected either way. + cgroupLimitPaths: withCgroupSelfPaths(cgroupSelf, CgroupMemoryLimitPaths()), + cgroupUsagePaths: withCgroupSelfUsagePaths(cgroupSelf, cgroupMemoryUsagePaths), + cgroupCPUPaths: withCgroupSelfCPUPaths(cgroupSelf, cgroupCPUPaths), prevGPU: map[fdinfoClient]engineCounters{}, disks: map[string]*diskEntry{}, statfs: osStatfs, diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 585215e49..2e80b8176 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -232,7 +232,7 @@ func TestMemoryCorrectedByCgroupLimitAndUsage(t *testing.T) { usageBody: "2147483648\n", statFile: "memory.stat", statBody: "total_inactive_file 1073741824\n", - inactiveKey: "total_inactive_file", + inactiveKey: cgroupInactiveFileKeyV1, wantTotalMB: 4096, wantUsedMB: 1024, }, diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 6ed3ae152..59b1c1ef8 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -141,12 +141,19 @@ func trimJSONNull(raw json.RawMessage) json.RawMessage { // persisted. type CapabilityFetcher func(ctx context.Context, nodeURL string) (payload []byte, hash string, err error) -// capabilityFetchTimeout bounds one capability fetch. Node-side capability -// answers can involve ffmpeg probes on a cold cache — the node's own advertised -// probe budget reaches ~2 minutes — and the fetch runs detached from the -// health sweep, so the bound covers a genuinely cold node rather than -// abandoning it every sweep. -const capabilityFetchTimeout = 2 * time.Minute +// capabilityFetchTimeout is the backstop on one capability fetch, not its +// budget. +// +// The budget belongs to the fetcher, which knows the configured hardware and +// sizes each request against the node's own advertised probe matrix; that grows +// with the device count and passes two minutes at two devices, so a fixed bound +// here would cut short a node operating well inside its published contract. What +// this stops is the other failure: a fetcher that never returns pinning a +// goroutine and a node's inventory forever. It is deliberately far above any +// budget a real configuration produces — five minutes covers a node probing +// roughly seven devices — because a backstop that trips during ordinary +// operation is indistinguishable from the bug it was meant to catch. +const capabilityFetchTimeout = 5 * time.Minute // CapabilityRefreshTimeout is the bound RefreshNodeCapabilities puts on the // fetch it performs. It is exported for the one caller that has to hold an HTTP From b63e88cdc62ab02dd30f1c1f7b1a284eb5ca0bcc Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:03:59 -0400 Subject: [PATCH 043/163] test(jellycompat): wait on terminal state, not on the scrobble receipt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go check failed on d7e4176 with TestActiveEncodingsFallbackAllowsLaterAuthoritativeStop, which this branch does not otherwise touch. It is a real race in the test rather than a fluke: dispatchCompatScrobbleEventConfirmed puts the event on a buffered channel and returns, and the lease release that sets TerminalFallbackSent runs after it. A test that reads the store the instant it receives can win that race and see the flag unset, which is likelier on a loaded runner than on a laptop — it had passed the four CI runs before it. Four assertions had that shape, in both directions: two waiting for TerminalFallbackSent to appear and three for a completed event to be gone, all synchronizing on the channel receipt instead of on the state they assert. They now wait on the state, bounded, per the repository's rule that a test waits on observable state rather than on ordering it does not control. No production code changed. Verified by running the four at -count=30. Co-Authored-By: Claude Opus 5 (1M context) --- .../jellycompat/playback_scrobble_test.go | 79 +++++++++++++++---- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/internal/jellycompat/playback_scrobble_test.go b/internal/jellycompat/playback_scrobble_test.go index 221868bb4..8e39933cd 100644 --- a/internal/jellycompat/playback_scrobble_test.go +++ b/internal/jellycompat/playback_scrobble_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "runtime" "strconv" "strings" "sync" @@ -604,9 +605,9 @@ func TestActiveEncodingsFallbackAllowsLaterAuthoritativeStop(t *testing.T) { case <-time.After(time.Second): t.Fatal("timed out waiting for ActiveEncodings terminal fallback") } - terminal, ok := store.GetFinalizable("play-1", "token-1") - if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { - t.Fatalf("fallback terminal state = ok=%v session=%+v", ok, terminal) + terminal := awaitTerminalFallbackSent(t, store, "play-1", "token-1") + if terminal.TerminalAuthoritative { + t.Fatalf("fallback terminal state = %+v, want it not marked authoritative", terminal) } stoppedReq := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( @@ -685,9 +686,9 @@ func TestPositionlessLateStopPreservesAndDeliversPendingFallback(t *testing.T) { case <-time.After(time.Second): t.Fatal("timed out waiting for preserved terminal fallback") } - terminal, ok = store.GetFinalizable("play-1", "token-1") - if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { - t.Fatalf("delivered fallback state = ok=%v session=%+v", ok, terminal) + terminal = awaitTerminalFallbackSent(t, store, "play-1", "token-1") + if terminal.TerminalAuthoritative { + t.Fatalf("delivered fallback state = %+v, want it not marked authoritative", terminal) } } @@ -723,9 +724,8 @@ func TestStoppedScrobbleQueueFailureRetainsAndRetriesTerminalEvent(t *testing.T) case <-time.After(2 * time.Second): t.Fatal("timed out waiting for terminal queue retry") } - if _, ok := handler.playbackStore.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("authoritative terminal event remained after successful retry") - } + awaitTerminalCompleted(t, handler.playbackStore, "play-1", "token-1", + "authoritative terminal event remained after successful retry") } func TestStoppedScrobbleRestagesAfterTerminalPersistenceFailure(t *testing.T) { @@ -769,9 +769,8 @@ func TestStoppedScrobbleRestagesAfterTerminalPersistenceFailure(t *testing.T) { if calls := flakyStore.calls(); calls < 2 { t.Fatalf("stage calls = %d, want persistence retry", calls) } - if _, ok := flakyStore.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("restaged authoritative event remained after delivery") - } + awaitTerminalCompleted(t, flakyStore, "play-1", "token-1", + "restaged authoritative event remained after delivery") } func TestStoppedScrobblePreservesExplicitZeroPosition(t *testing.T) { @@ -827,9 +826,7 @@ func TestTerminalScrobbleRecoveryDeliversPersistedEventAfterRestart(t *testing.T case <-time.After(time.Second): t.Fatal("timed out waiting for recovered terminal event") } - if _, ok := store.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("recovered authoritative event remained pending") - } + awaitTerminalCompleted(t, store, "play-1", "token-1", "recovered authoritative event remained pending") } func TestTerminalScrobbleRecoveryWaitsForConfirmedProviderStop(t *testing.T) { @@ -1152,3 +1149,55 @@ func TestTeardownStillCleansLocalPlaybackAfterAnotherCallerClaimsStop(t *testing t.Fatalf("losing teardown emitted provider event: %+v", scrobbler.calls) } } + +// awaitTerminalFallbackSent waits for the delivery path to record that a +// fallback stop landed, and returns the session it recorded it on. +// +// Receiving the scrobble event is not a sufficient signal to read the store on. +// The event goes onto a buffered channel inside +// dispatchCompatScrobbleEventConfirmed, and the lease release that sets +// TerminalFallbackSent runs only after that dispatch returns — so a test that +// reads the store the instant it receives can win the race and see the flag +// unset, which is what made these two assertions fail intermittently under load. +// Waiting on the state itself makes them independent of how the two are +// scheduled. +func awaitTerminalFallbackSent(t *testing.T, store *PlaybackSessionStore, id, token string) *PlaybackSession { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + session, ok := store.GetFinalizable(id, token) + if ok && session.TerminalFallbackSent { + return session + } + if time.Now().After(deadline) { + t.Fatalf("terminal fallback was never recorded: ok=%v session=%+v", ok, session) + } + runtime.Gosched() + } +} + +// awaitTerminalCompleted waits for the delivery path to retire a terminal event. +// +// The mirror of awaitTerminalFallbackSent, and racy for the same reason: +// CompleteTerminal runs after the dispatch that puts the event on the scrobbler +// channel, so a test that reads the store the instant it receives can see the +// entry still present. Waiting for its absence removes the ordering dependence. +func awaitTerminalCompleted(t *testing.T, store terminalFinalizableStore, id, token, message string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + if _, ok := store.GetFinalizable(id, token); !ok { + return + } + if time.Now().After(deadline) { + t.Fatal(message) + } + runtime.Gosched() + } +} + +// terminalFinalizableStore is the one method these waits need, so they work on +// the concrete store and on the handler's interface field alike. +type terminalFinalizableStore interface { + GetFinalizable(id, compatToken string) (*PlaybackSession, bool) +} From 4909bba976b237533ccbecd2b1da582952518562 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:14:54 -0400 Subject: [PATCH 044/163] fix(nodes): size fetches per node, honor inherited cgroup limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are places the previous commit stopped one level short. The capability fetch budget read the cluster-wide playback settings, but a worker builds its probe matrix from its *effective* policy. A node overriding hw_device with two devices needs ~136s on a cluster configured with one, and was still cancelled at the 120s floor — every sweep, so its durable inventory never landed. CapabilityFetcher now takes the node rather than its URL, since the cost of the answer is a property of the node, and the budget prefers its overrides over the cluster setting. An override of "" still means inherit. The cgroup resolution found this process's own cgroup and then fell straight back to the mount root, skipping everything between. That is where the limit usually lives: a systemd unit inherits CPUQuota= or MemoryMax= from the slice containing it, and a container inherits from its pod cgroup. The leaf reads "max" while the kernel throttles anyway, so the walk reported the whole host to a process that has two cores. Limits are now read at every cgroup from the leaf to the root, and the tightest wins — for memory by dropping the break that took the first readable one, for CPU by walking the quota (paired with the period from the same level, since a quota over another cgroup's period describes no real budget). Usage is deliberately not walked: an ancestor's counts every sibling on the machine, while what this process consumed is what its own limit has to be measured against. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 45 ++++++++---- cmd/silo/main_test.go | 36 ++++++--- internal/nodemetrics/cgroupcpu.go | 43 +++++++++-- internal/nodemetrics/cgrouppath.go | 58 ++++++++++++++- internal/nodemetrics/cgrouppath_test.go | 97 +++++++++++++++++++++++++ internal/nodemetrics/sampler_test.go | 58 +++++++++++++++ internal/nodemetrics/system.go | 5 +- internal/nodepool/health.go | 10 ++- internal/nodepool/health_test.go | 6 +- 9 files changed, 321 insertions(+), 37 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 45fa4f7d3..b84991178 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -159,17 +159,22 @@ func resolveNodeIdentity() string { // which would make an old node look like it had capability tracking. // The request bound comes from budget rather than a constant. A cold node's // answer runs the whole FFmpeg probe matrix, and the size of that matrix is a -// function of how many hardware devices playback is configured with — a node -// with two render devices legitimately advertises a budget past two minutes. +// function of how many hardware devices that node will actually probe — two +// render devices legitimately push the advertised budget past two minutes. // Bounding every fetch at a fixed two minutes abandons such a node mid-probe // and reports a fetch failure for a node operating inside its published -// contract. -func nodeCapabilityFetcher(jwtSecret string, budget func() time.Duration) nodepool.CapabilityFetcher { +// contract. budget takes the node because the device set is a per-node +// property: an hw_device override is exactly a node that probes a different +// matrix from the one the cluster setting describes. +func nodeCapabilityFetcher(jwtSecret string, budget func(*nodepool.Node) time.Duration) nodepool.CapabilityFetcher { client := &http.Client{} - return func(ctx context.Context, nodeURL string) ([]byte, string, error) { - ctx, cancel := context.WithTimeout(ctx, budget()) + return func(ctx context.Context, node *nodepool.Node) ([]byte, string, error) { + if node == nil { + return nil, "", fmt.Errorf("node capability request has no node") + } + ctx, cancel := context.WithTimeout(ctx, budget(node)) defer cancel() - info, payload, status, err := transcodenode.FetchHWCapabilitiesPayload(ctx, client, nodeURL, jwtSecret) + info, payload, status, err := transcodenode.FetchHWCapabilitiesPayload(ctx, client, node.URL, jwtSecret) if err != nil { return nil, "", err } @@ -245,18 +250,32 @@ const libraryPathQueryTimeout = 2 * time.Second // first report land instead of timing out. const nodeCapabilityRequestTimeout = 2 * time.Minute -// nodeCapabilityProbeBudget reports how long to allow one capability fetch, -// read from the live configuration on every call so an operator adding a second -// render device does not have to restart the API for its fetches to stop being -// cut short. -func nodeCapabilityProbeBudget(live func() *config.Config) func() time.Duration { - return func() time.Duration { +// nodeCapabilityProbeBudget reports how long to allow one node's capability +// fetch. +// +// The cluster settings are read live on every call, so an operator adding a +// second render device does not have to restart the API for its fetches to stop +// being cut short. The node's own overrides win over them, because the worker +// builds its probe matrix from the policy it will actually run: a node that +// overrides hw_device with two devices needs the two-device budget even on a +// cluster configured with one, and it is precisely that node whose inventory +// would otherwise fail to refresh every sweep. +func nodeCapabilityProbeBudget(live func() *config.Config) func(*nodepool.Node) time.Duration { + return func(node *nodepool.Node) time.Duration { hwAccel, hwDevice := "", "" if live != nil { if current := live(); current != nil { hwAccel, hwDevice = current.Playback.HWAccel, current.Playback.HWDevice } } + if node != nil { + if node.HWAccelOverride != nil && *node.HWAccelOverride != "" { + hwAccel = *node.HWAccelOverride + } + if node.HWDeviceOverride != nil && *node.HWDeviceOverride != "" { + hwDevice = *node.HWDeviceOverride + } + } return max(nodeCapabilityRequestTimeout, tonemap.ProbeRequestTimeout(hwAccel, hwDevice)) } } diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index a792d38bc..8e037b347 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -399,7 +399,8 @@ func TestNodeCapabilityFetcherStoresTheNodesOwnBytes(t *testing.T) { })) t.Cleanup(server.Close) - payload, hash, err := nodeCapabilityFetcher("secret", nodeCapabilityProbeBudget(nil))(context.Background(), server.URL) + payload, hash, err := nodeCapabilityFetcher("secret", nodeCapabilityProbeBudget(nil))( + context.Background(), &nodepool.Node{ID: 1, URL: server.URL}) if err != nil { t.Fatalf("nodeCapabilityFetcher: %v", err) } @@ -430,8 +431,9 @@ func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { pair := &config.Config{} pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" - oneDevice := nodeCapabilityProbeBudget(func() *config.Config { return single })() - twoDevices := nodeCapabilityProbeBudget(func() *config.Config { return pair })() + clusterNode := &nodepool.Node{ID: 1, URL: "http://gpu-1"} + oneDevice := nodeCapabilityProbeBudget(func() *config.Config { return single })(clusterNode) + twoDevices := nodeCapabilityProbeBudget(func() *config.Config { return pair })(clusterNode) if want := tonemap.ProbeRequestTimeout("qsv", pair.Playback.HWDevice); twoDevices != want { t.Fatalf("two-device budget = %v, want the node's own advertised %v", twoDevices, want) @@ -445,13 +447,29 @@ func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { twoDevices, oneDevice) } - // Nothing configured, or no live config at all, still gets a usable bound - // rather than zero. - if got := nodeCapabilityProbeBudget(nil)(); got < nodeCapabilityRequestTimeout { + // A node's own override wins over the cluster setting, because the worker + // probes the policy it will actually run. This is the case a cluster-wide + // read cannot see: one device configured centrally, two on this node. + twoDeviceOverride := pair.Playback.HWDevice + overridden := &nodepool.Node{ID: 1, URL: "http://gpu-1", HWDeviceOverride: &twoDeviceOverride} + if got := nodeCapabilityProbeBudget(func() *config.Config { return single })(overridden); got != twoDevices { + t.Fatalf("overridden node budget = %v, want the two-device %v its own policy needs", got, twoDevices) + } + + // An override set to the empty string means "inherit", not "no devices". + empty := "" + inheriting := &nodepool.Node{ID: 1, URL: "http://gpu-1", HWDeviceOverride: &empty, HWAccelOverride: &empty} + if got := nodeCapabilityProbeBudget(func() *config.Config { return pair })(inheriting); got != twoDevices { + t.Fatalf("inheriting node budget = %v, want the cluster's %v", got, twoDevices) + } + + // Nothing configured, no live config, or no node at all still gets a usable + // bound rather than zero. + if got := nodeCapabilityProbeBudget(nil)(clusterNode); got < nodeCapabilityRequestTimeout { t.Fatalf("budget with no configuration = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) } - if got := nodeCapabilityProbeBudget(func() *config.Config { return nil })(); got < nodeCapabilityRequestTimeout { - t.Fatalf("budget with a nil config = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) + if got := nodeCapabilityProbeBudget(func() *config.Config { return nil })(nil); got < nodeCapabilityRequestTimeout { + t.Fatalf("budget with a nil config and nil node = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) } } @@ -460,7 +478,7 @@ func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { func TestCapabilityFetchBackstopExceedsTheAdvertisedBudget(t *testing.T) { pair := &config.Config{} pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" - budget := nodeCapabilityProbeBudget(func() *config.Config { return pair })() + budget := nodeCapabilityProbeBudget(func() *config.Config { return pair })(&nodepool.Node{ID: 1, URL: "http://gpu-1"}) if nodepool.CapabilityRefreshTimeout <= budget { t.Fatalf("health sweep backstop %v does not exceed the %v a two-device node advertises", diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index b48ba935a..3ccbaefa8 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -83,15 +83,48 @@ func (s *Sampler) cgroupCPU(now time.Time) (cgroupCPUSample, float64) { if err != nil { continue } - quota, err := readCgroupCPUQuota(paths) - if err != nil { - quota = 0 - } - return cgroupCPUSample{usageNS: usage, at: now, valid: true}, quota + return cgroupCPUSample{usageNS: usage, at: now, valid: true}, effectiveCgroupCPUQuota(paths) } return cgroupCPUSample{}, 0 } +// effectiveCgroupCPUQuota returns the tightest CPU budget in force on this +// cgroup, in cores, or 0 when nothing above it imposes one. +// +// Usage and quota are read from different places on purpose. Usage has to be +// this process's own — an ancestor's counts every sibling service on the +// machine — while the quota is whatever the kernel will actually throttle +// against, which is the smallest limit anywhere between here and the mount +// root. A systemd unit inside a slice with CPUQuota=, or a container under a +// limited pod cgroup, reads "max" at its leaf and is nonetheless capped; taking +// the leaf's answer would normalize a two-core service against sixty-four. +// +// The pair moves together at each level: a quota from one cgroup divided by a +// period from another describes no real budget. +func effectiveCgroupCPUQuota(paths cgroupCPUPath) float64 { + quotas := cgroupAncestorPaths(paths.quota) + periods := cgroupAncestorPaths(paths.period) + tightest := 0.0 + for i, quota := range quotas { + level := paths + level.quota = quota + if paths.period != "" { + if i >= len(periods) { + break + } + level.period = periods[i] + } + cores, err := readCgroupCPUQuota(level) + if err != nil || cores <= 0 { + continue + } + if tightest == 0 || cores < tightest { + tightest = cores + } + } + return tightest +} + // readCgroupCPUUsage returns cumulative cgroup CPU time in nanoseconds. func readCgroupCPUUsage(paths cgroupCPUPath) (int64, error) { var value int64 diff --git a/internal/nodemetrics/cgrouppath.go b/internal/nodemetrics/cgrouppath.go index 3a9f0ab34..41359a76c 100644 --- a/internal/nodemetrics/cgrouppath.go +++ b/internal/nodemetrics/cgrouppath.go @@ -101,15 +101,65 @@ func cgroupSelfFile(relative map[string]string, file string) string { return path.Join(cgroupMountRoot, controller, own, name) } -// withCgroupSelfPaths returns files preceded by their this-process equivalents, -// so a read tries the process's own cgroup before falling back to the root. +// cgroupAncestorPaths returns file and the same file name at every cgroup above +// it, nearest first, ending at the hierarchy mount root. +// +// A limit is not always written where the process sits. A systemd unit can +// inherit its quota from the slice that contains it, and a container can inherit +// one from its pod cgroup; in both cases the leaf reads "max" while the kernel +// throttles against an ancestor. Reading only the leaf would report the host's +// whole capacity for a process that has far less. +// +// The walk is by path, so every level it produces is a genuine ancestor of this +// process. Levels that hold no such file simply fail to read, which is how the +// v1 layouts skip the unified root they never had. +func cgroupAncestorPaths(file string) []string { + if file == "" { + return nil + } + // The file itself is always a level. Only the walk above it needs the file + // to live under the cgroup mount — a test harness pointing these at a temp + // directory has no hierarchy to climb, and must still read what it was given. + if !strings.HasPrefix(file, cgroupMountRoot+"/") { + return []string{file} + } + name := path.Base(file) + out := []string{file} + for dir := path.Dir(file); strings.HasPrefix(dir, cgroupMountRoot); dir = path.Dir(dir) { + if candidate := path.Join(dir, name); candidate != file { + out = append(out, candidate) + } + if dir == cgroupMountRoot { + break + } + } + return out +} + +// withCgroupSelfPaths returns files preceded by their this-process equivalents +// and every cgroup between the two, so a read sees each limit in force on this +// process rather than only the nearest and the root. +// +// The caller picks the tightest of what it can read, not the first: an ancestor +// with a finite limit binds a leaf that says "max", so stopping at the first +// readable file would report no limit for a process that has one. func withCgroupSelfPaths(relative map[string]string, files []string) []string { out := make([]string, 0, len(files)*2) + seen := make(map[string]bool, len(files)*2) + add := func(candidate string) { + if candidate == "" || seen[candidate] { + return + } + seen[candidate] = true + out = append(out, candidate) + } for _, file := range files { if own := cgroupSelfFile(relative, file); own != "" { - out = append(out, own) + for _, candidate := range cgroupAncestorPaths(own) { + add(candidate) + } } - out = append(out, file) + add(file) } return out } diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go index b5252ec9c..2610b06ce 100644 --- a/internal/nodemetrics/cgrouppath_test.go +++ b/internal/nodemetrics/cgrouppath_test.go @@ -135,11 +135,17 @@ func TestCgroupSelfFile(t *testing.T) { func TestWithCgroupSelfPathsKeepsTheRootFallback(t *testing.T) { relative := map[string]string{"": "system.slice/silo.service", "memory": "system.slice/silo.service"} got := withCgroupSelfPaths(relative, CgroupMemoryLimitPaths()) + // Every cgroup between this process and the root, then the root path the + // list started with. The intermediate levels are the point: a leaf that says + // "max" inside a slice that does not is exactly the case being covered. want := []string{ "/sys/fs/cgroup/system.slice/silo.service/memory.max", + "/sys/fs/cgroup/system.slice/memory.max", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/system.slice/silo.service/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/system.slice/memory.limit_in_bytes", "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory.limit_in_bytes", } if !slices.Equal(got, want) { t.Fatalf("withCgroupSelfPaths() =\n%v\nwant\n%v", got, want) @@ -203,3 +209,94 @@ func TestWithCgroupSelfUsagePathsRewritesEveryFileOrNone(t *testing.T) { t.Fatalf("inactiveFile = %q, want the layout's own key preserved", rewritten.inactiveFile) } } + +// A limit is not always written where the process sits: a systemd unit inherits +// its quota from the slice containing it, and a container from its pod cgroup. +// The leaf reads "max" and the kernel throttles anyway, so a walk that stops at +// the leaf reports the whole host to a process that has two cores. +func TestCgroupAncestorPaths(t *testing.T) { + got := cgroupAncestorPaths("/sys/fs/cgroup/kubepods/burstable/podabc/container1/cpu.max") + want := []string{ + "/sys/fs/cgroup/kubepods/burstable/podabc/container1/cpu.max", + "/sys/fs/cgroup/kubepods/burstable/podabc/cpu.max", + "/sys/fs/cgroup/kubepods/burstable/cpu.max", + "/sys/fs/cgroup/kubepods/cpu.max", + cgroupCPUPaths[0].quota, + } + if !slices.Equal(got, want) { + t.Fatalf("cgroupAncestorPaths() =\n%v\nwant\n%v", got, want) + } + + // The mount root itself has nowhere to climb to. + if got := cgroupAncestorPaths(cgroupCPUPaths[0].quota); !slices.Equal(got, []string{cgroupCPUPaths[0].quota}) { + t.Fatalf("cgroupAncestorPaths(root) = %v, want just the file", got) + } + // A path outside the hierarchy still reads itself: a test harness pointing + // these at a temp directory has no ancestors, and must not lose its file. + if got := cgroupAncestorPaths("/tmp/fake/cpu.max"); !slices.Equal(got, []string{"/tmp/fake/cpu.max"}) { + t.Fatalf("cgroupAncestorPaths(outside) = %v, want just the file", got) + } + if got := cgroupAncestorPaths(""); got != nil { + t.Fatalf("cgroupAncestorPaths(\"\") = %v, want none", got) + } +} + +// The quota a process is throttled against is the tightest anywhere above it, +// and it has to be paired with the period from the same cgroup — a quota from +// one level over a period from another describes no real budget. +func TestEffectiveCgroupCPUQuotaTakesTheTightestAncestor(t *testing.T) { + root := t.TempDir() + leaf := filepath.Join(root, "system.slice", "silo.service") + slice := filepath.Join(root, "system.slice") + for _, dir := range []string{leaf, slice} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + + // cgroupAncestorPaths only climbs inside the real cgroup mount, so the walk + // is exercised here by handing effectiveCgroupCPUQuota each level directly. + quotaAt := func(dir string) float64 { + return effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(dir, "cpu.max")}) + } + + // The service says "max" while its slice allows two cores. + write(leaf, "cpu.max", "max 100000\n") + write(slice, "cpu.max", "200000 100000\n") + if got := quotaAt(leaf); got != 0 { + t.Fatalf("leaf alone = %v cores, want 0 — it imposes none", got) + } + if got := quotaAt(slice); got != 2 { + t.Fatalf("slice = %v cores, want 2", got) + } + + // A leaf tighter than its slice wins, and a looser one loses. + write(leaf, "cpu.max", "100000 100000\n") + if got := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(leaf, "cpu.max")}); got != 1 { + t.Fatalf("tighter leaf = %v cores, want 1", got) + } +} + +// v1 keeps quota and period in separate files, so both have to move together as +// the walk climbs. +func TestEffectiveCgroupCPUQuotaPairsQuotaWithItsOwnPeriod(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "cpu.cfs_quota_us"), []byte("400000\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cpu.cfs_period_us"), []byte("100000\n"), 0o644); err != nil { + t.Fatal(err) + } + got := effectiveCgroupCPUQuota(cgroupCPUPath{ + quota: filepath.Join(dir, "cpu.cfs_quota_us"), + period: filepath.Join(dir, "cpu.cfs_period_us"), + }) + if got != 4 { + t.Fatalf("v1 quota = %v cores, want 4", got) + } +} diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 2e80b8176..42552ec55 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -804,3 +804,61 @@ func TestSampleGPUDoesNotDuplicateAnAlreadyNamedDevice(t *testing.T) { t.Fatalf("sessions = %d, want the workload joined onto the enriched entry", gpus[0].Sessions) } } + +// A leaf cgroup that imposes no memory limit does not mean the process has +// none: a systemd unit inside a slice with MemoryMax=, or a container under a +// limited pod cgroup, is capped by an ancestor while its own memory.max reads +// "max". Taking the first readable limit would report the host's RAM to a +// process the kernel will OOM-kill at 2 GiB. +func TestMemoryLimitTakesTheTightestCgroupInForce(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + tree.write("net/dev", "") + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + // Leaf first, as the resolved list is ordered. The leaf allows 8 GiB, but the + // slice containing it allows only 2 — so the first readable limit is not the + // one the kernel kills against, which is what makes this more than an + // ordering detail. + leaf := write("leaf.max", "8589934592\n") + slice := write("slice.max", "2147483648\n") + root := write("root.max", "68719476736\n") + + usage := write("memory.current", "1073741824\n") + stat := write("memory.stat", "inactive_file 0\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupLimitPaths = []string{leaf, slice, root} + s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}} + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != 2048 { + t.Fatalf("MemTotalMB = %d, want the 2048 the slice allows, not the leaf's looser 8192", system.MemTotalMB) + } + // Total came from a cgroup, so used has to come from one too. + if system.MemUsedMB != 1024 { + t.Fatalf("MemUsedMB = %d, want the cgroup's own 1024", system.MemUsedMB) + } + + // The other shape of the same bug: a leaf that imposes nothing at all while + // an ancestor does. + unlimited := write("unlimited.max", "max\n") + s = newTestSampler(t, tree, clock, Options{}) + s.cgroupLimitPaths = []string{unlimited, slice, root} + s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}} + s.sample(context.Background()) + if got := s.Snapshot().System.MemTotalMB; got != 2048 { + t.Fatalf("MemTotalMB = %d with an unlimited leaf, want the slice's 2048", got) + } +} diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go index 79c865ef9..35f0a30fe 100644 --- a/internal/nodemetrics/system.go +++ b/internal/nodemetrics/system.go @@ -205,6 +205,10 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { } } + // Every limit in force is read, not just the first: the list runs from this + // process's own cgroup up through its ancestors, and a leaf that says "max" + // can still sit inside a slice or pod cgroup that does not. The kernel + // OOM-kills against the tightest of them, so that is the one to report. capped := false for _, path := range s.cgroupLimitPaths { limit, err := ReadCgroupMemoryLimit(path) @@ -218,7 +222,6 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { totalBytes = limit capped = true } - break } // Only when the total above is the cgroup's. A container with no memory diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 59b1c1ef8..2bf5c5498 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -139,7 +139,13 @@ func trimJSONNull(raw json.RawMessage) json.RawMessage { // package never has to understand — or import — the playback capability model. // An empty hash means the payload cannot be tracked for change and must not be // persisted. -type CapabilityFetcher func(ctx context.Context, nodeURL string) (payload []byte, hash string, err error) +// The whole node is passed rather than its URL because the answer's cost is a +// property of the node: a worker builds its probe matrix from its *effective* +// acceleration policy, so one carrying an hw_device override with two devices +// legitimately takes longer to answer than the cluster-wide setting predicts. A +// fetcher that sizes its request from the cluster setting alone cancels such a +// node mid-probe every sweep, and its inventory never lands. +type CapabilityFetcher func(ctx context.Context, node *Node) (payload []byte, hash string, err error) // capabilityFetchTimeout is the backstop on one capability fetch, not its // budget. @@ -391,7 +397,7 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply } fetchCtx, cancel := context.WithTimeout(ctx, capabilityFetchTimeout) defer cancel() - payload, hash, err := fetch(fetchCtx, n.URL) + payload, hash, err := fetch(fetchCtx, n) if err != nil { slog.WarnContext(ctx, "node capability fetch failed", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL, "error", err) diff --git a/internal/nodepool/health_test.go b/internal/nodepool/health_test.go index 53b50cdb8..03214e5df 100644 --- a/internal/nodepool/health_test.go +++ b/internal/nodepool/health_test.go @@ -24,10 +24,10 @@ type fakeCapabilityFetcher struct { err error } -func (f *fakeCapabilityFetcher) fetch(_ context.Context, nodeURL string) ([]byte, string, error) { +func (f *fakeCapabilityFetcher) fetch(_ context.Context, node *Node) ([]byte, string, error) { f.mu.Lock() defer f.mu.Unlock() - f.calls = append(f.calls, nodeURL) + f.calls = append(f.calls, node.URL) return f.payload, f.hash, f.err } @@ -418,7 +418,7 @@ func newBlockingFetcher() *blockingFetcher { return &blockingFetcher{started: make(chan struct{}, 1), release: make(chan struct{})} } -func (f *blockingFetcher) fetch(ctx context.Context, _ string) ([]byte, string, error) { +func (f *blockingFetcher) fetch(ctx context.Context, _ *Node) ([]byte, string, error) { f.calls.Add(1) select { case f.started <- struct{}{}: From 8c70b6dbe4dbfe4220e1cecdce62ed518fa235f9 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:43:18 -0400 Subject: [PATCH 045/163] fix(nodes): count tone-map probes, derive the fetch backstop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-probe gate counted detached hardware probes but not detached tone-map ones, which have the identical shape: probeCached hands its work to singleflight.DoChan on a background context, so the flight outlives buildCapabilitySnapshot's gate claim, and the matrix behind it is real FFmpeg smoke encodes. An operator re-probing at that moment invalidated the tone-map cache and started a second matrix beside the first. tonemap now exports ProbesInFlight on the same terms as playback — claimed on the calling goroutine before dispatch, handed to a watcher when the caller gives up — and beginReprobe adds both sources. The sweep's five-minute backstop was a guess, and the thing it bounds has no ceiling: nine render devices legitimately ask for 311 seconds, so the backstop cancelled a node inside its published contract and its inventory never populated. It is now derived from the budget the fetcher gave itself, plus a minute, so the fetcher's own deadline always fires first and the constant is only the floor that applies when no budget is wired. One number, one owner. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 7 ++- internal/nodepool/health.go | 57 +++++++++++++++++++----- internal/nodepool/health_test.go | 33 ++++++++++++++ internal/tonemap/probe.go | 44 +++++++++++++++++++ internal/tonemap/probe_test.go | 61 ++++++++++++++++++++++++++ internal/transcodenode/gpugate_test.go | 15 +++++++ internal/transcodenode/reprobe.go | 2 +- 7 files changed, 205 insertions(+), 14 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 93afe8499..bc09167f0 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1251,8 +1251,11 @@ func main() { deps.NodePlanner = nodepool.NewPlanner(proxyPool, transcodePool) healthChecker := nodepool.NewHealthChecker(proxyPool, transcodePool, nodeRepo) - healthChecker.SetCapabilityFetcher( - nodeCapabilityFetcher(cfg.Auth.JWTSecret, nodeCapabilityProbeBudget(configWatcher.Config))) + capabilityBudget := nodeCapabilityProbeBudget(configWatcher.Config) + healthChecker.SetCapabilityFetcher(nodeCapabilityFetcher(cfg.Auth.JWTSecret, capabilityBudget)) + // The sweep's backstop is derived from the same budget, so it can never + // be the deadline that fires first on a node with many devices. + healthChecker.SetCapabilityFetchBudget(capabilityBudget) deps.NodeHealthChecker = healthChecker healthChecker.Start(appCtx) slog.Info("node pools initialized", "proxy_nodes", len(proxyNodes), "transcode_nodes", len(transcodeNodes)) diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 2bf5c5498..ef3b3d2ac 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -147,20 +147,27 @@ func trimJSONNull(raw json.RawMessage) json.RawMessage { // node mid-probe every sweep, and its inventory never lands. type CapabilityFetcher func(ctx context.Context, node *Node) (payload []byte, hash string, err error) -// capabilityFetchTimeout is the backstop on one capability fetch, not its -// budget. +// capabilityFetchTimeout is the floor under the backstop on one capability +// fetch. It is not the budget. // // The budget belongs to the fetcher, which knows the configured hardware and -// sizes each request against the node's own advertised probe matrix; that grows -// with the device count and passes two minutes at two devices, so a fixed bound -// here would cut short a node operating well inside its published contract. What -// this stops is the other failure: a fetcher that never returns pinning a -// goroutine and a node's inventory forever. It is deliberately far above any -// budget a real configuration produces — five minutes covers a node probing -// roughly seven devices — because a backstop that trips during ordinary -// operation is indistinguishable from the bug it was meant to catch. +// sizes each request against the node's own advertised probe matrix. That matrix +// grows with the device count without bound — nine render devices legitimately +// ask for over five minutes — so no fixed number here can be both a real +// backstop and safe against cutting a node short. What this stops is the other +// failure: a fetcher that never returns, pinning a goroutine and a node's +// inventory forever. +// +// So the backstop is derived from the fetcher's own budget and this floor is +// only what applies when no budget is wired. A backstop that trips during +// ordinary operation is indistinguishable from the bug it was meant to catch. const capabilityFetchTimeout = 5 * time.Minute +// capabilityFetchSlack is how far the sweep's backstop sits above the budget the +// fetcher gave itself, so the fetcher's own deadline is always the one that +// fires first and the failure an operator sees names the probe rather than this. +const capabilityFetchSlack = time.Minute + // CapabilityRefreshTimeout is the bound RefreshNodeCapabilities puts on the // fetch it performs. It is exported for the one caller that has to hold an HTTP // connection open across that fetch and must therefore size its own write @@ -181,6 +188,7 @@ type HealthChecker struct { // be running. mu sync.RWMutex capFetch CapabilityFetcher + capFetchBudget func(*Node) time.Duration onCapabilitiesChanged func(nodeURL string) // capabilityRefreshes tracks the detached capability fetches so shutdown — @@ -202,6 +210,33 @@ func NewHealthChecker(proxyPool *ProxyPool, transcodePool *TranscodePool, repo * } } +// SetCapabilityFetchBudget wires how long the fetcher will allow itself for one +// node, so the sweep's backstop can be derived from it rather than guessed. +// +// Without it the backstop falls back to capabilityFetchTimeout, which is right +// for the common configuration and too tight for a node with many devices — +// hence this, supplied by the same wiring that supplies the fetcher. +func (hc *HealthChecker) SetCapabilityFetchBudget(budget func(*Node) time.Duration) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.capFetchBudget = budget +} + +// capabilityFetchBackstop bounds one fetch above whatever the fetcher allowed +// itself, so the fetcher's deadline always fires first. +func (hc *HealthChecker) capabilityFetchBackstop(n *Node) time.Duration { + hc.mu.RLock() + budget := hc.capFetchBudget + hc.mu.RUnlock() + if budget == nil { + return capabilityFetchTimeout + } + return max(capabilityFetchTimeout, budget(n)+capabilityFetchSlack) +} + // SetCapabilityFetcher wires how the sweep retrieves a node's capability report // once the node reports a hash it has not stored. Without one the sweep behaves // exactly as it did before capability tracking. @@ -395,7 +430,7 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply if fetch == nil { return nil } - fetchCtx, cancel := context.WithTimeout(ctx, capabilityFetchTimeout) + fetchCtx, cancel := context.WithTimeout(ctx, hc.capabilityFetchBackstop(n)) defer cancel() payload, hash, err := fetch(fetchCtx, n) if err != nil { diff --git a/internal/nodepool/health_test.go b/internal/nodepool/health_test.go index 03214e5df..f8ac903df 100644 --- a/internal/nodepool/health_test.go +++ b/internal/nodepool/health_test.go @@ -506,3 +506,36 @@ func TestHealthCheckerDoesNotStackCapabilityFetchesForOneNode(t *testing.T) { t.Fatalf("capability fetches = %d after the report was stored, want 1", got) } } + +// The probe matrix a cold node runs grows with its device count without bound — +// nine render devices legitimately ask for over five minutes — so a fixed outer +// deadline cancels a node operating inside its published contract, and its +// inventory never populates. The backstop is derived from the budget the fetcher +// gave itself so the fetcher's own deadline always fires first. +func TestCapabilityFetchBackstopSitsAboveTheFetcherBudget(t *testing.T) { + checker := NewHealthChecker(NewProxyPool(), NewTranscodePool(), nil) + node := &Node{ID: 1, URL: "http://gpu-1"} + + if got := checker.capabilityFetchBackstop(node); got != capabilityFetchTimeout { + t.Fatalf("backstop with no budget wired = %v, want the %v floor", got, capabilityFetchTimeout) + } + + // A budget under the floor leaves the floor standing: it already clears the + // fetcher by a wide margin. + checker.SetCapabilityFetchBudget(func(*Node) time.Duration { return 2 * time.Minute }) + if got := checker.capabilityFetchBackstop(node); got != capabilityFetchTimeout { + t.Fatalf("backstop for a small budget = %v, want the %v floor", got, capabilityFetchTimeout) + } + + // A budget above it carries the backstop with it, always by the slack, so + // the fetch is never cut short by this. + big := 311 * time.Second + checker.SetCapabilityFetchBudget(func(*Node) time.Duration { return big }) + got := checker.capabilityFetchBackstop(node) + if got != big+capabilityFetchSlack { + t.Fatalf("backstop for a %v budget = %v, want %v", big, got, big+capabilityFetchSlack) + } + if got <= big { + t.Fatalf("backstop %v does not clear the %v the fetcher allowed itself", got, big) + } +} diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index fef236f6f..c6013726a 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -79,6 +80,13 @@ func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevic } probeCache.Unlock() + // Claimed here, on the calling goroutine, not inside the function below. + // DoChan schedules that function and returns without waiting for it, so a + // caller whose context is already done returns while the probe has not + // reached its first line — and this probe runs real smoke encodes on the + // GPU. Anything that needs the encoder to itself has to see them; see + // ProbesInFlight. + probesInFlight.Add(1) resultCh := probeCache.group.DoChan(key, func() (any, error) { probeCache.Lock() cached, ok := probeCache.entries[key] @@ -103,8 +111,16 @@ func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevic }) select { case <-ctx.Done(): + // The flight outlives this caller by design — its probe context is + // rooted at Background so a canceled request cannot kill work another + // request is waiting on — so the claim goes with the flight, not with us. + go func() { + <-resultCh + probesInFlight.Add(-1) + }() return nil, ctx.Err() case result := <-resultCh: + probesInFlight.Add(-1) if result.Err != nil { return nil, result.Err } @@ -189,6 +205,34 @@ func InvalidateProbeCache() { probeCache.entries = make(map[string]probeCacheEntry) } +// probesInFlight counts tone-map probes this process has claimed the encoder +// for, including ones whose caller has already given up on them. +var probesInFlight atomic.Int64 + +// ProbesInFlight reports how many tone-map probes this process has claimed the +// encoder for. +// +// The matrix behind one of these is real FFmpeg smoke encodes, and a probe +// outlives its caller by design: the singleflight task runs on a background +// context so a canceled request cannot kill work another request is waiting on. +// A component that released its own claim on the GPU when its call returned — +// the transcode node's capability build does exactly that — can therefore leave +// encodes running with nothing accounting for them. Anything that needs the +// encoder exclusively must add this to whatever else it counts as busy, or it +// will start a second matrix beside the first and publish the collision as a +// capability failure. +// +// It counts claims rather than processes and errs high, for the same reason its +// hardware-probe counterpart does: an overcount costs a retry, an undercount +// costs a false verdict. +func ProbesInFlight() int { + count := probesInFlight.Load() + if count < 0 { + return 0 + } + return int(count) +} + // probeCacheEntryCurrent reports whether a complete result or unexpired // incomplete result may be reused. func probeCacheEntryCurrent(entry probeCacheEntry, now time.Time) bool { diff --git a/internal/tonemap/probe_test.go b/internal/tonemap/probe_test.go index 2cbc37a3b..3a48e5615 100644 --- a/internal/tonemap/probe_test.go +++ b/internal/tonemap/probe_test.go @@ -5,7 +5,9 @@ import ( "errors" "os" "path/filepath" + "runtime" "strings" + "sync" "sync/atomic" "testing" "time" @@ -409,3 +411,62 @@ func resetProbeCache(t *testing.T) { t.Helper() InvalidateProbeCache() } + +// A tone-map probe outlives its caller by design, so a component that released +// its own claim on the GPU when its call returned can leave smoke encodes +// running with nothing accounting for them. The count is what lets the transcode +// node's re-probe gate see that. +func TestProbesInFlightCountsADetachedProbe(t *testing.T) { + awaitNoProbesInFlight(t) + + started := make(chan struct{}) + release := make(chan struct{}) + // The probe runs several commands; only the first needs to announce itself. + var announce, released sync.Once + t.Cleanup(func() { released.Do(func() { close(release) }) }) + + // The probe has to be running before the caller gives up, or the flight + // finishes on the canceled context and there is nothing detached to count. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := probeCached(ctx, "ffmpeg", BackendQSV, "/dev/dri/renderD128", + func(context.Context, string, ...string) ([]byte, error) { + announce.Do(func() { close(started) }) + <-release + return nil, errors.New("probe abandoned") + }, time.Now) + done <- err + }() + + <-started + cancel() + if err := <-done; err == nil { + t.Fatal("an abandoned probe reported success") + } + + // Checked the instant the caller returned, which is when its own claim on + // the encoder goes away while the smoke encode keeps running. + if got := ProbesInFlight(); got < 1 { + t.Fatalf("ProbesInFlight() = %d the moment the caller returned, want at least 1", got) + } + released.Do(func() { close(release) }) + awaitNoProbesInFlight(t) +} + +// awaitNoProbesInFlight waits for every claim on the encoder to be released, +// including ones detached from a caller that has already returned. Waiting on +// the counter rather than on a delay keeps this independent of machine load. +func awaitNoProbesInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if got := ProbesInFlight(); got == 0 { + return + } else if time.Now().After(deadline) { + t.Fatalf("ProbesInFlight() = %d, want every detached probe released", got) + } + runtime.Gosched() + } +} diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go index 16e3976b1..cb0198b1e 100644 --- a/internal/transcodenode/gpugate_test.go +++ b/internal/transcodenode/gpugate_test.go @@ -125,3 +125,18 @@ func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { t.Fatal("re-probe refused once no probe was in flight") } } + +// Tone-map probes detach from their caller exactly as hardware probes do, and +// they run their own FFmpeg smoke encodes, so a re-probe that counted only the +// hardware ones would start a second matrix beside a running first. +func TestGPUGateCountsEveryDetachedProbeSource(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(0, 2) + if ok { + t.Fatal("re-probe admitted while detached smoke encodes were still running") + } + if busy != 2 { + t.Fatalf("busy = %d, want both detached probe sources counted", busy) + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index 4f324581d..61700fe63 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -60,7 +60,7 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques // collide with the smoke encode minutes into the probe. The gate is the // same exclusion the transcode-start path consults, so from here until the // deferred release no new GPU work is admitted. - busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load()), playback.HWProbesInFlight()) + busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load()), playback.HWProbesInFlight()+tonemap.ProbesInFlight()) if !ok { slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", "component", "transcodenode", "active_jobs", busy) From ee9cb3b75cc7ba1aa0b2ef1861199b2b864a3a0f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:00:38 -0400 Subject: [PATCH 046/163] fix(nodes): six review findings across metrics, gating, and capability writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Memory usage was read from this process's leaf cgroup while the limit came from wherever the tightest one was. That pairing is wrong whenever the binding limit is an ancestor's — a pod cgroup shared with sidecars, a slice shared with other services — because what fills that limit is everything charged to it, not just Silo. The dashboard showed headroom that did not exist right up until the parent OOMed. The limit, usage, stat and page-cache key now travel as one level, and usage is read from whichever level's limit binds. This corrects reasoning I got wrong one commit ago, where I argued usage should deliberately not follow the walk. Encoder warmup ran a real smoke encode on a goroutine with no gate claim, while the listener was already open — so a re-probe in a node's first seconds saw an idle gate. It holds the gate now, held rather than requested, since warmup is already running and refusing it would only skip it. A proxy's re-probe invalidated the hardware and tone-map caches but never returned its retired metric sources, so a repaired driver stayed invisible for the breaker's ten-minute interval. It calls RetrySources like the transcode node does. Capability writes were fenced on node id and URL only. Every API replica sweeps independently, so a slower fetch of an older report could land after a newer one and take the durable GPU identities and drift state back with it. The write is now a compare-and-set against the hash the caller read before fetching — no clock comparison, and the loser simply discards a report that no longer describes the row it came from. ProbeRequestTimeoutMillis is now hashed. It was grouped with Source and NodeURL as "describes the report, not the host", but unlike those it does not vary by caller: it is the node's own statement of how long its answer takes, and the control plane sizes real deadlines from the stored copy. A node upgraded to a build needing longer changed nothing else, so the sweep never refetched and the API kept canceling its re-probes against a budget it had outgrown. A cpuset caps CPU without setting a quota, leaving cpu.max saying "max". A process pinned to two CPUs on a sixty-four core host divided its own busy time by sixty-four and reported three percent while saturated. The effective cpuset is read and the tighter of the two caps wins. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/cgroupcpu.go | 64 +++++++++ internal/nodemetrics/meminfo.go | 21 ++- internal/nodemetrics/sampler.go | 29 +++-- internal/nodemetrics/sampler_test.go | 121 ++++++++++++++++-- internal/nodemetrics/system.go | 42 +++--- internal/nodepool/health.go | 12 +- internal/nodepool/repository.go | 21 ++- .../nodepool/repository_capabilities_test.go | 61 +++++++-- internal/playback/capabilityhash.go | 21 ++- internal/playback/capabilityhash_test.go | 23 +++- internal/proxy/reprobe.go | 6 + internal/transcodenode/reprobe_test.go | 30 +++++ internal/transcodenode/server.go | 7 + 13 files changed, 381 insertions(+), 77 deletions(-) diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index 3ccbaefa8..9f1cdac41 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -66,6 +66,70 @@ var cgroupCPUPaths = []cgroupCPUPath{ }, } +// cgroupCPUSetPaths lists where a cgroup publishes the CPUs it may run on, v2 +// first. The "effective" file is what the kernel actually allows after +// intersecting with every ancestor, which is the number a process is really +// bounded by; the plain file is the request, and is read only where the +// effective one is absent. +var cgroupCPUSetPaths = []string{ + "/sys/fs/cgroup/cpuset.cpus.effective", + "/sys/fs/cgroup/cpuset/cpuset.effective_cpus", + "/sys/fs/cgroup/cpuset.cpus", + "/sys/fs/cgroup/cpuset/cpuset.cpus", +} + +// cgroupCPUSetCores counts the CPUs this cgroup may run on, or 0 when it is not +// restricted to a subset. +// +// A cpuset is the other way a deployment caps CPU, and unlike a CFS quota it +// leaves cpu.max saying "max". Without reading it, a process pinned to two CPUs +// on a sixty-four core host divides its own busy time by sixty-four and reports +// three percent while it is saturated — which defeats the whole point of the +// correction. +// +// The effective file already accounts for ancestors, so unlike the quota this +// needs no walk of its own; where only the pre-intersection file exists, the +// per-level candidates cover the same ground. +func cgroupCPUSetCores(paths []string) int { + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + continue + } + if count := countCPUSetEntries(string(raw)); count > 0 { + return count + } + } + return 0 +} + +// countCPUSetEntries counts the CPUs in a Linux cpu list ("0-3,8,12-13"). +// An unparseable or empty list counts nothing rather than guessing. +func countCPUSetEntries(list string) int { + total := 0 + for _, part := range strings.Split(strings.TrimSpace(list), ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + low, high, ranged := strings.Cut(part, "-") + first, err := strconv.Atoi(strings.TrimSpace(low)) + if err != nil || first < 0 { + continue + } + if !ranged { + total++ + continue + } + last, err := strconv.Atoi(strings.TrimSpace(high)) + if err != nil || last < first { + continue + } + total += last - first + 1 + } + return total +} + // cgroupCPUSample is one cumulative CPU-time reading. Only differences between // two readings mean anything. type cgroupCPUSample struct { diff --git a/internal/nodemetrics/meminfo.go b/internal/nodemetrics/meminfo.go index efb511a2b..581bef4a1 100644 --- a/internal/nodemetrics/meminfo.go +++ b/internal/nodemetrics/meminfo.go @@ -28,15 +28,24 @@ const ( // It returns a fresh slice per call so a caller iterating it cannot reorder the // preference for everyone else. func CgroupMemoryLimitPaths() []string { - return []string{ - "/sys/fs/cgroup/memory.max", // cgroup v2 - "/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1 + paths := make([]string, 0, len(cgroupMemoryUsagePaths)) + for _, level := range cgroupMemoryUsagePaths { + paths = append(paths, level.limit) } + return paths } -// cgroupUsagePath pairs one cgroup version's current-usage file with the stat -// file and key that names its page cache. +// cgroupUsagePath pairs one cgroup level's limit with the current-usage file, +// the stat file, and the key that names its page cache. +// +// All four move together because a limit and the usage measured against it have +// to describe the same cgroup. When the binding limit comes from an ancestor — +// a pod cgroup shared with sidecars, a systemd slice shared with other services +// — the memory that fills it is everything charged to that ancestor, not just +// this process's leaf. Pairing the parent's capacity with the leaf's working +// set shows headroom that does not exist, right up until the parent OOMs. type cgroupUsagePath struct { + limit string usage string stat string inactiveFile string @@ -50,11 +59,13 @@ type cgroupUsagePath struct { // an OOM kill (and is what `docker stats` reports). var cgroupMemoryUsagePaths = []cgroupUsagePath{ { + limit: "/sys/fs/cgroup/memory.max", usage: "/sys/fs/cgroup/memory.current", stat: "/sys/fs/cgroup/memory.stat", inactiveFile: cgroupInactiveFileKeyV2, }, { + limit: "/sys/fs/cgroup/memory/memory.limit_in_bytes", usage: "/sys/fs/cgroup/memory/memory.usage_in_bytes", stat: "/sys/fs/cgroup/memory/memory.stat", inactiveFile: cgroupInactiveFileKeyV1, diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 178bc3463..5b93166e2 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -79,9 +79,10 @@ type Sampler struct { // bind-mounted when this sampler runs in Docker nested inside an LXC // container. See procDirFor for why it takes priority when present. hostProcDir string - cgroupLimitPaths []string cgroupUsagePaths []cgroupUsagePath - cgroupCPUPaths []cgroupCPUPath + // cgroupCPUSetPaths are the cpuset files consulted when no CFS quota binds. + cgroupCPUSetPaths []string + cgroupCPUPaths []cgroupCPUPath snapshot atomic.Pointer[Snapshot] @@ -156,14 +157,14 @@ func NewSampler(opts Options) *Sampler { // mount root, so a systemd unit with CPUQuota= or MemoryMax= is // measured against its limit rather than against the whole machine. // See cgrouppath.go; a container is unaffected either way. - cgroupLimitPaths: withCgroupSelfPaths(cgroupSelf, CgroupMemoryLimitPaths()), - cgroupUsagePaths: withCgroupSelfUsagePaths(cgroupSelf, cgroupMemoryUsagePaths), - cgroupCPUPaths: withCgroupSelfCPUPaths(cgroupSelf, cgroupCPUPaths), - prevGPU: map[fdinfoClient]engineCounters{}, - disks: map[string]*diskEntry{}, - statfs: osStatfs, - runNVIDIASMI: runNVIDIASMI, - nvidiaBreaker: &sourceBreaker{name: "nvidia-smi"}, + cgroupUsagePaths: withCgroupSelfUsagePaths(cgroupSelf, cgroupMemoryUsagePaths), + cgroupCPUPaths: withCgroupSelfCPUPaths(cgroupSelf, cgroupCPUPaths), + cgroupCPUSetPaths: withCgroupSelfPaths(cgroupSelf, cgroupCPUSetPaths), + prevGPU: map[fdinfoClient]engineCounters{}, + disks: map[string]*diskEntry{}, + statfs: osStatfs, + runNVIDIASMI: runNVIDIASMI, + nvidiaBreaker: &sourceBreaker{name: "nvidia-smi"}, } s.snapshot.Store(&Snapshot{}) return s @@ -282,6 +283,14 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { } sample, quota := s.cgroupCPU(now) + // A cpuset caps CPU without setting a quota, so a process pinned to two + // CPUs reads cpu.max = "max" and would otherwise be measured against the + // whole host. Whichever cap is tighter is the one it actually runs under. + if pinned := cgroupCPUSetCores(s.cgroupCPUSetPaths); pinned > 0 { + if quota <= 0 || float64(pinned) < quota { + quota = float64(pinned) + } + } if quota > 0 { cores = cgroupQuotaCores(quota, hostCores) } diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 42552ec55..432fbbe86 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -64,7 +64,6 @@ func newTestSampler(t *testing.T, tree *procTree, clock *fakeClock, opts Options // have a real /host/proc; tests that exercise the lxcfs override set this // explicitly to a tree that does. s.hostProcDir = filepath.Join(tree.root, "no-such-host-proc") - s.cgroupLimitPaths = nil s.cgroupUsagePaths = nil s.cgroupCPUPaths = nil s.statfs = func(string) (fsStats, error) { return fsStats{}, os.ErrNotExist } @@ -271,8 +270,8 @@ func TestMemoryCorrectedByCgroupLimitAndUsage(t *testing.T) { } s := newTestSampler(t, tree, clock, Options{}) - s.cgroupLimitPaths = []string{filepath.Join(cgroupDir, tc.limitFile)} s.cgroupUsagePaths = []cgroupUsagePath{{ + limit: filepath.Join(cgroupDir, tc.limitFile), usage: filepath.Join(cgroupDir, tc.usageFile), stat: filepath.Join(cgroupDir, tc.statFile), inactiveFile: tc.inactiveKey, @@ -704,8 +703,7 @@ func TestMemoryStatsDoesNotMixCgroupUsageWithHostTotal(t *testing.T) { if err := os.WriteFile(usage, []byte("1048576\n"), 0o600); err != nil { t.Fatalf("write cgroup usage: %v", err) } - s.cgroupLimitPaths = []string{filepath.Join(t.TempDir(), "absent")} - s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage}} + s.cgroupUsagePaths = []cgroupUsagePath{{limit: filepath.Join(t.TempDir(), "absent"), usage: usage}} used, total := s.memoryStats() if total != 65536*1024 { @@ -733,8 +731,7 @@ func TestMemoryStatsUsesCgroupUsageBesideACgroupLimit(t *testing.T) { if err := os.WriteFile(usage, []byte("1048576\n"), 0o600); err != nil { t.Fatalf("write cgroup usage: %v", err) } - s.cgroupLimitPaths = []string{limit} - s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage}} + s.cgroupUsagePaths = []cgroupUsagePath{{limit: limit, usage: usage}} used, total := s.memoryStats() if total != 8388608 { @@ -834,31 +831,127 @@ func TestMemoryLimitTakesTheTightestCgroupInForce(t *testing.T) { slice := write("slice.max", "2147483648\n") root := write("root.max", "68719476736\n") - usage := write("memory.current", "1073741824\n") + // The slice's usage counts every service under it, not just this one: that + // is the population its limit actually bounds. + leafUsage := write("leaf.current", "1073741824\n") + sliceUsage := write("slice.current", "1610612736\n") + rootUsage := write("root.current", "3221225472\n") stat := write("memory.stat", "inactive_file 0\n") s := newTestSampler(t, tree, clock, Options{}) - s.cgroupLimitPaths = []string{leaf, slice, root} - s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}} + s.cgroupUsagePaths = []cgroupUsagePath{ + {limit: leaf, usage: leafUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: slice, usage: sliceUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: root, usage: rootUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + } s.sample(context.Background()) system := s.Snapshot().System if system.MemTotalMB != 2048 { t.Fatalf("MemTotalMB = %d, want the 2048 the slice allows, not the leaf's looser 8192", system.MemTotalMB) } - // Total came from a cgroup, so used has to come from one too. - if system.MemUsedMB != 1024 { - t.Fatalf("MemUsedMB = %d, want the cgroup's own 1024", system.MemUsedMB) + // Used comes from the *same* cgroup the limit did. Pairing the slice's + // 2 GiB capacity with this service's own 1 GiB would show half the volume + // free while the sibling services under that slice have filled it. + if system.MemUsedMB != 1536 { + t.Fatalf("MemUsedMB = %d, want the 1536 charged to the slice whose limit binds", system.MemUsedMB) } // The other shape of the same bug: a leaf that imposes nothing at all while // an ancestor does. unlimited := write("unlimited.max", "max\n") s = newTestSampler(t, tree, clock, Options{}) - s.cgroupLimitPaths = []string{unlimited, slice, root} - s.cgroupUsagePaths = []cgroupUsagePath{{usage: usage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}} + s.cgroupUsagePaths = []cgroupUsagePath{ + {limit: unlimited, usage: leafUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: slice, usage: sliceUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: root, usage: rootUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + } s.sample(context.Background()) if got := s.Snapshot().System.MemTotalMB; got != 2048 { t.Fatalf("MemTotalMB = %d with an unlimited leaf, want the slice's 2048", got) } } + +// A cpuset is the other way a deployment caps CPU, and it leaves cpu.max saying +// "max". A process pinned to two CPUs on a sixty-four core host would otherwise +// divide its own busy time by sixty-four and report three percent while it is +// saturated. +func TestCPUCorrectedByCpusetWithoutAQuota(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // An eight-core host, so the two allowed CPUs are visibly a subset. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 9900)) + + dir := t.TempDir() + cpuset := filepath.Join(dir, "cpuset.cpus.effective") + if err := os.WriteFile(cpuset, []byte("2-3\n"), 0o644); err != nil { + t.Fatalf("write cpuset: %v", err) + } + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + if err := os.WriteFile(quota, []byte("max 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.cgroupCPUSetPaths = []string{cpuset} + s.sample(context.Background()) + + // 10 seconds of CPU over 5 seconds of wall time on two allowed CPUs: pegged, + // while the host is 1% busy. + writeUsage(10_000_000) + tree.write("stat", hostStat(200, 19800)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 100 { + t.Fatalf("CPUPct = %d, want 100 — the process is saturating every CPU it may use", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the 2 CPUs the cpuset allows, not the host's 8", system.Cores) + } +} + +func TestCountCPUSetEntries(t *testing.T) { + for _, tc := range []struct { + list string + want int + }{ + {"0-3", 4}, + {"0-3,8,12-13", 7}, + {"5", 1}, + {" 0-1 , 4 \n", 3}, + {"", 0}, + {"garbage", 0}, + // A reversed or negative range describes nothing; counting it would + // invent a budget out of a malformed file. + {"5-2", 0}, + {"-1", 0}, + } { + t.Run(tc.list, func(t *testing.T) { + if got := countCPUSetEntries(tc.list); got != tc.want { + t.Fatalf("countCPUSetEntries(%q) = %d, want %d", tc.list, got, tc.want) + } + }) + } +} diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go index 35f0a30fe..26251fd76 100644 --- a/internal/nodemetrics/system.go +++ b/internal/nodemetrics/system.go @@ -208,10 +208,12 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { // Every limit in force is read, not just the first: the list runs from this // process's own cgroup up through its ancestors, and a leaf that says "max" // can still sit inside a slice or pod cgroup that does not. The kernel - // OOM-kills against the tightest of them, so that is the one to report. - capped := false - for _, path := range s.cgroupLimitPaths { - limit, err := ReadCgroupMemoryLimit(path) + // OOM-kills against the tightest of them, so that is the one to report — and + // the level it came from is kept, because the usage that fills it is + // everything charged to that cgroup, not just this process's own. + var binding *cgroupUsagePath + for i, level := range s.cgroupUsagePaths { + limit, err := ReadCgroupMemoryLimit(level.limit) if err != nil || limit <= 0 { continue } @@ -220,7 +222,7 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { // give it. if totalBytes == 0 || limit < totalBytes { totalBytes = limit - capped = true + binding = &s.cgroupUsagePaths[i] } } @@ -230,8 +232,8 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { // — "1 GiB of 64 GiB" on a machine that is nearly out of memory, because the // two numbers describe different domains. Whichever domain total came from, // used has to come from the same one. - if capped { - if usage, ok := s.cgroupMemoryUsage(); ok { + if binding != nil { + if usage, ok := cgroupMemoryUsage(*binding); ok { usedBytes = usage } } @@ -241,20 +243,20 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { return usedBytes, totalBytes } -// cgroupMemoryUsage returns the working set of this process's memory cgroup: -// current charge minus reclaimable file pages. -func (s *Sampler) cgroupMemoryUsage() (int64, bool) { - for _, paths := range s.cgroupUsagePaths { - usage, err := readCgroupSingleValue(paths.usage) - if err != nil { - continue - } - if inactive, err := readCgroupStatKey(paths.stat, paths.inactiveFile); err == nil && inactive > 0 && inactive <= usage { - usage -= inactive - } - return usage, true +// cgroupMemoryUsage returns the working set of one memory cgroup: its current +// charge minus reclaimable file pages. +// +// The level is the caller's choice rather than a search, because the only level +// worth measuring is the one whose limit binds — see memoryStats. +func cgroupMemoryUsage(level cgroupUsagePath) (int64, bool) { + usage, err := readCgroupSingleValue(level.usage) + if err != nil { + return 0, false + } + if inactive, err := readCgroupStatKey(level.stat, level.inactiveFile); err == nil && inactive > 0 && inactive <= usage { + usage -= inactive } - return 0, false + return usage, true } func clampPercent(value int) int { diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index ef3b3d2ac..dd5c7af49 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -454,15 +454,17 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply note, driftBaseline := resolveDriftNote(n.CapabilityDrift, n.CapabilityDriftBaseline, drift, parsed, payload) refreshedAt := time.Now() if hc.repo != nil { - // Fenced on the URL this payload was fetched from: the fetch is - // detached and bounded at two minutes, so the row may since have been - // repointed at a different worker. - if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline); err != nil { + // Fenced on the URL this payload was fetched from — the fetch is + // detached and bounded, so the row may since have been repointed at a + // different worker — and on the report it is replacing, so a slower + // sweep on another replica cannot land an older report on top of a + // newer one. + if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline, n.CapabilitiesHash); err != nil { if errors.Is(err, ErrNodeMoved) { // Not a failure to report loudly: the node was edited or // removed while this fetch ran, and the next sweep will fetch // whatever the row addresses now. - slog.InfoContext(ctx, "discarded a capability report for a node that changed identity mid-fetch", + slog.InfoContext(ctx, "discarded a capability report the row no longer expects", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) return err } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 4d9155829..7c1074236 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -486,18 +486,27 @@ func (r *Repository) UpdateHealth(ctx context.Context, id int, checkedURL string // shared-GPU work on that reading until another sweep corrected it. Trailing // slashes are ignored on both sides because the pools normalize URLs and the // column does not. -func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) error { +// It is also fenced on the report it believes it is replacing. Every API +// replica runs its own health sweep, so two can fetch successive reports from +// one node concurrently; without this a slower fetch of an older report lands +// after a newer one and overwrites it, taking the durable GPU identities and +// drift state back with it until some later sweep repairs them. Comparing +// against the hash the caller read before fetching makes the write a +// compare-and-set: whichever replica gets there first wins, and the loser +// discards a report that no longer describes the row it was derived from. +// Clock skew between replicas does not enter into it. +func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte, replacing *string) error { tag, err := r.pool.Exec(ctx, `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4, capability_drift = $5, capability_drift_baseline = $7 - WHERE id = $1 AND rtrim(url, '/') = rtrim($6, '/')`, - id, capabilities, hash, refreshedAt, drift, fetchedFrom, driftBaseline) + WHERE id = $1 AND rtrim(url, '/') = rtrim($6, '/') AND capabilities_hash IS NOT DISTINCT FROM $8`, + id, capabilities, hash, refreshedAt, drift, fetchedFrom, driftBaseline, replacing) if err != nil { return fmt.Errorf("update node capabilities: %w", err) } if tag.RowsAffected() == 0 { - // Either the row is gone or it no longer addresses the worker this - // payload came from. Both mean the same thing to the caller: do not - // publish it. + // The row is gone, it no longer addresses the worker this payload came + // from, or another replica has already stored a different report. All + // three mean the same thing to the caller: do not publish it. return ErrNodeMoved } return nil diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index cd2e7456f..993c3c0fb 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -64,7 +64,7 @@ func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { payload := json.RawMessage(`{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"]}`) refreshedAt := time.Now().UTC().Truncate(time.Millisecond) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:abc", refreshedAt, nil, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:abc", refreshedAt, nil, nil, nil); err != nil { t.Fatalf("update capabilities: %v", err) } @@ -113,7 +113,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { note := "verified hardware backends lost: nvenc; resolved backend nvenc -> none" payload := json.RawMessage(`{"resolved":"none"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:degraded", time.Now(), ¬e, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:degraded", time.Now(), ¬e, nil, nil); err != nil { t.Fatalf("update capabilities with drift: %v", err) } reloaded, err := repo.GetByID(ctx, node.ID) @@ -125,7 +125,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { } recovered := json.RawMessage(`{"resolved":"nvenc"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, recovered, "sha256:recovered", time.Now(), nil, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, recovered, "sha256:recovered", time.Now(), nil, nil, ptrString("sha256:degraded")); err != nil { t.Fatalf("update capabilities without drift: %v", err) } reloaded, err = repo.GetByID(ctx, node.ID) @@ -139,7 +139,7 @@ func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { func TestRepositoryUpdateCapabilitiesUnknownNode(t *testing.T) { repo := NewRepository(newNodeTestPool(t)) - err := repo.UpdateCapabilities(context.Background(), -1, "http://gone", []byte(`{}`), "sha256:abc", time.Now(), nil, nil) + err := repo.UpdateCapabilities(context.Background(), -1, "http://gone", []byte(`{}`), "sha256:abc", time.Now(), nil, nil, nil) if !errors.Is(err, ErrNodeMoved) { t.Fatalf("err = %v, want ErrNodeMoved", err) } @@ -168,7 +168,7 @@ func TestRepositoryUpdateCapabilitiesRefusesAfterAURLEdit(t *testing.T) { t.Fatalf("repoint node: %v", err) } - err = repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"qsv"}`), "sha256:stale", time.Now(), nil, nil) + err = repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"qsv"}`), "sha256:stale", time.Now(), nil, nil, nil) if !errors.Is(err, ErrNodeMoved) { t.Fatalf("err = %v, want ErrNodeMoved after the row was repointed", err) } @@ -198,7 +198,7 @@ func TestRepositoryUpdateCapabilitiesIgnoresATrailingSlash(t *testing.T) { t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) normalized := strings.TrimSuffix(node.URL, "/") - if err := repo.UpdateCapabilities(ctx, node.ID, normalized, []byte(`{"resolved":"qsv"}`), "sha256:ok", time.Now(), nil, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, normalized, []byte(`{"resolved":"qsv"}`), "sha256:ok", time.Now(), nil, nil, nil); err != nil { t.Fatalf("UpdateCapabilities with a normalized URL: %v", err) } } @@ -223,7 +223,7 @@ func TestRepositoryUpdateClearsWorkerStateWhenTheURLMoves(t *testing.T) { note := "verified hardware backends lost: qsv" payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:old", time.Now(), ¬e, []byte(`{"backends":["qsv"]}`)); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:old", time.Now(), ¬e, []byte(`{"backends":["qsv"]}`), nil); err != nil { t.Fatalf("store capabilities: %v", err) } if err := repo.UpdateHealth(ctx, node.ID, node.URL, true, 2, 0, []byte(`{"system":{"cpu_pct":41}}`)); err != nil { @@ -267,7 +267,7 @@ func TestRepositoryUpdateKeepsWorkerStateWithoutAMove(t *testing.T) { t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) - if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:keep", time.Now(), nil, nil); err != nil { + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:keep", time.Now(), nil, nil, nil); err != nil { t.Fatalf("store capabilities: %v", err) } @@ -285,3 +285,48 @@ func TestRepositoryUpdateKeepsWorkerStateWithoutAMove(t *testing.T) { t.Fatal("a trailing-slash change was treated as a different worker") } } + +func ptrString(value string) *string { return &value } + +// Every API replica runs its own health sweep, so two can fetch successive +// reports from one node at once. Without a fence on the report being replaced, +// a slower fetch of the older one lands last and takes the durable GPU +// identities and drift state back with it. +func TestUpdateCapabilitiesRefusesAReportThatFollowsAStaleOne(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("capability-cas-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://capability-cas-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"qsv"}`), "sha256:first", time.Now(), nil, nil, nil); err != nil { + t.Fatalf("first report: %v", err) + } + // A second replica stores its newer report. + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"nvenc"}`), "sha256:second", time.Now(), nil, nil, ptrString("sha256:first")); err != nil { + t.Fatalf("second report: %v", err) + } + // The first replica's *other* in-flight fetch finally lands. It read the + // row before either write, so it must not overwrite what is there now. + overtaken := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"vaapi"}`), "sha256:overtaken", time.Now(), nil, nil, ptrString("sha256:first")) + if !errors.Is(overtaken, ErrNodeMoved) { + t.Fatalf("overtaken report error = %v, want ErrNodeMoved", overtaken) + } + + stored, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:second" { + t.Fatalf("stored hash = %v, want the newer report kept", stored.CapabilitiesHash) + } +} diff --git a/internal/playback/capabilityhash.go b/internal/playback/capabilityhash.go index 181114200..4b0a00ea3 100644 --- a/internal/playback/capabilityhash.go +++ b/internal/playback/capabilityhash.go @@ -14,11 +14,20 @@ import ( // comparable token, so a reader (the node health sweep, an operator) detects // change without diffing a whole report or re-probing. // -// Only hardware identity and capability are hashed. Fields that describe the -// report rather than the host — Source, NodeURL, ProbeRequestTimeoutMillis, and -// the hash itself — are excluded, because a report of unchanged hardware must -// keep its hash no matter who asked for it or how. IntelDetected is excluded as -// well: it is derived from the render devices already covered. +// Only hardware identity and capability are hashed. Fields that vary with who +// asked rather than with the host — Source, NodeURL, and the hash itself — are +// excluded, because a report of unchanged hardware must keep its hash no matter +// who asked for it. IntelDetected is excluded as well: it is derived from the +// render devices already covered. +// +// ProbeRequestTimeoutMillis is included, though it describes the report rather +// than the hardware. It is the node's own statement of how long its answer may +// take, and the control plane sizes real deadlines from the stored copy. A node +// upgraded to a build that needs longer changes nothing else about itself, so +// leaving it out of the change signal meant the sweep never refetched and the +// API kept canceling that node's re-probes against a budget it had outgrown. +// It is stable for a given build and configuration, so including it costs one +// refetch at upgrade and nothing after. // // Every slice is ordered here rather than trusted from the input: probe and // filesystem enumeration order is incidental, so two reports of the same host @@ -42,6 +51,7 @@ type canonicalCapability struct { RenderDevices []string `json:"render_devices"` RenderDeviceDetails []canonicalRenderDevice `json:"render_device_details"` NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` + ProbeRequestTimeout int64 `json:"probe_request_timeout_ms"` DetectedBackends []canonicalDetectedBackend `json:"detected_backends"` Transformations []canonicalTransformation `json:"transformations"` ToneMapCapabilities []canonicalToneMap `json:"tone_map_capabilities"` @@ -87,6 +97,7 @@ func canonicalCapabilities(info HWAccelInfo) canonicalCapability { RenderDevices: sortedStrings(info.RenderDevices), RenderDeviceDetails: canonicalRenderDevices(info.RenderDeviceDetails), NVIDIAGPUUUIDs: sortedStrings(info.NVIDIAGPUUUIDs), + ProbeRequestTimeout: info.ProbeRequestTimeoutMillis, DetectedBackends: canonicalDetectedBackends(info.DetectedBackends), Transformations: canonicalTransformations(info.Transformations), ToneMapCapabilities: canonicalToneMaps(info.ToneMapCapabilities), diff --git a/internal/playback/capabilityhash_test.go b/internal/playback/capabilityhash_test.go index e08a37da3..216eaa694 100644 --- a/internal/playback/capabilityhash_test.go +++ b/internal/playback/capabilityhash_test.go @@ -52,15 +52,14 @@ func TestComputeCapabilityHashIgnoresSliceOrder(t *testing.T) { } } -// Fields that describe the report rather than the host must not move the hash: -// otherwise the same node hashes differently depending on who asked. -func TestComputeCapabilityHashIgnoresReportMetadata(t *testing.T) { +// Fields that vary with who asked must not move the hash: otherwise the same +// node hashes differently depending on the caller. +func TestComputeCapabilityHashIgnoresPerCallerMetadata(t *testing.T) { info := sampleCapabilityInfo() want := ComputeCapabilityHash(info) info.Source = "remote" info.NodeURL = "http://node-7:8080" - info.ProbeRequestTimeoutMillis = 42_000 info.CapabilityHash = "sha256:stale" if got := ComputeCapabilityHash(info); got != want { @@ -68,6 +67,22 @@ func TestComputeCapabilityHashIgnoresReportMetadata(t *testing.T) { } } +// The advertised probe budget does move it, though it describes the report +// rather than the hardware. The control plane sizes real deadlines from the +// stored copy, so a node upgraded to a build that needs longer — changing +// nothing else about itself — has to reach the sweep, or the API keeps +// canceling that node's re-probes against a budget it has outgrown. +func TestComputeCapabilityHashTracksTheAdvertisedProbeBudget(t *testing.T) { + info := sampleCapabilityInfo() + info.ProbeRequestTimeoutMillis = 111_000 + before := ComputeCapabilityHash(info) + + info.ProbeRequestTimeoutMillis = 136_000 + if got := ComputeCapabilityHash(info); got == before { + t.Fatal("a node that raised its advertised probe budget hashed identically; the sweep would never refetch it") + } +} + func TestComputeCapabilityHashIsPrefixedSHA256(t *testing.T) { hash := ComputeCapabilityHash(sampleCapabilityInfo()) if !strings.HasPrefix(hash, "sha256:") { diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go index 3deca0641..ff21978cb 100644 --- a/internal/proxy/reprobe.go +++ b/internal/proxy/reprobe.go @@ -41,6 +41,12 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques playback.InvalidateHWProbeCache() tonemap.InvalidateProbeCache() + // The resource sampler retires nvidia-smi after repeated failure, and a + // driver that was broken at start is exactly what a re-probe is called for. + // A proxy samples the same GPU a transcode node does, so it needs the same + // nudge — without it the node re-verifies its encoders here and still + // reports no GPU utilization until the breaker's own retry interval. + s.metrics.RetrySources() // buildCapabilitySnapshotLocked owns the probe deadline, so a re-probe can // never cost more than a cold capability fetch already may. diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go index becd895db..35106a3dc 100644 --- a/internal/transcodenode/reprobe_test.go +++ b/internal/transcodenode/reprobe_test.go @@ -365,3 +365,33 @@ func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { } server.gpu.endReprobe() } + +// Warmup is a real smoke encode, and the listener opens while it may still be +// running: an admin re-probe arriving in a node's first seconds would otherwise +// see an idle gate and run its matrix beside it, publishing a false hardware +// failure on a session-limited GPU. +func TestReprobeCapabilitiesRefusedWhileEncoderWarmupRuns(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + + // The gate is what warmup holds; observed here rather than by starting a + // real ffmpeg, which the test host has no hardware for. + server.gpu.holdWork() + if got := server.activeJobs.Load(); got != 0 { + t.Fatalf("active jobs = %d, want the warmup window where nothing is registered", got) + } + + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while the encoder is warming", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } + + server.gpu.endWork() + if _, ok := server.gpu.beginReprobe(0, 0); !ok { + t.Fatal("re-probe refused after warmup finished") + } + server.gpu.endReprobe() +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index a2b3f7c49..1575c5a7a 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -410,6 +410,13 @@ func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} playbackCfg := cfg.Playback go func() { defer close(done) + // Warmup is a real smoke encode, and the listener opens while this + // goroutine may still be running — so an admin re-probe arriving in the + // first seconds of a node's life would otherwise see an idle gate and + // run its matrix beside it. Held rather than requested: warmup is + // already started and refusing it here would only skip it. + s.gpu.holdWork() + defer s.gpu.endWork() if err := playback.WarmHardwareEncoder(ctx, playbackCfg.FFmpegPath, playbackCfg.HWAccel, playbackCfg.HWDevice); err != nil { slog.DebugContext(ctx, "transcode node hardware encoder warmup failed", "component", "transcodenode", "error", err) } From 213e318112f0a3e53f3b58ebc607d7f85f8c8b2a Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:09:30 -0400 Subject: [PATCH 047/163] fix(playback): re-probe a node whose capabilities changed mid-lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capability lookup overtaken by an invalidation handed the caller the pre-invalidation report and only declined to cache it. That protects later requests and not this one — and this one goes on to select transformations and a tone-map executor from it, possibly naming hardware the newer report says is gone. It now re-probes once. Not "treat as unavailable", which was the other option: most hash changes are not the hardware going away — a driver update, a new identity field, a raised probe budget all move it, and the last of those became true in the commit before this one — so refusing would reject playback the report in hand describes perfectly well, and on a single-transcode-node deployment there is nothing to fall back to. A second overtaken read is used as before, because a node changing faster than two probes can read it is a different problem and failing here does not fix it. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/playback_v3.go | 55 ++++++++++---- .../playback_v3_capability_retry_test.go | 76 +++++++++++++++++++ 2 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 internal/api/handlers/playback_v3_capability_retry_test.go diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index e2f83cd9b..dd7b59b99 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -372,19 +372,45 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } - // Snapshot the invalidation count before probing: anything this fetch - // learns describes the node as it was when the request left. - invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) - requestCtx, cancel := context.WithTimeout(ctx, h.remoteToneMapProbeTimeoutV3(nodeURL)) - defer cancel() - info, err := fetchRemoteTranscodeCapabilities(requestCtx, nodeURL, h.JWTSecret) - completedAt := time.Now() + // An invalidation landing mid-fetch means the report this probe is about to + // return describes the node as it was, not as it is — and this caller is + // about to select transformations and a tone-map executor from it. Re-probe + // rather than plan on it. + // + // Bounded at two attempts, and the second result is used even if it is + // overtaken too. Failing the request instead would be worse than a slightly + // stale inventory: most hash changes are not "the hardware went away" — a + // driver update, a new identity field, a raised probe budget all move it — + // so refusing would reject playback that the report in hand describes + // perfectly well, and on a single-transcode-node deployment there is nothing + // to fall back to. A node changing faster than two probes can read it is a + // different problem, and one this path cannot fix by failing. + var ( + info playback.HWAccelInfo + err error + completedAt time.Time + overtaken bool + ) + for attempt := range v3CapabilityFetchAttempts { + // Snapshot the invalidation count before probing: anything this fetch + // learns describes the node as it was when the request left. + invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) + requestCtx, cancel := context.WithTimeout(ctx, h.remoteToneMapProbeTimeoutV3(nodeURL)) + info, err = fetchRemoteTranscodeCapabilities(requestCtx, nodeURL, h.JWTSecret) + cancel() + completedAt = time.Now() + overtaken = h.nodeCapabilityInvalidationsV3(nodeURL) != invalidations + if !overtaken || attempt+1 == v3CapabilityFetchAttempts { + break + } + } + if err != nil { h.v3NodeCapabilitiesMu.Lock() if h.v3NodeCapabilities == nil { h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) } - if h.v3NodeCapabilityInvalidations[nodeURL] != invalidations { + if overtaken { h.v3NodeCapabilitiesMu.Unlock() return v3NodeCapabilityCache{}, err } @@ -403,11 +429,10 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR probeRequestTimeout: playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout), } h.v3NodeCapabilitiesMu.Lock() - if h.v3NodeCapabilityInvalidations[nodeURL] != invalidations { - // The node's hardware changed while this probe was in flight. Hand the - // result to this caller, which has nothing better, but leave the cache - // empty so the next lookup re-probes instead of serving it for a full - // TTL to everyone else. + if overtaken { + // Still overtaken after the retry. Hand the result to this caller, which + // has nothing better, but leave the cache empty so the next lookup + // re-probes instead of serving it for a full TTL to everyone else. h.v3NodeCapabilitiesMu.Unlock() return entry, nil } @@ -419,6 +444,10 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } +// v3CapabilityFetchAttempts is how many times one lookup will re-probe a node +// whose capabilities changed while it was being read. +const v3CapabilityFetchAttempts = 2 + func (h *PlaybackHandler) nodeCapabilityInvalidationsV3(nodeURL string) uint64 { h.v3NodeCapabilitiesMu.Lock() defer h.v3NodeCapabilitiesMu.Unlock() diff --git a/internal/api/handlers/playback_v3_capability_retry_test.go b/internal/api/handlers/playback_v3_capability_retry_test.go new file mode 100644 index 000000000..fb8dfe76f --- /dev/null +++ b/internal/api/handlers/playback_v3_capability_retry_test.go @@ -0,0 +1,76 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +// A capability lookup overtaken by an invalidation was handing the caller the +// pre-invalidation report, and that caller goes on to pick transformations and +// a tone-map executor from it — possibly hardware the newer report says is +// gone. It re-probes instead. +func TestLookupRemoteCapabilitiesRefetchesWhenOvertakenMidFlight(t *testing.T) { + handler := NewPlaybackHandler(nil) + var fetches atomic.Int32 + + // Captured after the server exists; the cache is keyed by the URL the lookup + // was given, which is not the Host header the node sees. + var nodeURL string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempt := fetches.Add(1) + if attempt == 1 { + // The health sweep notices the node changed while this first read is + // still in flight. The counter is bumped directly rather than through + // RefreshNodeCapabilitiesV3, which would also start a background + // re-probe and make the fetch count say nothing about this lookup. + handler.v3NodeCapabilitiesMu.Lock() + if handler.v3NodeCapabilityInvalidations == nil { + handler.v3NodeCapabilityInvalidations = make(map[string]uint64) + } + handler.v3NodeCapabilityInvalidations[nodeURL]++ + handler.v3NodeCapabilitiesMu.Unlock() + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "resolved": "qsv", + "render_devices": []string{"/dev/dri/renderD128"}, + "transformations": []map[string]any{ + {"name": "hdr_to_sdr_tone_map", "executor": "qsv", "recipe_version": "v3"}, + }, + }) + })) + t.Cleanup(node.Close) + nodeURL = node.URL + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), nodeURL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + if got := fetches.Load(); got != 2 { + t.Fatalf("fetches = %d, want the overtaken read repeated exactly once", got) + } +} + +// The ordinary path must not pay for that: a lookup nothing invalidates reads +// the node exactly once. +func TestLookupRemoteCapabilitiesFetchesOnceWhenNothingInvalidates(t *testing.T) { + handler := NewPlaybackHandler(nil) + var fetches atomic.Int32 + + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fetches.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"resolved": "qsv"}) + })) + t.Cleanup(node.Close) + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), node.URL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + if got := fetches.Load(); got != 1 { + t.Fatalf("fetches = %d, want a single read on the uncontended path", got) + } +} From c38adf66deabe958247075d523035d1f13aa53da Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:35:58 -0400 Subject: [PATCH 048/163] fix(nodes): pin the node row, bind CPU and memory to the same cgroup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The memory walk was dead code. Adding `limit` to cgroupUsagePath made memoryStats select the binding level from it, but withCgroupSelfUsagePaths still rewrote only usage and stat — so every emitted level carried the mount-root limit and a process's own memory.max was never read at all. Levels are now built as coordinated limit/usage/stat tuples, one per cgroup from this process to the root, deduplicated by limit. CPU had the same pairing error the memory path had before it: usage from the leaf, quota from whichever ancestor bound. A quota on a shared slice is spent by every service under it, so Silo at ten percent beside a sibling at ninety reported ten while the group was saturated and being throttled. effectiveCgroupCPUQuota now returns the level it chose and usage is read from there. The warmup gate claim was inside the goroutine, which is the window it was meant to close: the scheduler makes no promise about when that goroutine runs, so a re-probe could still find an idle gate. It is taken before the goroutine is spawned. Losing the capability compare-and-set was reported as ErrNodeMoved, which told the replica to discard and move on — so it kept comparing against a hash the row no longer had, lost the same write every sweep, and served stale GPU identities until something unrelated reloaded its pools. A CAS loss is now ErrCapabilitiesSuperseded and the replica adopts the stored row, including firing its own playback-capability invalidation. The advertised-budget ceiling was a round five minutes, already below the 311 seconds a nine-device node legitimately asks for, so the API canceled that node's re-probe before its own deadline every time. It is derived from the probe formula at a documented device count instead. It is still a ceiling: the value comes off the wire from a worker. On a split-horizon deployment the only match is NODE_NAME, so renaming a node severed the association permanently — the worker kept its last policy while the API dispatched the row's current one. The watcher now remembers the row id it resolved to and prefers it, since neither the url nor the name survives an edit. Also: newTestSampler left cgroupCPUSetPaths pointing at the real host, so the new cpuset correction read the CI runner's cgroup and failed a test that passes on macOS. Every cgroup source is cleared there now, with a test asserting the fixture reads nothing under /sys/fs/cgroup or /proc. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/playback_v3_test.go | 7 +- internal/downloads/remote_preparer_test.go | 9 +- .../remote_transcode_reconstruct_test.go | 7 +- internal/nodeconfig/watcher.go | 77 +++++++++-- internal/nodemetrics/cgroupcpu.go | 46 +++++-- internal/nodemetrics/cgrouppath.go | 42 ++++-- internal/nodemetrics/cgrouppath_test.go | 130 ++++++++++++++++-- internal/nodemetrics/sampler_test.go | 33 +++++ internal/nodepool/health.go | 51 +++++++ internal/nodepool/repository.go | 27 +++- .../nodepool/repository_capabilities_test.go | 10 +- internal/playback/capabilityhash_test.go | 23 ++++ internal/playback/gpudetect.go | 16 ++- internal/playback/gpudetect_test.go | 9 +- internal/tonemap/probe.go | 21 +++ internal/transcodenode/server.go | 13 +- 16 files changed, 447 insertions(+), 74 deletions(-) diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 2e1a3ad45..ef859a683 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -5498,11 +5498,14 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { handler.v3NodeCapabilitiesMu.Lock() delete(handler.v3NodeCapabilities, remote.URL) handler.v3NodeCapabilitiesMu.Unlock() - probeTimeoutMillis = (10 * time.Minute).Milliseconds() + probeTimeoutMillis = (24 * time.Hour).Milliseconds() if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), remote.URL, false); err != nil { t.Fatal(err) } - if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), 5*time.Minute; got != want { + // Still bounded — the value comes off the wire from a worker — but at the + // ceiling the probe formula produces rather than a round number that a real + // nine-device node already exceeds. + if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), tonemap.MaxProbeRequestTimeout(); got != want { t.Fatalf("bounded remote probe timeout = %s, want %s", got, want) } } diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index ea32555aa..2595dd3de 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -808,7 +808,14 @@ func TestNormalizeRemoteToneMapProbeTimeout(t *testing.T) { {name: "missing", want: 5 * time.Second}, {name: "too small", millis: time.Second.Milliseconds(), want: 5 * time.Second}, {name: "node specific", millis: (161 * time.Second).Milliseconds(), want: 161 * time.Second}, - {name: "too large", millis: (10 * time.Minute).Milliseconds(), want: 5 * time.Minute}, + { + // The ceiling is derived from the probe formula, not picked, so the + // expectation is too — a round number was already below what a + // nine-device node legitimately advertises. + name: "too large", + millis: (24 * time.Hour).Milliseconds(), + want: tonemap.MaxProbeRequestTimeout(), + }, } { t.Run(test.name, func(t *testing.T) { if got := normalizeRemoteToneMapProbeTimeout(test.millis); got != test.want { diff --git a/internal/jellycompat/remote_transcode_reconstruct_test.go b/internal/jellycompat/remote_transcode_reconstruct_test.go index 9e1f602d4..003c91a51 100644 --- a/internal/jellycompat/remote_transcode_reconstruct_test.go +++ b/internal/jellycompat/remote_transcode_reconstruct_test.go @@ -1060,8 +1060,11 @@ func TestRemoteTranscodeStartTimeoutCoversColdProbePreflightAndReadiness(t *test if got := handler.remoteTranscodeStartTimeout(request, (137 * time.Second).Milliseconds()); got != want { t.Fatalf("remote transcode start timeout = %v, want %v", got, want) } - maxWant := 5*time.Minute + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout - if got := handler.remoteTranscodeStartTimeout(request, (10 * time.Minute).Milliseconds()); got != maxWant { + // Still bounded — the advertisement comes off the wire from a worker — but + // at the ceiling the probe formula produces, not a round number a real + // nine-device node already exceeds. + maxWant := tonemap.MaxProbeRequestTimeout() + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout + if got := handler.remoteTranscodeStartTimeout(request, (24 * time.Hour).Milliseconds()); got != maxWant { t.Fatalf("bounded remote transcode start timeout = %v, want %v", got, maxWant) } fallbackWant := compatRemoteNodeProbeFallbackTimeout + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index 992892019..a067f8f6b 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -79,6 +79,33 @@ type Watcher struct { missingRowLogged bool duplicateRowLogged bool ambiguousNameLogged bool + // nodeRowID is the row this worker has resolved to, kept so a later rename + // or repoint cannot sever the association. Zero until a lookup succeeds. + nodeRowID int +} + +// rememberNodeRowID records the row a lookup resolved to. +func (w *Watcher) rememberNodeRowID(id int) { + if id <= 0 { + return + } + w.mu.Lock() + w.nodeRowID = id + w.mu.Unlock() +} + +// rememberedNodeRowID returns the row this worker previously resolved to. +func (w *Watcher) rememberedNodeRowID() (int, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + return w.nodeRowID, w.nodeRowID > 0 +} + +// forgetNodeRowID drops a remembered row that no longer exists. +func (w *Watcher) forgetNodeRowID() { + w.mu.Lock() + w.nodeRowID = 0 + w.mu.Unlock() } // NewWatcher creates a new config watcher. Call Start to begin watching. The @@ -360,8 +387,32 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName st if w.pool == nil { return nodeHWOverrides{}, false, errors.New("no database pool") } - overrides, matched, err := w.queryOverrideRows(ctx, - `SELECT url, hw_accel_override, hw_device_override FROM stream_nodes + // The row this worker already resolved to, by its immutable id. + // + // Neither of the identities below survives an edit: a repoint changes the + // url, and a rename changes the name — and on a split-horizon deployment the + // name is the only match there was, so renaming a node severs the + // association permanently. The worker then keeps whatever policy it last + // read while the API dispatches the row's current one, which is the exact + // backend/device mismatch this overlay exists to prevent. Once the row has + // been identified, its id is what identifies it. + if id, ok := w.rememberedNodeRowID(); ok { + overrides, _, matched, err := w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE id = $1`, id) + if err != nil { + return nodeHWOverrides{}, false, err + } + if len(matched) > 0 { + return overrides, true, nil + } + // The row was deleted. Forget it and fall back to the identities, which + // is how a re-registered node finds its new row. + w.forgetNodeRowID() + } + + overrides, id, matched, err := w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes WHERE rtrim(url, '/') = rtrim($1, '/') ORDER BY id LIMIT 2`, nodeURL) if err != nil { return nodeHWOverrides{}, false, err @@ -370,6 +421,7 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName st w.logDuplicateNodeRows(ctx, nodeURL, matched) } if len(matched) > 0 { + w.rememberNodeRowID(id) return overrides, true, nil } @@ -381,8 +433,8 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName st if nodeName == "" { return nodeHWOverrides{}, false, nil } - overrides, matched, err = w.queryOverrideRows(ctx, - `SELECT url, hw_accel_override, hw_device_override FROM stream_nodes + overrides, id, matched, err = w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes WHERE name = $1 ORDER BY id LIMIT 2`, nodeName) if err != nil { return nodeHWOverrides{}, false, err @@ -391,6 +443,7 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName st case 0: return nodeHWOverrides{}, false, nil case 1: + w.rememberNodeRowID(id) return overrides, true, nil default: w.logAmbiguousNodeName(ctx, nodeName, matched) @@ -400,34 +453,36 @@ func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName st // queryOverrideRows runs one identity query and returns the first row plus the // urls of every row it matched, so callers can act on ambiguity. -func (w *Watcher) queryOverrideRows(ctx context.Context, query, arg string) (nodeHWOverrides, []string, error) { +func (w *Watcher) queryOverrideRows(ctx context.Context, query string, arg any) (nodeHWOverrides, int, []string, error) { rows, err := w.pool.Query(ctx, query, arg) if err != nil { - return nodeHWOverrides{}, nil, fmt.Errorf("query node acceleration overrides: %w", err) + return nodeHWOverrides{}, 0, nil, fmt.Errorf("query node acceleration overrides: %w", err) } defer rows.Close() var ( overrides nodeHWOverrides + firstID int matched []string ) for rows.Next() { var ( + id int url string row nodeHWOverrides ) - if err := rows.Scan(&url, &row.HWAccel, &row.HWDevice); err != nil { - return nodeHWOverrides{}, nil, fmt.Errorf("scan node acceleration overrides: %w", err) + if err := rows.Scan(&id, &url, &row.HWAccel, &row.HWDevice); err != nil { + return nodeHWOverrides{}, 0, nil, fmt.Errorf("scan node acceleration overrides: %w", err) } if len(matched) == 0 { - overrides = row + overrides, firstID = row, id } matched = append(matched, url) } if err := rows.Err(); err != nil { - return nodeHWOverrides{}, nil, fmt.Errorf("read node acceleration overrides: %w", err) + return nodeHWOverrides{}, 0, nil, fmt.Errorf("read node acceleration overrides: %w", err) } - return overrides, matched, nil + return overrides, firstID, matched, nil } // logAmbiguousNodeName warns once per process: several registered nodes share diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index 9f1cdac41..df8baa643 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -143,35 +143,51 @@ type cgroupCPUSample struct { // the cgroup imposes no quota). func (s *Sampler) cgroupCPU(now time.Time) (cgroupCPUSample, float64) { for _, paths := range s.cgroupCPUPaths { - usage, err := readCgroupCPUUsage(paths) + if _, err := readCgroupCPUUsage(paths); err != nil { + continue + } + binding, quota := effectiveCgroupCPUQuota(paths) + // Usage comes from whichever level supplied the binding quota, not from + // the leaf. They are one measurement: a quota shared with sibling + // services throttles on their CPU time too, so dividing only this + // process's by it reports ten percent for a group that is saturated and + // being throttled — the same pairing error the memory path had. + usage, err := readCgroupCPUUsage(binding) if err != nil { continue } - return cgroupCPUSample{usageNS: usage, at: now, valid: true}, effectiveCgroupCPUQuota(paths) + return cgroupCPUSample{usageNS: usage, at: now, valid: true}, quota } return cgroupCPUSample{}, 0 } // effectiveCgroupCPUQuota returns the tightest CPU budget in force on this -// cgroup, in cores, or 0 when nothing above it imposes one. +// cgroup, in cores, together with the level that imposes it. // -// Usage and quota are read from different places on purpose. Usage has to be -// this process's own — an ancestor's counts every sibling service on the -// machine — while the quota is whatever the kernel will actually throttle -// against, which is the smallest limit anywhere between here and the mount -// root. A systemd unit inside a slice with CPUQuota=, or a container under a -// limited pod cgroup, reads "max" at its leaf and is nonetheless capped; taking -// the leaf's answer would normalize a two-core service against sixty-four. +// The quota is whatever the kernel will actually throttle against, which is the +// smallest limit anywhere between here and the mount root. A systemd unit inside +// a slice with CPUQuota=, or a container under a limited pod cgroup, reads "max" +// at its leaf and is nonetheless capped; taking the leaf's answer would +// normalize a two-core service against sixty-four. // -// The pair moves together at each level: a quota from one cgroup divided by a +// The level is returned because usage has to be read from it too. A quota on a +// shared ancestor is spent by every service under it, so this process's own CPU +// time over that quota describes nothing: Silo at ten percent beside a sibling +// at ninety reports ten, while the group is saturated and throttled. +// +// Everything within a level moves together: a quota from one cgroup divided by a // period from another describes no real budget. -func effectiveCgroupCPUQuota(paths cgroupCPUPath) float64 { +func effectiveCgroupCPUQuota(paths cgroupCPUPath) (cgroupCPUPath, float64) { quotas := cgroupAncestorPaths(paths.quota) periods := cgroupAncestorPaths(paths.period) - tightest := 0.0 + usages := cgroupAncestorPaths(paths.usage) + binding, tightest := paths, 0.0 for i, quota := range quotas { level := paths level.quota = quota + if i < len(usages) { + level.usage = usages[i] + } if paths.period != "" { if i >= len(periods) { break @@ -183,10 +199,10 @@ func effectiveCgroupCPUQuota(paths cgroupCPUPath) float64 { continue } if tightest == 0 || cores < tightest { - tightest = cores + tightest, binding = cores, level } } - return tightest + return binding, tightest } // readCgroupCPUUsage returns cumulative cgroup CPU time in nanoseconds. diff --git a/internal/nodemetrics/cgrouppath.go b/internal/nodemetrics/cgrouppath.go index 41359a76c..825824328 100644 --- a/internal/nodemetrics/cgrouppath.go +++ b/internal/nodemetrics/cgrouppath.go @@ -29,8 +29,10 @@ import ( // (/docker/) that does not exist under its own mount, so the rewritten // path simply fails to open and the root read happens as before. -// cgroupMountRoot is where the cgroup hierarchy is mounted on Linux. -const cgroupMountRoot = "/sys/fs/cgroup" +// cgroupMountRoot is where the cgroup hierarchy is mounted on Linux. It is a +// var so a test can point the ancestor walk at a temporary tree; nothing in +// production writes it. +var cgroupMountRoot = "/sys/fs/cgroup" // cgroupRelativePaths reports this process's path within each cgroup hierarchy, // read from /self/cgroup. @@ -185,17 +187,39 @@ func withCgroupSelfCPUPaths(relative map[string]string, layouts []cgroupCPUPath) return out } -// withCgroupSelfUsagePaths is withCgroupSelfPaths for the memory usage layouts. +// withCgroupSelfUsagePaths is withCgroupSelfPaths for the memory layouts, which +// name three files that have to move together: memoryStats picks the level whose +// *limit* binds and then reads usage from that same level, so a tuple whose +// limit and usage come from different cgroups measures one population against +// another's capacity. +// +// Every level from this process's own cgroup up to the mount root is emitted, +// because the binding limit is often an ancestor's — a slice with MemoryMax=, a +// pod cgroup shared with sidecars — while the leaf says "max". func withCgroupSelfUsagePaths(relative map[string]string, layouts []cgroupUsagePath) []cgroupUsagePath { out := make([]cgroupUsagePath, 0, len(layouts)*2) + seen := make(map[string]bool, len(layouts)*2) + add := func(level cgroupUsagePath) { + if level.limit == "" || seen[level.limit] { + return + } + seen[level.limit] = true + out = append(out, level) + } for _, layout := range layouts { - own := layout - own.usage = cgroupSelfFile(relative, layout.usage) - own.stat = cgroupSelfFile(relative, layout.stat) - if own.usage != "" && own.stat != "" { - out = append(out, own) + if own := cgroupSelfFile(relative, layout.limit); own != "" { + usageName, statName := path.Base(layout.usage), path.Base(layout.stat) + for _, ancestor := range cgroupAncestorPaths(own) { + dir := path.Dir(ancestor) + add(cgroupUsagePath{ + limit: ancestor, + usage: path.Join(dir, usageName), + stat: path.Join(dir, statName), + inactiveFile: layout.inactiveFile, + }) + } } - out = append(out, layout) + add(layout) } return out } diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go index 2610b06ce..596233130 100644 --- a/internal/nodemetrics/cgrouppath_test.go +++ b/internal/nodemetrics/cgrouppath_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" ) @@ -190,23 +191,62 @@ func TestWithCgroupSelfUsagePathsRewritesEveryFileOrNone(t *testing.T) { relative := map[string]string{"memory": "system.slice/silo.service"} got := withCgroupSelfUsagePaths(relative, cgroupMemoryUsagePaths) - if len(got) != len(cgroupMemoryUsagePaths)+1 { - t.Fatalf("got %d layouts, want only the v1 one rewritten alongside the originals", len(got)) + // The v1 layout resolves, so it contributes one tuple per cgroup from this + // process up to the mount root, followed by the root layouts the list + // started with. The v2 layout has no "" membership here and is carried + // through unrewritten. + want := []cgroupUsagePath{ + { + limit: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: "/sys/fs/cgroup/memory/system.slice/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory/system.slice/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/system.slice/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: cgroupMemoryUsagePaths[1].limit, + usage: cgroupMemoryUsagePaths[1].usage, + stat: cgroupMemoryUsagePaths[1].stat, + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: "/sys/fs/cgroup/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, } - var rewritten *cgroupUsagePath - for i, layout := range got { - if layout.usage == "/sys/fs/cgroup/memory/system.slice/silo.service/memory.usage_in_bytes" { - rewritten = &got[i] + var v1 []cgroupUsagePath + for _, level := range got { + if strings.Contains(level.limit, "limit_in_bytes") { + v1 = append(v1, level) } } - if rewritten == nil { - t.Fatalf("no rewritten v1 layout in %+v", got) + if !slices.Equal(v1, want) { + t.Fatalf("v1 levels =\n%+v\nwant\n%+v", v1, want) } - if rewritten.stat != "/sys/fs/cgroup/memory/system.slice/silo.service/memory.stat" { - t.Fatalf("stat = %q, want it rewritten beside its usage file", rewritten.stat) + + // Every emitted level names all three files from one cgroup: memoryStats + // picks by limit and then reads usage from the same tuple, so a mismatch + // measures one population against another's capacity. + for _, level := range got { + if level.limit == "" { + continue + } + dir := filepath.Dir(level.limit) + if filepath.Dir(level.usage) != dir || filepath.Dir(level.stat) != dir { + t.Fatalf("level %+v mixes cgroups; limit, usage and stat must share one directory", level) + } } - if rewritten.inactiveFile != cgroupInactiveFileKeyV1 { - t.Fatalf("inactiveFile = %q, want the layout's own key preserved", rewritten.inactiveFile) + + // With no membership the list is exactly what it was. + if got := withCgroupSelfUsagePaths(nil, cgroupMemoryUsagePaths); !slices.Equal(got, cgroupMemoryUsagePaths) { + t.Fatalf("withCgroupSelfUsagePaths(nil) = %+v, want the original list", got) } } @@ -262,7 +302,8 @@ func TestEffectiveCgroupCPUQuotaTakesTheTightestAncestor(t *testing.T) { // cgroupAncestorPaths only climbs inside the real cgroup mount, so the walk // is exercised here by handing effectiveCgroupCPUQuota each level directly. quotaAt := func(dir string) float64 { - return effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(dir, "cpu.max")}) + _, cores := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(dir, "cpu.max")}) + return cores } // The service says "max" while its slice allows two cores. @@ -277,7 +318,7 @@ func TestEffectiveCgroupCPUQuotaTakesTheTightestAncestor(t *testing.T) { // A leaf tighter than its slice wins, and a looser one loses. write(leaf, "cpu.max", "100000 100000\n") - if got := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(leaf, "cpu.max")}); got != 1 { + if _, got := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(leaf, "cpu.max")}); got != 1 { t.Fatalf("tighter leaf = %v cores, want 1", got) } } @@ -292,7 +333,7 @@ func TestEffectiveCgroupCPUQuotaPairsQuotaWithItsOwnPeriod(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "cpu.cfs_period_us"), []byte("100000\n"), 0o644); err != nil { t.Fatal(err) } - got := effectiveCgroupCPUQuota(cgroupCPUPath{ + _, got := effectiveCgroupCPUQuota(cgroupCPUPath{ quota: filepath.Join(dir, "cpu.cfs_quota_us"), period: filepath.Join(dir, "cpu.cfs_period_us"), }) @@ -300,3 +341,62 @@ func TestEffectiveCgroupCPUQuotaPairsQuotaWithItsOwnPeriod(t *testing.T) { t.Fatalf("v1 quota = %v cores, want 4", got) } } + +// The quota and the usage measured against it have to come from the same +// cgroup. A quota on a shared ancestor is spent by every service under it, so +// this process's own CPU time over that quota describes nothing: Silo at ten +// percent beside a sibling at ninety would report ten, while the group is +// saturated and being throttled. +func TestEffectiveCgroupCPUQuotaReturnsTheLevelThatBinds(t *testing.T) { + root := t.TempDir() + // The walk only climbs inside the cgroup mount, so the fixture becomes one. + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + // The service imposes nothing; the slice above it allows two cores and + // carries the usage of everything under it. + write(leaf, "cpu.max", "max 100000\n") + write(leaf, "cpu.stat", "usage_usec 1000000\n") + write(root, "cpu.max", "200000 100000\n") + write(root, "cpu.stat", "usage_usec 9000000\n") + + binding, cores := effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 2 { + t.Fatalf("cores = %v, want the slice's 2", cores) + } + if want := filepath.Join(root, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the slice's own %q", binding.usage, want) + } + + // Nothing above it binds: the level stays the leaf's, so an unconstrained + // or leaf-limited process still measures itself. + write(root, "cpu.max", "max 100000\n") + write(leaf, "cpu.max", "100000 100000\n") + binding, cores = effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 1 { + t.Fatalf("cores = %v, want the leaf's 1", cores) + } + if want := filepath.Join(leaf, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the leaf's own %q", binding.usage, want) + } +} diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 432fbbe86..bbbc83f37 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "time" ) @@ -64,8 +65,12 @@ func newTestSampler(t *testing.T, tree *procTree, clock *fakeClock, opts Options // have a real /host/proc; tests that exercise the lxcfs override set this // explicitly to a tree that does. s.hostProcDir = filepath.Join(tree.root, "no-such-host-proc") + // Every cgroup source is cleared, not just the ones a given test sets: these + // default to real host paths, and on Linux a test that forgot one reads the + // machine it is running on instead of its fixture. s.cgroupUsagePaths = nil s.cgroupCPUPaths = nil + s.cgroupCPUSetPaths = nil s.statfs = func(string) (fsStats, error) { return fsStats{}, os.ErrNotExist } s.runNVIDIASMI = func(context.Context) ([]byte, error) { return nil, os.ErrNotExist } return s @@ -955,3 +960,31 @@ func TestCountCPUSetEntries(t *testing.T) { }) } } + +// Every cgroup source defaults to a real host path, so a fixture that forgets +// one silently measures the machine the tests run on. That is invisible on a +// developer's macOS laptop and decides the result on a Linux CI runner — which +// is exactly how TestCPUWithoutCgroupQuotaUsesHostCores started failing only in +// CI. This asserts the constructor leaves nothing pointing at the system. +func TestNewTestSamplerReadsNoRealCgroupPaths(t *testing.T) { + s := newTestSampler(t, newProcTree(t), newFakeClock(), Options{}) + + var configured []string + for _, level := range s.cgroupUsagePaths { + configured = append(configured, level.limit, level.usage, level.stat) + } + for _, level := range s.cgroupCPUPaths { + configured = append(configured, level.usage, level.quota, level.period) + } + configured = append(configured, s.cgroupCPUSetPaths...) + configured = append(configured, s.procDir, s.hostProcDir) + + for _, path := range configured { + if path == "" { + continue + } + if strings.HasPrefix(path, cgroupMountRoot) || path == "/proc" { + t.Fatalf("test sampler reads the real host path %q", path) + } + } +} diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index dd5c7af49..e3c4c212d 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -402,6 +402,47 @@ func (hc *HealthChecker) RefreshNodeCapabilities(ctx context.Context, n *Node) e // in flight is at least as fresh as the one the caller wanted. var ErrCapabilityRefreshInFlight = errors.New("node capability refresh already in flight") +// adoptStoredCapabilities brings this replica's in-memory node up to whatever +// is stored, after another writer won the capability compare-and-set. +// +// Without it the losing replica keeps the pre-fetch hash in memory, compares +// against it on every sweep, refetches, and loses the same write again — while +// its pools serve GPU identities and a drift note that the row has moved past. +// Reading the row is cheap next to the capability fetch that preceded it. +func (hc *HealthChecker) adoptStoredCapabilities( + ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc, onChanged func(string), +) { + if hc.repo == nil { + return + } + stored, err := hc.repo.GetByID(ctx, n.ID) + if err != nil || stored == nil || stored.CapabilitiesHash == nil { + return + } + refreshedAt := time.Now() + if stored.CapabilitiesRefreshedAt != nil { + refreshedAt = *stored.CapabilitiesRefreshedAt + } + if applyCapabilities != nil { + applyCapabilities(stored.ID, stored.URL, stored.Capabilities, *stored.CapabilitiesHash, + refreshedAt, stored.CapabilityDrift, stored.CapabilityDriftBaseline) + } + // The planning cache is per replica, so the winner's invalidation did not + // reach this one. A capability change is exactly what it exists to hear. + if onChanged != nil && !sameOptionalHash(n.CapabilitiesHash, stored.CapabilitiesHash) { + onChanged(stored.URL) + } +} + +// sameOptionalHash compares two stored capability hashes, treating absence as a +// value rather than as a match. +func sameOptionalHash(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + // applyCapabilitiesFor returns the pool writer for a node's type, or nil when // this checker has no pool for it — which is the normal state for a node row // that is disabled and therefore in no pool. @@ -460,6 +501,16 @@ func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, apply // sweep on another replica cannot land an older report on top of a // newer one. if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline, n.CapabilitiesHash); err != nil { + if errors.Is(err, ErrCapabilitiesSuperseded) { + // Another replica stored a report first. Its answer is the + // current one, so this replica adopts the row rather than + // discarding and retrying: left alone it would keep comparing + // against a hash the row no longer has, lose the same write + // every sweep, and serve stale GPU identities and drift state + // until something unrelated reloaded the pools. + hc.adoptStoredCapabilities(ctx, n, applyCapabilities, onChanged) + return err + } if errors.Is(err, ErrNodeMoved) { // Not a failure to report loudly: the node was edited or // removed while this fetch ran, and the next sweep will fetch diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 7c1074236..01d1c27b8 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -504,14 +504,27 @@ func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom return fmt.Errorf("update node capabilities: %w", err) } if tag.RowsAffected() == 0 { - // The row is gone, it no longer addresses the worker this payload came - // from, or another replica has already stored a different report. All - // three mean the same thing to the caller: do not publish it. - return ErrNodeMoved + // Three ways to get here, and they do not mean the same thing. The row + // being gone or repointed is terminal for this payload; another replica + // having stored a different report is not — that replica's answer is the + // current one, and this one's in-memory copy is now behind it. Telling + // them apart costs one read and saves a replica sweeping forever against + // a hash the row no longer has. + current, err := r.GetByID(ctx, id) + if err != nil || current == nil || !sameStoredURL(current.URL, fetchedFrom) { + return ErrNodeMoved + } + return ErrCapabilitiesSuperseded } return nil } +// sameStoredURL compares a stored node URL with the one a payload was fetched +// from, ignoring the trailing slash the pools normalize and the column does not. +func sameStoredURL(stored, fetchedFrom string) bool { + return strings.TrimRight(stored, "/") == strings.TrimRight(fetchedFrom, "/") +} + // Sentinel errors. var ( ErrNodeNotFound = errors.New("stream node not found") @@ -523,4 +536,10 @@ var ( // edited to address a different worker. The result must be discarded rather // than published against whatever the row is now. ErrNodeMoved = errors.New("stream node no longer matches the fetched identity") + + // ErrCapabilitiesSuperseded reports that the row still addresses this + // worker but another writer stored a different report first. The payload is + // discarded; unlike ErrNodeMoved the caller has something to learn from the + // row, because the report now on it is newer than the one it started from. + ErrCapabilitiesSuperseded = errors.New("stream node capabilities were superseded by another writer") ) diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go index 993c3c0fb..d3922f026 100644 --- a/internal/nodepool/repository_capabilities_test.go +++ b/internal/nodepool/repository_capabilities_test.go @@ -318,8 +318,14 @@ func TestUpdateCapabilitiesRefusesAReportThatFollowsAStaleOne(t *testing.T) { // row before either write, so it must not overwrite what is there now. overtaken := repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"vaapi"}`), "sha256:overtaken", time.Now(), nil, nil, ptrString("sha256:first")) - if !errors.Is(overtaken, ErrNodeMoved) { - t.Fatalf("overtaken report error = %v, want ErrNodeMoved", overtaken) + // Superseded, not moved: the row still addresses this worker, and the + // caller has something to learn from it. The distinction is what stops the + // losing replica from sweeping against a stale hash forever. + if !errors.Is(overtaken, ErrCapabilitiesSuperseded) { + t.Fatalf("overtaken report error = %v, want ErrCapabilitiesSuperseded", overtaken) + } + if errors.Is(overtaken, ErrNodeMoved) { + t.Fatal("a superseded report reported as a moved node; the caller would discard rather than reconcile") } stored, err := repo.GetByID(ctx, node.ID) diff --git a/internal/playback/capabilityhash_test.go b/internal/playback/capabilityhash_test.go index 216eaa694..2617ec49e 100644 --- a/internal/playback/capabilityhash_test.go +++ b/internal/playback/capabilityhash_test.go @@ -3,6 +3,7 @@ package playback import ( "strings" "testing" + "time" "github.com/Silo-Server/silo-server/internal/tonemap" ) @@ -142,3 +143,25 @@ func TestComputeCapabilityHashOfEmptyReport(t *testing.T) { t.Fatal("empty report hashed the same as a populated one") } } + +// The ceiling on a node-advertised budget has to sit above what a real +// configuration asks for. Picked as a round five minutes it was already below a +// nine-device node's legitimate 311 seconds, so the API canceled that node's +// re-probe before its own deadline every time and its inventory never landed. +func TestNormalizeProbeRequestTimeoutAdmitsALargeButRealBudget(t *testing.T) { + nineDevices := tonemap.ProbeRequestTimeout(tonemap.BackendQSV, + "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131,"+ + "/dev/dri/renderD132,/dev/dri/renderD133,/dev/dri/renderD134,/dev/dri/renderD135,/dev/dri/renderD136") + + got := NormalizeProbeRequestTimeout(nineDevices.Milliseconds(), time.Minute) + if got != nineDevices { + t.Fatalf("normalized = %v, want the %v a nine-device node legitimately asks for", got, nineDevices) + } + + // It is still a ceiling: the value comes off the wire from a worker, and a + // caller holds a connection open for it. + absurd := 24 * time.Hour + if got := NormalizeProbeRequestTimeout(absurd.Milliseconds(), time.Minute); got != tonemap.MaxProbeRequestTimeout() { + t.Fatalf("normalized = %v for an absurd advertisement, want the %v ceiling", got, tonemap.MaxProbeRequestTimeout()) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 6fa485721..dcbcbd465 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -182,13 +182,17 @@ type HWAccelInfo struct { ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms,omitempty"` } -const ( - probeRequestMinTimeout = 5 * time.Second - probeRequestMaxTimeout = 5 * time.Minute -) +const probeRequestMinTimeout = 5 * time.Second // NormalizeProbeRequestTimeout bounds a node-advertised probe budget while // preserving the caller's established fallback for a missing advertisement. +// +// The ceiling exists because the value comes off the wire from a worker, and a +// caller holds a connection open for it. It is derived from the probe formula +// rather than picked, because a round number picked once was already below what +// a nine-device node legitimately asks for: the API then canceled that node's +// re-probe before its own deadline, every time, and its inventory never landed. +// A ceiling that binds a real configuration is indistinguishable from a bug. func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Duration { if millis <= 0 { return fallback @@ -196,8 +200,8 @@ func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Dur if millis < probeRequestMinTimeout.Milliseconds() { return probeRequestMinTimeout } - if millis > probeRequestMaxTimeout.Milliseconds() { - return probeRequestMaxTimeout + if ceiling := tonemap.MaxProbeRequestTimeout(); millis > ceiling.Milliseconds() { + return ceiling } return time.Duration(millis) * time.Millisecond } diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index c26cf376b..0d7dc172d 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -473,7 +473,14 @@ func TestNormalizeProbeRequestTimeout(t *testing.T) { {name: "negative uses caller fallback", millis: -1, fallback: 2 * time.Minute, want: 2 * time.Minute}, {name: "too small", millis: time.Second.Milliseconds(), fallback: 2 * time.Minute, want: 5 * time.Second}, {name: "advertised", millis: (137 * time.Second).Milliseconds(), fallback: 2 * time.Minute, want: 137 * time.Second}, - {name: "too large", millis: (10 * time.Minute).Milliseconds(), fallback: 2 * time.Minute, want: 5 * time.Minute}, + { + // The ceiling is derived from the probe formula rather than picked, + // so the assertion is too. + name: "too large", + millis: (24 * time.Hour).Milliseconds(), + fallback: 2 * time.Minute, + want: tonemap.MaxProbeRequestTimeout(), + }, } { t.Run(test.name, func(t *testing.T) { if got := NormalizeProbeRequestTimeout(test.millis, test.fallback); got != test.want { diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index c6013726a..2155ab2d2 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -292,6 +292,27 @@ func ProbeEndpointTimeout(hardwareBackend, hardwareDevice string) time.Duration return ProbeTotalTimeout(backend, hardwareDevice) + probeEndpointSlack } +// MaxProbedDevices is the device count the ceiling on an advertised probe +// budget is derived from. +// +// The matrix grows with the device set, which has no hard limit — an operator +// can list as many render devices as the host has. This is the largest set a +// caller will keep waiting for, chosen well above any real GPU node so that a +// legitimate configuration never meets it and the ceiling only ever bounds a +// worker advertising something absurd. +const MaxProbedDevices = 16 + +// MaxProbeRequestTimeout is the largest budget a node may advertise and be +// believed, derived from the same formula the node itself uses so the two +// cannot drift apart. +func MaxProbeRequestTimeout() time.Duration { + devices := make([]string, 0, MaxProbedDevices) + for i := range MaxProbedDevices { + devices = append(devices, defaultDRIRenderDevice+strconv.Itoa(i)) + } + return ProbeRequestTimeout(BackendQSV, strings.Join(devices, ",")) +} + // ProbeRequestTimeout gives a remote caller additional transport and response // margin beyond the server-side endpoint budget. func ProbeRequestTimeout(hardwareBackend, hardwareDevice string) time.Duration { diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 1575c5a7a..991deecca 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -408,14 +408,15 @@ func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} return done } playbackCfg := cfg.Playback + // Claimed here, before the goroutine exists. Warmup is a real smoke encode + // and the listener opens while it may still be running, so an admin re-probe + // arriving in a node's first seconds must not see an idle gate — and a claim + // taken inside the goroutine leaves exactly that window, since the scheduler + // makes no promise about when it runs. Held rather than requested: warmup is + // already committed by the time it could be refused. + s.gpu.holdWork() go func() { defer close(done) - // Warmup is a real smoke encode, and the listener opens while this - // goroutine may still be running — so an admin re-probe arriving in the - // first seconds of a node's life would otherwise see an idle gate and - // run its matrix beside it. Held rather than requested: warmup is - // already started and refusing it here would only skip it. - s.gpu.holdWork() defer s.gpu.endWork() if err := playback.WarmHardwareEncoder(ctx, playbackCfg.FFmpegPath, playbackCfg.HWAccel, playbackCfg.HWDevice); err != nil { slog.DebugContext(ctx, "transcode node hardware encoder warmup failed", "component", "transcodenode", "error", err) From c5aef36691f7356b0c62d8575f077da2c347a3bb Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:15:18 -0400 Subject: [PATCH 049/163] fix(nodes): agree on one cgroup for CPU, supersede VideoToolbox probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings, two of them regressions from the last two commits. A CFS quota on a shared ancestor and a tighter cpuset on the leaf are caps on different populations, and the previous commit made cgroupCPU return the quota level's usage while cpuStats went on replacing only the denominator with the cpuset size. The ancestor's usage counts every sibling under that quota; divided by this process's smaller private cpuset it pins the node at a hundred percent while Silo is idle. Which cap binds now decides which cgroup's usage is read, so the two always describe the same thing, and the decision moved into cgroupCPU because that is where it can be acted on. Clearing the VideoToolbox cache does not supersede a probe already in flight: that call stays registered under an unchanged key, so a rebuild joins it instead of starting cold, and its completion repopulates the map the operator asked to empty — publishing the verdict the re-probe was meant to discard. The invalidation generation now leads its cache key, the same mechanism the other hardware probes already use. This closes a gap in the VideoToolbox invalidation added during the main merge. The proxy re-probe took capabilityBuildMu and nothing else. That mutex does not cover a probe whose caller has gone: both singleflights run on background contexts, so an abandoned capability request releases it while ffmpeg is still encoding. It now refuses with 409 while either counter is non-zero, the same rule the transcode node applies — two smoke encodes contending for one card report working hardware as failed regardless of which kind of node they are on. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/cgroupcpu.go | 21 ++++++- internal/nodemetrics/cgrouppath_test.go | 63 +++++++++++++++++++++ internal/nodemetrics/sampler.go | 15 ++--- internal/playback/gpudetect.go | 19 ++++++- internal/playback/gpudetect_publish_test.go | 54 ++++++++++++++++++ internal/proxy/reprobe.go | 36 +++++++++++- internal/proxy/reprobe_test.go | 34 +++++++++++ internal/proxy/server.go | 3 + 8 files changed, 229 insertions(+), 16 deletions(-) diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index df8baa643..6032c6486 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -140,18 +140,33 @@ type cgroupCPUSample struct { // cgroupCPU returns this process's cgroup CPU reading for the given instant, // and the CPU budget in cores that reading must be normalized against (0 when -// the cgroup imposes no quota). -func (s *Sampler) cgroupCPU(now time.Time) (cgroupCPUSample, float64) { +// nothing caps it). +// +// pinned is the size of this process's cpuset, or 0 when it is not restricted +// to a subset. It is a second kind of cap and it belongs here rather than at the +// caller, because which constraint binds decides which cgroup's usage the +// reading has to come from — and the two answers are different populations. +func (s *Sampler) cgroupCPU(now time.Time, pinned int) (cgroupCPUSample, float64) { for _, paths := range s.cgroupCPUPaths { if _, err := readCgroupCPUUsage(paths); err != nil { continue } - binding, quota := effectiveCgroupCPUQuota(paths) // Usage comes from whichever level supplied the binding quota, not from // the leaf. They are one measurement: a quota shared with sibling // services throttles on their CPU time too, so dividing only this // process's by it reports ten percent for a group that is saturated and // being throttled — the same pairing error the memory path had. + binding, quota := effectiveCgroupCPUQuota(paths) + + // A cpuset applies to this cgroup, not to the ancestor that owns the + // quota. So when it is the tighter cap the measurement moves back down + // with it: the ancestor's usage counts siblings that do not share this + // cpuset, and dividing that by a smaller private budget pins the node at + // a hundred percent while Silo is idle. + if pinned > 0 && (quota <= 0 || float64(pinned) < quota) { + binding, quota = paths, float64(pinned) + } + usage, err := readCgroupCPUUsage(binding) if err != nil { continue diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go index 596233130..57c6ce82b 100644 --- a/internal/nodemetrics/cgrouppath_test.go +++ b/internal/nodemetrics/cgrouppath_test.go @@ -6,6 +6,7 @@ import ( "slices" "strings" "testing" + "time" ) // writeSelfCgroup lays down a /self/cgroup with the given body. @@ -400,3 +401,65 @@ func TestEffectiveCgroupCPUQuotaReturnsTheLevelThatBinds(t *testing.T) { t.Fatalf("binding usage = %q, want the leaf's own %q", binding.usage, want) } } + +// A quota on a shared ancestor and a tighter cpuset on the leaf are caps on +// different populations. Taking the ancestor's usage — which counts every +// sibling under that quota — and dividing it by this process's smaller private +// cpuset pins the node at a hundred percent while Silo is idle. +func TestCgroupCPUMovesUsageDownWhenTheCpusetBinds(t *testing.T) { + root := t.TempDir() + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + // The slice allows eight cores and has burned a lot of CPU across every + // service under it; this one has burned very little and may use two CPUs. + write(root, "cpu.max", "800000 100000\n") + write(root, "cpu.stat", "usage_usec 9000000\n") + write(leaf, "cpu.max", "max 100000\n") + write(leaf, "cpu.stat", "usage_usec 1000000\n") + + s := &Sampler{cgroupCPUPaths: []cgroupCPUPath{{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: filepath.Join(leaf, "cpu.max"), + }}} + + // No cpuset: the slice's quota binds, so its usage is the right numerator. + sample, quota := s.cgroupCPU(time.Now(), 0) + if quota != 8 { + t.Fatalf("quota = %v, want the slice's 8 cores", quota) + } + if want := int64(9_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want the slice's %d", sample.usageNS, want) + } + + // A two-CPU cpuset is tighter, and it applies to this cgroup — so the + // measurement moves back down with it. + sample, quota = s.cgroupCPU(time.Now(), 2) + if quota != 2 { + t.Fatalf("quota = %v, want the cpuset's 2", quota) + } + if want := int64(1_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want this cgroup's own %d, not the slice's", sample.usageNS, want) + } + + // A cpuset looser than the quota changes nothing. + sample, quota = s.cgroupCPU(time.Now(), 32) + if quota != 8 { + t.Fatalf("quota = %v, want the slice's 8 to still bind", quota) + } + if want := int64(9_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want the slice's %d", sample.usageNS, want) + } +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 5b93166e2..e5bd99689 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -282,15 +282,12 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { cores = runtime.NumCPU() } - sample, quota := s.cgroupCPU(now) - // A cpuset caps CPU without setting a quota, so a process pinned to two - // CPUs reads cpu.max = "max" and would otherwise be measured against the - // whole host. Whichever cap is tighter is the one it actually runs under. - if pinned := cgroupCPUSetCores(s.cgroupCPUSetPaths); pinned > 0 { - if quota <= 0 || float64(pinned) < quota { - quota = float64(pinned) - } - } + // A cpuset caps CPU without setting a quota, so a process pinned to two CPUs + // reads cpu.max = "max" and would otherwise be measured against the whole + // host. Whichever cap is tighter is the one it actually runs under, and + // cgroupCPU decides that, because the answer also settles which cgroup's + // usage the reading comes from. + sample, quota := s.cgroupCPU(now, cgroupCPUSetCores(s.cgroupCPUSetPaths)) if quota > 0 { cores = cgroupQuotaCores(quota, hostCores) } diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index dcbcbd465..061d7fe57 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -129,6 +129,11 @@ type DetectedBackend struct { // the process lifetime; successful fully-capable probes are cached forever. var videoToolboxProbeRetryDelay = time.Minute +// videoToolboxProbeStarted is a test seam: it fires inside a VideoToolbox probe +// flight so a test can order an invalidation against it by channel receipt +// rather than by sleeping. Production leaves it nil. +var videoToolboxProbeStarted func() + type videoToolboxProbeEntry struct { result hardwareProbeResult expiresAt time.Time // zero: cached for the process lifetime @@ -998,7 +1003,11 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard commandTimeout := hwProbeCommandTimeout retryDelay := videoToolboxProbeRetryDelay + started := videoToolboxProbeStarted go func() { + if started != nil { + started() + } probeCtx, cancel := context.WithTimeout(context.Background(), 4*commandTimeout+time.Second) defer cancel() result := probeFFmpegVideoToolboxContext(probeCtx, execPath, commandTimeout) @@ -1026,8 +1035,16 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard // invalidating a cached verdict when that spelling resolves to a replaced // executable. This matters for Homebrew upgrades that swap a symlink target // while Silo remains running. +// +// The invalidation generation leads it, for the same reason the other hardware +// probes bake it into theirs: clearing the cache alone does not supersede a +// probe that is already running. That call stays registered under its key, the +// rebuild joins it instead of starting a cold one, and its completion +// repopulates the map — so an operator's re-probe publishes the very verdict it +// was asked to discard. Moving the key leaves the old flight writing somewhere +// nobody will read. func videoToolboxProbeCacheKey(execPath string) string { - return execPath + "\x00" + ffmpegIdentityKey(execPath) + return strconv.FormatUint(hwProbeGeneration(), 10) + "\x00" + execPath + "\x00" + ffmpegIdentityKey(execPath) } // StartupRetryHWAccel returns the acceleration for the single retry after a diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 846161eb7..6568a867c 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -638,3 +638,57 @@ func awaitNoProbesInFlight(t *testing.T) { runtime.Gosched() } } + +// Clearing the VideoToolbox cache does not supersede a probe already running: +// that call stays registered under its key, so a rebuild joins it rather than +// starting a cold one, and its completion repopulates the map — the re-probe +// then publishes the verdict it was asked to discard. The generation in the key +// is what moves the rebuild onto a fresh flight. +func TestInvalidateHWProbeCacheSupersedesAnInFlightVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + + // The first flight parks inside the probe and stays there while the second + // call is made, so "did the second start its own flight" is decided by + // channel receipts rather than by how the two happen to be scheduled. + var starts atomic.Int32 + blocked := make(chan struct{}) + firstStarted := make(chan struct{}) + previous := videoToolboxProbeStarted + videoToolboxProbeStarted = func() { + if starts.Add(1) == 1 { + close(firstStarted) + <-blocked + } + } + t.Cleanup(func() { videoToolboxProbeStarted = previous }) + + var wg sync.WaitGroup + wg.Go(func() { + if result := cachedVideoToolboxProbe(ffmpeg.path); !result.available { + t.Errorf("in-flight probe failed: %s", result.reason) + } + }) + <-firstStarted + + InvalidateHWProbeCache() + + second := make(chan hardwareProbeResult, 1) + wg.Go(func() { second <- cachedVideoToolboxProbe(ffmpeg.path) }) + + // The second call must reach its own flight while the first is still parked. + deadline := time.Now().Add(5 * time.Second) + for starts.Load() < 2 { + if time.Now().After(deadline) { + t.Fatal("the re-probe joined the flight it was supposed to supersede") + } + runtime.Gosched() + } + + close(blocked) + if result := <-second; !result.available { + t.Fatalf("post-invalidation probe failed: %s", result.reason) + } + wg.Wait() +} diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go index ff21978cb..2b2e98fc8 100644 --- a/internal/proxy/reprobe.go +++ b/internal/proxy/reprobe.go @@ -2,6 +2,7 @@ package proxy import ( "encoding/json" + "fmt" "log/slog" "net/http" @@ -28,9 +29,11 @@ type reprobeCapabilitiesResponse struct { // of this handler for the full reasoning. A rebuild that does not finish keeps // the previously published hash. // -// Unlike the transcode node this does not refuse while busy. That guard is about -// encoder sessions: a proxy's jobs are remuxes and RPU strips, which hold no -// encoder slot for a probe's smoke encode to lose a race against. +// Unlike the transcode node this does not refuse while *jobs* are running. That +// guard is about encoder sessions: a proxy's jobs are remuxes and RPU strips, +// which hold no encoder slot for a probe's smoke encode to lose a race against. +// It does refuse while another probe is running, which is a different thing — +// see below. func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { // Held across the invalidation and the rebuild together: discarding the // verdicts and recomputing them has to be one step, or the scheduled @@ -39,6 +42,23 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques s.capabilityBuildMu.Lock() defer s.capabilityBuildMu.Unlock() + // The mutex is not enough on its own. A probe outlives its caller by + // design — both singleflights run on background contexts so a canceled + // request cannot kill work another request is waiting on — so a capability + // request abandoned mid-probe releases this mutex while ffmpeg is still + // encoding. Invalidating then starts a second matrix beside the first, and + // two smoke encodes contending for one card publish a hardware failure for + // hardware that is fine. That is the same false verdict the transcode node's + // gate exists to prevent, and it does not care which kind of node it is on. + if busy := s.probesInFlight(); busy > 0 { + slog.InfoContext(r.Context(), "proxy capability re-probe refused while probes are still running", + "component", "proxy", "probes_in_flight", busy) + http.Error(w, fmt.Sprintf( + "node is running %d hardware probe(s); a re-probe smoke-encodes on the GPU and two at once would report working hardware as failed. Retry shortly.", + busy), http.StatusConflict) + return + } + playback.InvalidateHWProbeCache() tonemap.InvalidateProbeCache() // The resource sampler retires nvidia-smi after repeated failure, and a @@ -68,3 +88,13 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques slog.WarnContext(r.Context(), "encode proxy re-probe result", "component", "proxy", "error", err) } } + +// probesInFlight counts the hardware and tone-map probes this process has +// claimed the encoder for. It is a method so a test can drive the refusal +// without reaching into either package's unexported singleflight. +func (s *Server) probesInFlight() int { + if s.countProbesInFlight != nil { + return s.countProbesInFlight() + } + return playback.HWProbesInFlight() + tonemap.ProbesInFlight() +} diff --git a/internal/proxy/reprobe_test.go b/internal/proxy/reprobe_test.go index c1a402b92..62f3fb77d 100644 --- a/internal/proxy/reprobe_test.go +++ b/internal/proxy/reprobe_test.go @@ -71,3 +71,37 @@ func TestProxyReprobeCapabilitiesRequiresBearer(t *testing.T) { t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) } } + +// A probe outlives its caller by design, so a capability request abandoned +// mid-probe releases capabilityBuildMu while ffmpeg is still encoding. A +// re-probe arriving then would start a second smoke-encode matrix beside the +// first, and two contending for one card publish a hardware failure for +// hardware that is fine — the same false verdict the transcode node's gate +// prevents, which does not care which kind of node it is on. +func TestProxyReprobeCapabilitiesRefusedWhileProbesAreRunning(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + server.storeCapabilityHash("sha256:previous") + server.countProbesInFlight = func() int { return 1 } + + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while a probe is still running", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } + + // And it is not a permanent refusal: once the detached probe lands, the + // re-probe an operator asked for goes through. + server.countProbesInFlight = func() int { return 0 } + recorder = httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d after the probes finished, want 200", recorder.Code) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index a37a1ac38..8fd98a4f9 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -73,6 +73,9 @@ type Server struct { // invalidation generation is what makes the re-probe honest — so without // this they would genuinely run at once. capabilityBuildMu sync.Mutex + // countProbesInFlight overrides the detached-probe count the re-probe route + // refuses on. Tests set it; production leaves it nil. + countProbesInFlight func() int } type remoteArtifactMissReporter interface { From 2e41c795faa4d3040971e1e80e0e513890dca7eb Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:30:31 -0400 Subject: [PATCH 050/163] fix(playback): publish macOS hardware, count its probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capable Mac was publishing resolved:"none" with no detected backends. The snapshot builder only walked backends on Linux, so the VideoToolbox probe that resolution has used since main landed #439 never ran on the path that hashes and stores a node's inventory. The API therefore recorded every Mac as software-only, planned software tone mapping for it, and gave the operator a re-probe with nothing to verify. Darwin now probes the same way resolution does, publishing a verified entry or an unverified one carrying its reason — and a probe cut short by the caller's deadline reports the walk incomplete rather than hashing a timeout as a hardware verdict. The VideoToolbox flight also went uncounted. It is rooted at Background like the other two singleflights, so it outlives its caller and keeps ffmpeg on the card after every caller has returned; a re-probe could see an idle encoder and start a new-generation probe beside it. It now raises the same counter, claimed before the goroutine exists. And the CPU quota walk kept the leaf when an ancestor published an identical quota. The two are not equivalent: the ancestor's budget is shared with siblings that can exhaust it, so it is the level whose usage describes what is being throttled. Silo at 0.2 cores beside a sibling at 1.8 under a shared two-core parent read ten percent while the parent was saturated. Ties now go outward. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/cgroupcpu.go | 8 +- internal/nodemetrics/cgrouppath_test.go | 39 +++++++++ internal/playback/gpudetect.go | 28 ++++++- internal/playback/gpudetect_publish_test.go | 91 +++++++++++++++++++++ 4 files changed, 164 insertions(+), 2 deletions(-) diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index 6032c6486..a893d40f0 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -213,7 +213,13 @@ func effectiveCgroupCPUQuota(paths cgroupCPUPath) (cgroupCPUPath, float64) { if err != nil || cores <= 0 { continue } - if tightest == 0 || cores < tightest { + // Ties go to the outer level, which the walk reaches later. Two cgroups + // publishing the same quota are not equivalent: the ancestor's is shared + // with siblings that can exhaust it, so it is the one whose usage + // describes what is being throttled. Silo at 0.2 cores beside a sibling + // at 1.8 under a shared two-core parent reads ten percent from the leaf + // while the parent is saturated. + if tightest == 0 || cores <= tightest { tightest, binding = cores, level } } diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go index 57c6ce82b..0a801c6d4 100644 --- a/internal/nodemetrics/cgrouppath_test.go +++ b/internal/nodemetrics/cgrouppath_test.go @@ -463,3 +463,42 @@ func TestCgroupCPUMovesUsageDownWhenTheCpusetBinds(t *testing.T) { t.Fatalf("usage = %d, want the slice's %d", sample.usageNS, want) } } + +// Two cgroups publishing the same quota are not equivalent. The ancestor's is +// shared with siblings that can exhaust it, so it is the level whose usage +// describes what is actually being throttled: Silo at 0.2 cores beside a +// sibling at 1.8 under a shared two-core parent reads ten percent from the leaf +// while the parent is saturated. +func TestEffectiveCgroupCPUQuotaPrefersTheOuterLevelOnATie(t *testing.T) { + root := t.TempDir() + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + write(leaf, "cpu.max", "200000 100000\n") + write(leaf, "cpu.stat", "usage_usec 200000\n") + write(root, "cpu.max", "200000 100000\n") + write(root, "cpu.stat", "usage_usec 2000000\n") + + binding, cores := effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 2 { + t.Fatalf("cores = %v, want 2 from either level", cores) + } + if want := filepath.Join(root, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the shared parent's %q", binding.usage, want) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 061d7fe57..957a8a49a 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -253,8 +253,27 @@ func DetectHWAccelWithFFmpegContextResult(ctx context.Context, hwAccel, ffmpegPa resolved := HWAccelNone var detected []DetectedBackend complete := true - if currentGOOS == linuxGOOS { + switch currentGOOS { + case linuxGOOS: resolved, detected, complete = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) + case darwinGOOS: + // macOS has no render devices to walk, but it does have hardware: the + // same VideoToolbox probe resolution uses. Without this a capable Mac + // publishes resolved:"none" with no detected backends, and the API + // stores that as its durable inventory — a software-only node, planned + // for software tone mapping, with an operator re-probe that cannot + // verify otherwise because there is nothing here to verify. + entry := DetectedBackend{Backend: transcodeHWVideoToolbox} + if ok, reason := ffmpegSupportsVideoToolboxContext(ctx, ffmpegPath); ok { + entry.Verified = true + resolved = transcodeHWVideoToolbox + } else { + entry.Reason = reason + // A probe cut short by the caller's deadline is not a verdict about + // the hardware, and must not be hashed as one. + complete = ctx.Err() == nil + } + detected = append(detected, entry) } if configured := strings.TrimSpace(hwAccel); configured != "" && configured != hwAccelAuto { resolved = configured @@ -999,6 +1018,12 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard } call := &videoToolboxProbeCall{done: make(chan struct{})} videoToolboxProbes.inFlight[cacheKey] = call + // Counted for the life of the smoke encode, not the life of the caller: the + // goroutine below is rooted at Background so an abandoned request cannot + // kill work another is waiting on, and it keeps ffmpeg on the card after + // every caller has returned. Raised here, before the goroutine exists, so + // the claim cannot be observed unraised by anyone this call returns to. + hwProbesInFlight.Add(1) videoToolboxProbes.Unlock() commandTimeout := hwProbeCommandTimeout @@ -1010,6 +1035,7 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard } probeCtx, cancel := context.WithTimeout(context.Background(), 4*commandTimeout+time.Second) defer cancel() + defer hwProbesInFlight.Add(-1) result := probeFFmpegVideoToolboxContext(probeCtx, execPath, commandTimeout) entry := videoToolboxProbeEntry{result: result} if !result.available || !result.hevcAvailable { diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index 6568a867c..b54aa5b10 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -692,3 +692,94 @@ func TestInvalidateHWProbeCacheSupersedesAnInFlightVideoToolboxProbe(t *testing. } wg.Wait() } + +// A capable Mac was publishing resolved:"none" with no detected backends: the +// snapshot builder only walked backends on Linux, so the VideoToolbox probe +// that resolution uses never ran here. The API stored that as the node's +// durable inventory — software-only, planned for software tone mapping, with an +// operator re-probe that could not verify otherwise. +func TestDetectHWAccelPublishesVideoToolboxOnDarwin(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWVideoToolbox { + t.Fatalf("Resolved = %q, want videotoolbox", info.Resolved) + } + if len(info.DetectedBackends) != 1 || info.DetectedBackends[0].Backend != transcodeHWVideoToolbox || + !info.DetectedBackends[0].Verified { + t.Fatalf("DetectedBackends = %+v, want a verified videotoolbox entry", info.DetectedBackends) + } +} + +// A Mac whose probe fails publishes the failure rather than silence, so an +// operator can see why — but a probe cut short by the caller's deadline is not +// a verdict about the hardware and must not be hashed as one. +func TestDetectHWAccelReportsAFailedVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) != 1 || info.DetectedBackends[0].Verified || + info.DetectedBackends[0].Reason == "" { + t.Fatalf("DetectedBackends = %+v, want an unverified entry carrying a reason", info.DetectedBackends) + } + + hung := writeFakeFFmpeg(t, fakeFFmpegProbe{hang: true}) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + t.Cleanup(cancel) + if _, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, hung.path, ""); !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete for a deadline inside the probe", err) + } +} + +// The VideoToolbox flight outlives its caller like the others, so it has to be +// counted like the others — otherwise a re-probe sees an idle encoder and a +// new-generation probe starts beside the one still running. +func TestHWProbesInFlightCountsADetachedVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + awaitNoProbesInFlight(t) + + release := make(chan struct{}) + var released, announced sync.Once + started := make(chan struct{}) + previous := videoToolboxProbeStarted + videoToolboxProbeStarted = func() { + announced.Do(func() { close(started) }) + <-release + } + t.Cleanup(func() { + videoToolboxProbeStarted = previous + released.Do(func() { close(release) }) + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + _ = cachedVideoToolboxProbeContext(ctx, ffmpeg.path) + }() + <-started + cancel() + <-done + + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d with a detached VideoToolbox probe running, want at least 1", got) + } + released.Do(func() { close(release) }) + awaitNoProbesInFlight(t) +} From 6d4dda399d9bd7fa84f2d8f2c583a743ecebd8ac Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:45:25 -0400 Subject: [PATCH 051/163] fix(nodes): poll the nodes page, prefer the outer cgroup on a memory tie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useAdminNodes set staleTime and nothing else. staleTime marks data old; it does not schedule anything — so the GPU, disk and health columns this PR adds froze at whatever they were when the page mounted and refreshed only on focus, reconnect, or a mutation. An operator watching a node saturate, a scratch volume fill, or a health check start failing would see none of it. It polls on the 30s health cadence now, gated on page activity like the library queues, so backgrounded admin tabs stop asking. The memory walk kept the leaf when an ancestor published an identical limit, which is the tie the CPU walk was just taught to break outward. The ancestor's budget is shared with siblings that can fill it, so it is the one whose usage says how much is left; reading the leaf shows headroom right up until the parent OOMs. The comparison is against the running choice rather than the host figure, so a cgroup limit that merely equals host RAM still reads as no limit at all — covered by its own test, since loosening the comparison is exactly how that would have broken. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/sampler_test.go | 69 ++++++++++++++++++++++++++++ internal/nodemetrics/system.go | 17 +++++-- web/src/hooks/queries/admin/nodes.ts | 19 ++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index bbbc83f37..01f720b95 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -988,3 +988,72 @@ func TestNewTestSamplerReadsNoRealCgroupPaths(t *testing.T) { } } } + +// Two cgroups publishing the same memory limit are not equivalent: the +// ancestor's is shared with siblings that can fill it, so it is the one whose +// usage says how much is left. Reading the leaf instead shows headroom right up +// until the parent OOMs — the mirror of the CPU quota tie-break. +func TestMemoryLimitPrefersTheOuterCgroupOnATie(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + const twoGiB = "2147483648\n" + stat := write("memory.stat", "inactive_file 0\n") + s.cgroupUsagePaths = []cgroupUsagePath{ + { + limit: write("leaf.max", twoGiB), usage: write("leaf.current", "536870912\n"), + stat: stat, inactiveFile: cgroupInactiveFileKeyV2, + }, + { + limit: write("slice.max", twoGiB), usage: write("slice.current", "1879048192\n"), + stat: stat, inactiveFile: cgroupInactiveFileKeyV2, + }, + } + + used, total := s.memoryStats() + if total != 2147483648 { + t.Fatalf("total = %d, want the 2 GiB both levels publish", total) + } + if want := int64(1879048192); used != want { + t.Fatalf("used = %d, want the %d charged to the shared parent, not this service's own", used, want) + } +} + +// A cgroup limit that merely equals the host's memory is no limit at all, and +// must not start reading cgroup usage just because the tie-break loosened. +func TestMemoryLimitEqualToHostRAMIsNotALimit(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 65536 kB\nMemAvailable: 1024 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + s.cgroupUsagePaths = []cgroupUsagePath{{ + limit: write("memory.max", "67108864\n"), // exactly MemTotal + usage: write("memory.current", "1048576\n"), + stat: write("memory.stat", "inactive_file 0\n"), inactiveFile: cgroupInactiveFileKeyV2, + }} + + used, total := s.memoryStats() + if total != 65536*1024 { + t.Fatalf("total = %d, want the host's MemTotal", total) + } + if want := int64(64512 * 1024); used != want { + t.Fatalf("used = %d, want the host's used figure %d rather than the cgroup working set", used, want) + } +} diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go index 26251fd76..6b231c590 100644 --- a/internal/nodemetrics/system.go +++ b/internal/nodemetrics/system.go @@ -212,15 +212,26 @@ func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { // the level it came from is kept, because the usage that fills it is // everything charged to that cgroup, not just this process's own. var binding *cgroupUsagePath + hostTotal := totalBytes for i, level := range s.cgroupUsagePaths { limit, err := ReadCgroupMemoryLimit(level.limit) if err != nil || limit <= 0 { continue } - // A limit above the host's memory is not a limit worth reporting; it - // would only make a node look like it has headroom the kernel cannot + // A limit at or above the host's memory is not a limit worth reporting; + // it would only make a node look like it has headroom the kernel cannot // give it. - if totalBytes == 0 || limit < totalBytes { + if hostTotal > 0 && limit >= hostTotal { + continue + } + // Ties go to the outer level, which the list reaches later. Two cgroups + // publishing the same limit are not equivalent: the ancestor's is shared + // with siblings that can fill it, so it is the one whose usage says how + // much of it is left. Reading the leaf instead shows headroom right up + // until the parent OOMs. Compared against the running choice rather than + // against the host figure, so a cgroup limit that merely equals host RAM + // still reads as no limit at all. + if binding == nil || limit <= totalBytes { totalBytes = limit binding = &s.cgroupUsagePaths[i] } diff --git a/web/src/hooks/queries/admin/nodes.ts b/web/src/hooks/queries/admin/nodes.ts index 523a45e4a..036952c73 100644 --- a/web/src/hooks/queries/admin/nodes.ts +++ b/web/src/hooks/queries/admin/nodes.ts @@ -8,16 +8,35 @@ import type { ReprobeNodeResult, } from "@/api/types"; import { adminKeys } from "../keys"; +import { usePageActivity } from "@/hooks/usePageActivity"; import { describeReprobeOutcome } from "@/pages/adminNodesPresentation"; import { toast } from "sonner"; const ADMIN_STALE_TIME = 30_000; +/** + * Polled on the node health cadence, because this row now carries live + * readings rather than configuration. + * + * `staleTime` alone marks data old; it does not schedule anything. Without an + * interval the GPU, disk and health columns froze at whatever they were when + * the page mounted, refreshing only on focus, reconnect or a mutation — so an + * operator watching a node saturate, a scratch volume fill, or a health check + * start failing would see none of it. The server persists a fresh sample every + * 30 seconds, so asking more often only costs requests. + * + * Gated on page activity: a backgrounded or frozen tab has nobody reading it, + * and polling every admin tab a browser has open is how a small deployment + * ends up serving its own dashboard. + */ export function useAdminNodes() { + const pageActivity = usePageActivity(); + return useQuery({ queryKey: adminKeys.nodes(), queryFn: () => api("/admin/nodes").then((d) => d ?? []), staleTime: ADMIN_STALE_TIME, + refetchInterval: pageActivity.canApplyRealtimeUpdates ? ADMIN_STALE_TIME : false, }); } From 935016eb267db9108790873a392cfe71bc942bbb Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:02:03 -0400 Subject: [PATCH 052/163] fix(playback): keep a node's learned probe budget across invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budget a node advertises was stored inside its capability cache entry, so RefreshNodeCapabilitiesV3 deleted it along with the inventory. The background refresh that same invalidation launches then sized itself from the 120s fallback — short of the ~136s a two-device node legitimately asks for, and short precisely when the matrix is coldest, because an acceleration change is what made it cold. Planning lost its inventory at the one moment it had just been told the old one was wrong. The two describe different things and are invalidated for different reasons: the inventory is about the node's hardware, the budget is about how long that node takes to answer, which an acceleration change does not alter. They now live in separate maps under the same lock, and the budget is recorded even when the report it came with is discarded as overtaken — what a node says its read costs is true regardless of whether that particular answer is still current. Not done by leaving a stub entry behind: a zero-value entry with a nil error reads as a usable inventory to the stale-snapshot branch in lookupRemoteCapabilitiesV3, which would have handed planning a node with no transformations and no tone-map capabilities at all. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/playback.go | 7 ++++ internal/api/handlers/playback_v3.go | 38 ++++++++++++++--- .../playback_v3_capability_retry_test.go | 42 +++++++++++++++++++ .../api/handlers/playback_v3_union_test.go | 4 +- 4 files changed, 83 insertions(+), 8 deletions(-) diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index b0cd6984c..5c5b7b2ad 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -253,6 +253,13 @@ type PlaybackHandler struct { v3ToneMapProbe func(context.Context, string, string, string) (tonemap.Capabilities, error) v3NodeCapabilitiesMu sync.Mutex v3NodeCapabilities map[string]v3NodeCapabilityCache + // v3NodeProbeBudgets holds what each node last said a capability read of it + // costs, guarded by v3NodeCapabilitiesMu. It is kept apart from the + // inventory above because the two are invalidated for different reasons: an + // acceleration change makes the inventory wrong and the next read slow, + // while how long that node takes to answer is unchanged. See + // remoteToneMapProbeTimeoutV3. + v3NodeProbeBudgets map[string]time.Duration // v3NodeCapabilityInvalidations counts invalidations per node URL, guarded // by v3NodeCapabilitiesMu. A probe that started before the count moved // describes hardware the health sweep has already reported as changed, so diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index dd7b59b99..dd723ff16 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -81,7 +81,6 @@ type v3NodeCapabilityCache struct { toneMapCapabilities tonemap.Capabilities err error expiresAt time.Time - probeRequestTimeout time.Duration } type preparedTransportV3 struct { @@ -306,16 +305,39 @@ func (h *PlaybackHandler) localToneMapProbeTimeoutV3() time.Duration { return tonemap.ProbeEndpointTimeout(cfg.HWAccel, cfg.HWDevice) } +// remoteToneMapProbeTimeoutV3 returns how long to allow one capability read of +// a node, from the budget that node last advertised. +// +// The budget is kept apart from the inventory because it describes the node +// rather than its hardware, and the two are invalidated for different reasons. +// An acceleration change makes the inventory wrong and the matrix cold — which +// is exactly when the next read is slowest — while how long that node takes to +// answer has not changed. Storing them together meant an invalidation dropped +// the budget with the inventory and the refresh it triggered fell back to two +// minutes, short of the ~136 seconds a two-device node legitimately asks for, +// so protocol-v3 planning lost its inventory precisely after an invalidation. func (h *PlaybackHandler) remoteToneMapProbeTimeoutV3(nodeURL string) time.Duration { h.v3NodeCapabilitiesMu.Lock() - entry := h.v3NodeCapabilities[nodeURL] + budget := h.v3NodeProbeBudgets[nodeURL] h.v3NodeCapabilitiesMu.Unlock() - if entry.probeRequestTimeout > 0 { - return entry.probeRequestTimeout + if budget > 0 { + return budget } return remoteNodeProbeFallbackTimeout } +// rememberNodeProbeBudgetV3 records what a node says its capability read costs. +// Callers must hold v3NodeCapabilitiesMu. +func (h *PlaybackHandler) rememberNodeProbeBudgetLockedV3(nodeURL string, budget time.Duration) { + if budget <= 0 { + return + } + if h.v3NodeProbeBudgets == nil { + h.v3NodeProbeBudgets = make(map[string]time.Duration) + } + h.v3NodeProbeBudgets[nodeURL] = budget +} + func (h *PlaybackHandler) toneMapPlanningTimeoutV3(localFallbackAllowed bool) time.Duration { if localFallbackAllowed { return v3NodeCapabilityPlanTimeout @@ -418,7 +440,7 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR h.v3NodeCapabilitiesMu.Unlock() return current, nil } - h.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{err: err, expiresAt: completedAt.Add(v3NodeCapabilityErrorTTL), probeRequestTimeout: entry.probeRequestTimeout} + h.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{err: err, expiresAt: completedAt.Add(v3NodeCapabilityErrorTTL)} h.v3NodeCapabilitiesMu.Unlock() return v3NodeCapabilityCache{}, err } @@ -426,9 +448,13 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR transformations: append([]playback.TransformationV3(nil), info.Transformations...), toneMapCapabilities: append(tonemap.Capabilities(nil), info.ToneMapCapabilities...), expiresAt: completedAt.Add(v3NodeCapabilityTTL), - probeRequestTimeout: playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout), } h.v3NodeCapabilitiesMu.Lock() + // Recorded even when the entry below is discarded as overtaken: what the + // node says its read costs is true regardless of whether this particular + // answer is still current. + h.rememberNodeProbeBudgetLockedV3(nodeURL, + playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout)) if overtaken { // Still overtaken after the retry. Hand the result to this caller, which // has nothing better, but leave the cache empty so the next lookup diff --git a/internal/api/handlers/playback_v3_capability_retry_test.go b/internal/api/handlers/playback_v3_capability_retry_test.go index fb8dfe76f..14b244ff2 100644 --- a/internal/api/handlers/playback_v3_capability_retry_test.go +++ b/internal/api/handlers/playback_v3_capability_retry_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "sync/atomic" "testing" + "time" ) // A capability lookup overtaken by an invalidation was handing the caller the @@ -74,3 +75,44 @@ func TestLookupRemoteCapabilitiesFetchesOnceWhenNothingInvalidates(t *testing.T) t.Fatalf("fetches = %d, want a single read on the uncontended path", got) } } + +// An acceleration change makes a node's inventory wrong and its next capability +// matrix cold — which is exactly when the read is slowest. Dropping the learned +// budget along with the inventory sent the refresh that invalidation triggers +// back to the 120s fallback, short of what a two-device node legitimately asks +// for, so planning lost its inventory precisely after the invalidation. +func TestRefreshNodeCapabilitiesKeepsTheLearnedProbeBudget(t *testing.T) { + handler := NewPlaybackHandler(nil) + + advertised := 136_000 + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "resolved": "qsv", + "probe_request_timeout_ms": advertised, + }) + })) + t.Cleanup(node.Close) + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), node.URL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + learned := handler.remoteToneMapProbeTimeoutV3(node.URL) + if want := 136 * time.Second; learned != want { + t.Fatalf("learned budget = %v, want the advertised %v", learned, want) + } + + handler.RefreshNodeCapabilitiesV3(node.URL) + + if got := handler.remoteToneMapProbeTimeoutV3(node.URL); got != learned { + t.Fatalf("budget after invalidation = %v, want the learned %v — the refresh it triggers is sized from this", + got, learned) + } + // The inventory itself is still discarded; only the budget survives. + handler.v3NodeCapabilitiesMu.Lock() + _, cached := handler.v3NodeCapabilities[node.URL] + handler.v3NodeCapabilitiesMu.Unlock() + if cached { + t.Fatal("invalidation left the inventory cached") + } +} diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go index 850441145..2e1259a5d 100644 --- a/internal/api/handlers/playback_v3_union_test.go +++ b/internal/api/handlers/playback_v3_union_test.go @@ -782,9 +782,9 @@ func TestLookupRemoteCapabilitiesV3PreservesConcurrentFreshSuccessOnRefetchFailu handler.JWTSecret = "test-secret" handler.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) handler.v3NodeCapabilities[remote.URL] = v3NodeCapabilityCache{ - expiresAt: time.Now().Add(-time.Second), - probeRequestTimeout: time.Second, + expiresAt: time.Now().Add(-time.Second), } + handler.v3NodeProbeBudgets = map[string]time.Duration{remote.URL: time.Second} type lookupResult struct { entry v3NodeCapabilityCache err error From 7397af2f06f9dd1de98c948236dec09a6039d5f5 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:07:14 -0400 Subject: [PATCH 053/163] test(playback): set the fresh manifest's mtime instead of inheriting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go check failed on TestTranscodeThrottlerIgnoresOutputFromAnEarlierGeneration, which this branch does not touch — throttle.go, transcode_manager.go and transcode_manifest_test.go are all unchanged against main. It is a latent flake the branch happened to surface. Staleness is ManifestModTime.Before(GenerationStartedAt), and the test set GenerationStartedAt to time.Now() and then relied on a later write landing after it. On a filesystem that truncates mtime to the second — which a CI runner's overlay does and a developer's APFS does not — the write made at `now` is stored below `now`, so the manifest this generation just produced reads as older than the generation itself and the throttler never pauses. Confirmed by simulating the truncation locally: a second-truncated mtime is Before(now) every time the wall clock is not exactly on a second. The fresh manifest's mtime is now set explicitly, the same way the stale one already was, so the assertion no longer depends on filesystem granularity. Co-Authored-By: Claude Opus 5 (1M context) --- internal/playback/transcode_manifest_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/playback/transcode_manifest_test.go b/internal/playback/transcode_manifest_test.go index 574976472..8d948ec03 100644 --- a/internal/playback/transcode_manifest_test.go +++ b/internal/playback/transcode_manifest_test.go @@ -1164,7 +1164,17 @@ func TestTranscodeThrottlerIgnoresOutputFromAnEarlierGeneration(t *testing.T) { } // Once this generation writes its own manifest, throttling works normally. + // + // The mtime is set rather than inherited from the write. Staleness is + // decided by ManifestModTime.Before(GenerationStartedAt), and a filesystem + // with coarse mtime granularity — which a CI runner's overlay can have and a + // developer's APFS does not — truncates a write made at `now` back below it, + // so the fresh manifest reads as older than the generation that produced it. + fresh := now.Add(time.Second) writeManifestRange(t, tempDir, 225, 293, ".ts") + if err := os.Chtimes(filepath.Join(tempDir, "stream.m3u8"), fresh, fresh); err != nil { + t.Fatalf("chtimes manifest: %v", err) + } throttler.CheckOnce() if !throttler.paused { t.Fatal("expected throttler to pause on this generation's own produced head") From 4b71f13d7ceae406375bd6d5d4d0c1afc9431edc Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:14:16 -0400 Subject: [PATCH 054/163] fix(nodes): count probes at claim time, cap the CPU budget at the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit beginReprobe took the other-work count as a value, evaluated at the call site before the gate lock was acquired. A request descheduled in that window could read zero, and a capability build that started and was abandoned meanwhile would leave its background probe running while the gate still believed the encoder idle — so the re-probe claimed it and ran a second smoke matrix beside the first. It now takes a callback and reads the count under the lock that grants the claim, which is the only instant the answer is binding. Separately, cgroupQuotaCores caps the reported core count at the host's but the normalization budget was left uncapped, so a quota larger than the machine — a 128-core quota on a 64-core host, which is not a limit at all because it cannot be spent — divided a fully saturated node's usage by a number it could never reach. Sixty-four cores pegged reported fifty percent. The budget is capped the same way the core count already was, so the percentage and its denominator describe one machine. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/sampler.go | 9 +++++ internal/nodemetrics/sampler_test.go | 56 ++++++++++++++++++++++++++ internal/transcodenode/gpugate.go | 20 +++++---- internal/transcodenode/gpugate_test.go | 55 ++++++++++++++++++++----- internal/transcodenode/reprobe.go | 4 +- internal/transcodenode/reprobe_test.go | 12 +++--- 6 files changed, 131 insertions(+), 25 deletions(-) diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index e5bd99689..95c14221b 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -294,7 +294,16 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { if !sample.valid { return busyPct, cores } + // The budget is capped at what the host can actually give, the same way the + // reported core count is. A quota above the machine's core count is not a + // limit — a 128-core quota on a 64-core host cannot be spent — so dividing + // by it reports a workload saturating every CPU it has as fifty percent + // busy. cores has already been through that cap; budget has to agree with + // it or the percentage and the denominator describe different machines. budget := quota + if hostCores > 0 && budget > float64(hostCores) { + budget = float64(hostCores) + } if budget <= 0 { budget = float64(cores) } diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 01f720b95..608719c4d 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -1057,3 +1057,59 @@ func TestMemoryLimitEqualToHostRAMIsNotALimit(t *testing.T) { t.Fatalf("used = %d, want the host's used figure %d rather than the cgroup working set", used, want) } } + +// A quota above the machine's core count is not a limit — a 128-core quota on a +// 64-core host cannot be spent. cgroupQuotaCores already caps the reported core +// count at the host's; the normalization budget has to agree with it, or a +// workload saturating every CPU it has reports fifty percent busy. +func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // A four-core host. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 4 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 9900)) + + dir := t.TempDir() + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + // Eight cores of quota on a four-core host. + if err := os.WriteFile(quota, []byte("800000 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.sample(context.Background()) + + // 20 seconds of CPU over 5 seconds of wall time: every one of the four CPUs + // the host has, saturated. + writeUsage(20_000_000) + tree.write("stat", hostStat(200, 19800)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 4 { + t.Fatalf("Cores = %d, want the host's 4 — an 8-core quota is not a limit here", system.Cores) + } + if system.CPUPct != 100 { + t.Fatalf("CPUPct = %d, want 100 — the node has spent every CPU it can", system.CPUPct) + } +} diff --git a/internal/transcodenode/gpugate.go b/internal/transcodenode/gpugate.go index 2f1370aa0..371f05cb1 100644 --- a/internal/transcodenode/gpugate.go +++ b/internal/transcodenode/gpugate.go @@ -77,22 +77,28 @@ func (g *gpuGate) endWork() { // beginReprobe claims the encoder exclusively, or reports the work in progress // that stopped it. // -// activeJobs is the node's own running-session count and detachedProbes is -// playback.HWProbesInFlight. Both are passed in rather than read here so the -// gate stays a lock over its own state, and so the count that refuses a caller -// is the same one reported back to it. +// otherWork counts everything the gate does not track itself: the node's own +// running sessions, and the probes still running for callers that have gone +// away. It is a function rather than a value because it is read here, under the +// lock that grants the claim. Sampling it at the call site leaves a window — the +// request can be descheduled between reading zero and acquiring the lock, and a +// capability build that starts and is abandoned in that window leaves its +// background probe running while the count the gate sees still says idle. // -// detachedProbes is the piece that is not this node's own bookkeeping. A +// The detached probes are the piece that is not this node's own bookkeeping. A // hardware probe runs on a background context so an abandoned caller cannot // kill work another request is waiting on, which means the capability build // releases its gate claim while ffmpeg may still be encoding. Without counting // those, a re-probe claims an encoder that is not free and its smoke matrix // races the one already running — publishing the false hardware verdict this // gate exists to prevent, by the same mechanism, one layer down. -func (g *gpuGate) beginReprobe(activeJobs, detachedProbes int) (busy int, ok bool) { +func (g *gpuGate) beginReprobe(otherWork func() int) (busy int, ok bool) { g.mu.Lock() defer g.mu.Unlock() - busy = g.workers + activeJobs + detachedProbes + busy = g.workers + if otherWork != nil { + busy += otherWork() + } if g.reprobing || busy > 0 { return busy, false } diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go index cb0198b1e..b31e14a9e 100644 --- a/internal/transcodenode/gpugate_test.go +++ b/internal/transcodenode/gpugate_test.go @@ -15,14 +15,14 @@ func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { } // activeJobs is still 0 here: the counter only moves once ffmpeg is running, // which is exactly the window a point-in-time check missed. - if busy, ok := gate.beginReprobe(0, 0); ok { + if busy, ok := gate.beginReprobe(otherWork(0)); ok { t.Fatal("re-probe admitted while a transcode was starting") } else if busy != 1 { t.Fatalf("busy = %d, want the admitted work counted", busy) } gate.endWork() - if _, ok := gate.beginReprobe(0, 0); !ok { + if _, ok := gate.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused after the work finished") } } @@ -32,7 +32,7 @@ func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { var gate gpuGate - busy, ok := gate.beginReprobe(2, 0) + busy, ok := gate.beginReprobe(otherWork(2)) if ok { t.Fatal("re-probe admitted on a node running transcodes") } @@ -47,13 +47,13 @@ func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { func TestGPUGateRefusesWorkWhileReprobing(t *testing.T) { var gate gpuGate - if _, ok := gate.beginReprobe(0, 0); !ok { + if _, ok := gate.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused on an idle gate") } if gate.beginWork() { t.Fatal("GPU work admitted while a re-probe held the encoder") } - if _, ok := gate.beginReprobe(0, 0); ok { + if _, ok := gate.beginReprobe(otherWork(0)); ok { t.Fatal("a second concurrent re-probe was admitted") } @@ -74,7 +74,7 @@ func TestGPUGateEndWorkDoesNotUnderflow(t *testing.T) { if !gate.beginWork() { t.Fatal("beginWork refused after unbalanced releases") } - if _, ok := gate.beginReprobe(0, 0); ok { + if _, ok := gate.beginReprobe(otherWork(0)); ok { t.Fatal("re-probe admitted while one unit of work was outstanding") } } @@ -87,14 +87,14 @@ func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { var gate gpuGate gate.holdWork() - if busy, ok := gate.beginReprobe(0, 0); ok { + if busy, ok := gate.beginReprobe(otherWork(0)); ok { t.Fatal("re-probe admitted while a session was still closing") } else if busy != 1 { t.Fatalf("busy = %d, want the closing session counted", busy) } gate.endWork() - if _, ok := gate.beginReprobe(0, 0); !ok { + if _, ok := gate.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused after the teardown finished") } @@ -113,7 +113,7 @@ func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { var gate gpuGate - busy, ok := gate.beginReprobe(0, 1) + busy, ok := gate.beginReprobe(otherWork(1)) if ok { t.Fatal("re-probe admitted while a detached smoke encode was still running") } @@ -121,7 +121,7 @@ func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { t.Fatalf("busy = %d, want the detached probe counted", busy) } - if _, ok := gate.beginReprobe(0, 0); !ok { + if _, ok := gate.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused once no probe was in flight") } } @@ -132,7 +132,7 @@ func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { func TestGPUGateCountsEveryDetachedProbeSource(t *testing.T) { var gate gpuGate - busy, ok := gate.beginReprobe(0, 2) + busy, ok := gate.beginReprobe(otherWork(2)) if ok { t.Fatal("re-probe admitted while detached smoke encodes were still running") } @@ -140,3 +140,36 @@ func TestGPUGateCountsEveryDetachedProbeSource(t *testing.T) { t.Fatalf("busy = %d, want both detached probe sources counted", busy) } } + +// otherWork builds the callback beginReprobe reads under its own lock, for +// tests that want a fixed count. +func otherWork(count int) func() int { return func() int { return count } } + +// The count has to be read under the lock that grants the claim, not sampled +// before it. A request descheduled between reading zero and acquiring the lock +// would otherwise claim an encoder that a capability build has since started +// and abandoned, leaving that build's background probe running. +func TestGPUGateReadsOtherWorkUnderTheClaimLock(t *testing.T) { + var gate gpuGate + + // Whatever this reports at the moment of the claim is what decides it: a + // probe that starts between the caller's own check and the lock is seen. + appeared := false + busy, ok := gate.beginReprobe(func() int { + appeared = true + return 1 + }) + if !appeared { + t.Fatal("beginReprobe did not consult the count while holding the lock") + } + if ok { + t.Fatal("re-probe admitted despite work appearing at claim time") + } + if busy != 1 { + t.Fatalf("busy = %d, want the work counted at claim time", busy) + } + + if _, ok := gate.beginReprobe(nil); !ok { + t.Fatal("re-probe refused with nothing else to count") + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index 61700fe63..6ed3fbeb2 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -60,7 +60,9 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques // collide with the smoke encode minutes into the probe. The gate is the // same exclusion the transcode-start path consults, so from here until the // deferred release no new GPU work is admitted. - busy, ok := s.gpu.beginReprobe(int(s.activeJobs.Load()), playback.HWProbesInFlight()+tonemap.ProbesInFlight()) + busy, ok := s.gpu.beginReprobe(func() int { + return int(s.activeJobs.Load()) + playback.HWProbesInFlight() + tonemap.ProbesInFlight() + }) if !ok { slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", "component", "transcodenode", "active_jobs", busy) diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go index 35106a3dc..fe72c1b65 100644 --- a/internal/transcodenode/reprobe_test.go +++ b/internal/transcodenode/reprobe_test.go @@ -160,7 +160,7 @@ func TestReprobeCapabilitiesRefusesWhileWorkIsStarting(t *testing.T) { // and the API retries on another node. func TestTranscodeStartRefusedWhileReprobing(t *testing.T) { server := newTestServer(t) - if _, ok := server.gpu.beginReprobe(0, 0); !ok { + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused on an idle node") } t.Cleanup(server.gpu.endReprobe) @@ -289,7 +289,7 @@ func TestCapabilitySnapshotRegistersAsGPUWork(t *testing.T) { case <-time.After(30 * time.Second): t.Fatal("the capability snapshot was never admitted as GPU work") } - if _, ok := server.gpu.beginReprobe(0, 0); ok { + if _, ok := server.gpu.beginReprobe(otherWork(0)); ok { server.gpu.endReprobe() t.Fatal("a re-probe was admitted while a capability snapshot held the encoder") } @@ -307,7 +307,7 @@ func TestCapabilitySnapshotRegistersAsGPUWork(t *testing.T) { func TestCapabilitySnapshotRefusedWhileReprobing(t *testing.T) { server := newTestServer(t) server.storeCapabilityHash("sha256:previous") - if _, ok := server.gpu.beginReprobe(0, 0); !ok { + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused on an idle node") } t.Cleanup(server.gpu.endReprobe) @@ -339,7 +339,7 @@ func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { // Observed from inside the teardown, which is the only instant that matters. err := server.retireGPUSession(func() error { jobsDuringClose = server.activeJobs.Load() - busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(int(jobsDuringClose), 0) + busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(otherWork(int(jobsDuringClose))) if reprobeAdmitted { server.gpu.endReprobe() } @@ -360,7 +360,7 @@ func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { } // The hold is released with the teardown, so an idle node re-probes again. - if _, ok := server.gpu.beginReprobe(int(server.activeJobs.Load()), 0); !ok { + if _, ok := server.gpu.beginReprobe(otherWork(int(server.activeJobs.Load()))); !ok { t.Fatal("re-probe refused after the teardown completed") } server.gpu.endReprobe() @@ -390,7 +390,7 @@ func TestReprobeCapabilitiesRefusedWhileEncoderWarmupRuns(t *testing.T) { } server.gpu.endWork() - if _, ok := server.gpu.beginReprobe(0, 0); !ok { + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { t.Fatal("re-probe refused after warmup finished") } server.gpu.endReprobe() From e4d4db1867e358a5a5d3db7378f3bacb801b0632 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:32:13 -0400 Subject: [PATCH 055/163] fix(playback): scale the hardware walk, cap the probed device set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more bounds that were guesses where the thing they bound is a function of the configuration. hwAccelWalkTimeout was a fixed thirty seconds while the walk probes every configured render device for both QSV and VAAPI. Three Intel devices need more than that, so the walk marked itself incomplete with every individual command still inside its own budget, and /hw-capabilities answered 503 for a node that was working. It is now derived from the matrix the walk will actually run. The advertised-budget ceiling assumes a largest device set (MaxProbedDevices), but the worker probed every entry it was given. A node configured with more than that advertised a budget its callers then clamped below what it needed, so its cold capability requests could never finish. Rather than raise a ceiling that has no natural value, the probe set is capped at the same number, which makes the two ends agree — and the truncation is logged once rather than folded silently into a shorter answer. Not changed: the dispatch window during a policy handoff, reported against nodes.go:224. Draining the node for the duration would take a transcode node out of rotation on every ordinary settings save, which on a single-node deployment is a guaranteed outage in place of a rare transient failure. The real fix remains sending backend and device from one snapshot, which is a change to the node start contract and needs client coordination. Co-Authored-By: Claude Opus 5 (1M context) --- internal/playback/gpudetect.go | 38 +++++++++++++++++++++++++---- internal/playback/gpudetect_test.go | 36 +++++++++++++++++++++++++++ internal/tonemap/probe.go | 21 ++++++++++++++++ internal/tonemap/probe_test.go | 34 ++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 5 deletions(-) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 957a8a49a..b8404157e 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -356,10 +356,38 @@ const hwAccelAuto = "auto" // probe passes wins. var hwAccelPreferenceOrder = []string{transcodeHWNVENC, transcodeHWQSV, transcodeHWVAAPI} -// hwAccelWalkTimeout bounds one full backend walk regardless of how many -// candidate devices a host exposes, so a wedged driver cannot stretch detection -// without limit. tonemap.probeEndpointSlack budgets a capability request for it. -const hwAccelWalkTimeout = 30 * time.Second +// hwAccelWalkSlack is the margin the walk deadline carries above the commands +// it may actually run, covering process spawn and the bookkeeping between them. +const hwAccelWalkSlack = 5 * time.Second + +// hwAccelWalkTimeout bounds one full backend walk, so a wedged driver cannot +// stretch detection without limit. tonemap.probeEndpointSlack budgets a +// capability request for it. +// +// It is derived from the matrix the walk will actually run rather than fixed, +// because that matrix grows with the device set: every configured render device +// is probed for both QSV and VAAPI, so three Intel devices legitimately need +// more than the thirty seconds a fixed bound allowed. The walk then marked +// itself incomplete while every individual command was still inside its own +// budget, and /hw-capabilities answered 503 for a node that was working — the +// same failure shape as a bound that is a guess rather than a derivation. +func hwAccelWalkTimeout(candidates hwCandidates) time.Duration { + commands := 0 + for _, backend := range hwAccelPreferenceOrder { + if !candidates.presentFor(backend) { + continue + } + probe, ok := hwBackendProbeFor(backend) + if !ok { + continue + } + commands += probe.commandCount * len(candidates.probeDevicesFor(backend)) + } + if commands == 0 { + commands = 1 + } + return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack +} // hwCandidates groups the candidate render devices by the backend each one can // plausibly drive, before any FFmpeg verification. @@ -485,7 +513,7 @@ func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCa if ctx == nil { ctx = context.Background() } - ctx, cancel := context.WithTimeout(ctx, hwAccelWalkTimeout) + ctx, cancel := context.WithTimeout(ctx, hwAccelWalkTimeout(candidates)) defer cancel() complete = true for _, backend := range hwAccelPreferenceOrder { diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 0d7dc172d..c0c404f12 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -1331,3 +1331,39 @@ func TestCachedVideoToolboxProbeInvalidatesWhenExecutableIsReplaced(t *testing.T t.Fatal("replacement binary reused the previous executable's positive verdict") } } + +// The walk deadline has to cover the matrix it will actually run, which grows +// with the device set: every configured render device is probed for both QSV +// and VAAPI. A fixed thirty seconds marked a three-device host incomplete while +// every individual command was still inside its own budget, and +// /hw-capabilities then answered 503 for a node that was working. +func TestHWAccelWalkTimeoutScalesWithTheDeviceSet(t *testing.T) { + one := hwCandidates{ + intel: []string{"/dev/dri/renderD128"}, + vaapi: []string{"/dev/dri/renderD128"}, + } + three := hwCandidates{ + intel: []string{"/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/dri/renderD130"}, + vaapi: []string{"/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/dri/renderD130"}, + } + + oneDevice, threeDevices := hwAccelWalkTimeout(one), hwAccelWalkTimeout(three) + if threeDevices <= oneDevice { + t.Fatalf("three-device walk %v is not longer than the one-device %v", threeDevices, oneDevice) + } + + // It covers what it will run: every command the matrix allows, at its own + // per-command bound, plus spawn slack. + wantThree := time.Duration(3*(3+2))*hwProbeCommandTimeout + hwAccelWalkSlack + if threeDevices != wantThree { + t.Fatalf("three-device walk = %v, want %v", threeDevices, wantThree) + } + if threeDevices <= 30*time.Second { + t.Fatalf("three-device walk = %v, which the old fixed 30s bound would have cut short", threeDevices) + } + + // A host with nothing to probe still gets a usable, non-zero deadline. + if got := hwAccelWalkTimeout(hwCandidates{}); got <= 0 { + t.Fatalf("empty walk timeout = %v, want a positive bound", got) + } +} diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index 2155ab2d2..d2e5aca14 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "errors" + "log/slog" "os" "os/exec" "slices" @@ -400,9 +401,29 @@ func probeDevices(value, backend string) []string { } return []string{defaultDRIRenderDevice} } + if len(devices) > MaxProbedDevices { + // Past the cap the budget every caller allows stops covering the matrix, + // so probing further would guarantee the request is canceled rather + // than finished. Truncating is the honest failure: the devices that are + // probed get real verdicts, and the omission is logged rather than + // silently folded into a shorter answer. + noteProbeDevicesTruncated(len(devices)) + devices = devices[:MaxProbedDevices] + } return devices } +// probeDevicesTruncatedLogged latches the truncation warning to one line per +// process: a device list is a standing configuration, not an event. +var probeDevicesTruncatedLogged sync.Once + +func noteProbeDevicesTruncated(configured int) { + probeDevicesTruncatedLogged.Do(func() { + slog.Warn("tone-map probe covers only the first configured devices; the rest are not verified", + "component", "tonemap", "configured", configured, "probed", MaxProbedDevices) + }) +} + // intersectSourceKinds preserves the left-hand ordering while retaining source // kinds supported by both sets. func intersectSourceKinds(left, right []SourceKind) []SourceKind { diff --git a/internal/tonemap/probe_test.go b/internal/tonemap/probe_test.go index 3a48e5615..ca677f131 100644 --- a/internal/tonemap/probe_test.go +++ b/internal/tonemap/probe_test.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "runtime" + "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -470,3 +472,35 @@ func awaitNoProbesInFlight(t *testing.T) { runtime.Gosched() } } + +// The advertised budget every caller clamps to assumes a largest device set. If +// the worker probed past it, it would advertise a budget those callers then cut +// below what it actually needs, and cold capability requests would be canceled +// short forever. The probe set is capped so the two ends agree. +func TestProbeDevicesCapsTheConfiguredSet(t *testing.T) { + configured := make([]string, 0, MaxProbedDevices+4) + for i := range MaxProbedDevices + 4 { + configured = append(configured, defaultDRIRenderDevice+strconv.Itoa(i)) + } + + got := probeDevices(strings.Join(configured, ","), BackendQSV) + if len(got) != MaxProbedDevices { + t.Fatalf("probed %d devices, want the %d cap", len(got), MaxProbedDevices) + } + if !slices.Equal(got, configured[:MaxProbedDevices]) { + t.Fatalf("probed %v, want the first %d configured", got, MaxProbedDevices) + } + + // The budget a node advertises for that capped set is therefore never above + // what its callers allow — which is the property the cap exists for. + if advertised, ceiling := ProbeRequestTimeout(BackendQSV, strings.Join(configured, ",")), + MaxProbeRequestTimeout(); advertised > ceiling { + t.Fatalf("advertised %v exceeds the %v callers allow", advertised, ceiling) + } + + // A set inside the cap is untouched. + small := configured[:3] + if got := probeDevices(strings.Join(small, ","), BackendQSV); !slices.Equal(got, small) { + t.Fatalf("probed %v, want the configured %v unchanged", got, small) + } +} From 9af0c2f87750698d465777d78e1849729354da6f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:01:09 -0400 Subject: [PATCH 056/163] fix(playback): compose the capability budget, judge drift by its baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probeEndpointSlack was a constant standing in for the hardware walk, with a comment saying it "cannot be derived — playback imports this package, not the other way round — so it is raised whenever either budget grows". The previous commit made the walk scale with the device set and did not raise it, which is exactly the failure the comment predicted: three Intel devices now need more than the endpoint budget allows, so an operator re-probe returns 503 with no probe having failed. Rather than raise it again, the composition moved to where both halves are in scope. playback.CapabilityEndpointTimeout adds the walk to tonemap's matrix, CapabilityRequestTimeout adds the transport margin, and MaxCapabilityRequestTimeout is the ceiling advertisements are clamped to; tonemap's slack now covers only what tonemap owns. Every caller that bounds a whole capability read — the node's own endpoints, the API's remote lookup, the download preparer, the health fetch budget — uses the composed value. localToneMapProbeTimeoutV3 was under-budgeted the same way: it resolves the backend first, which on Linux is a full walk. Separately, drift clearing required every backend on the node to verify before it would even consult the baseline. A node carrying a backend that has never worked — VAAPI failing beside a working QSV is an ordinary mixed host — could therefore never clear a note about a lost render device, however completely that device came back. Which backends had to return is the baseline's question and recoveredBy already answers it precisely; the gate in front now only rules out a report with nothing probed at all. A legacy note with no baseline still needs a wholly clean report, since it names nothing to check against. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 3 +- cmd/silo/main_test.go | 2 +- internal/api/handlers/playback_v3.go | 5 +- internal/api/handlers/playback_v3_test.go | 4 +- internal/downloads/remote_preparer.go | 6 ++- internal/downloads/remote_preparer_test.go | 6 +-- .../remote_transcode_reconstruct_test.go | 2 +- internal/nodepool/health.go | 31 ++++++++++- internal/nodepool/health_drift_test.go | 54 +++++++++++++++++++ internal/playback/capabilityhash_test.go | 6 +-- internal/playback/gpudetect.go | 33 +++++++++++- internal/playback/gpudetect_test.go | 2 +- internal/proxy/server.go | 3 +- internal/tonemap/probe.go | 5 ++ internal/transcodenode/server.go | 6 +-- internal/transcodenode/server_test.go | 9 +++- 16 files changed, 152 insertions(+), 25 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index bc09167f0..eacbe9e9b 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -110,7 +110,6 @@ import ( "github.com/Silo-Server/silo-server/internal/taskmanager/tasks" "github.com/Silo-Server/silo-server/internal/taskmanager/triggers" "github.com/Silo-Server/silo-server/internal/telemetry" - "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodenode" "github.com/Silo-Server/silo-server/internal/usercollections" "github.com/Silo-Server/silo-server/internal/userdb" @@ -276,7 +275,7 @@ func nodeCapabilityProbeBudget(live func() *config.Config) func(*nodepool.Node) hwDevice = *node.HWDeviceOverride } } - return max(nodeCapabilityRequestTimeout, tonemap.ProbeRequestTimeout(hwAccel, hwDevice)) + return max(nodeCapabilityRequestTimeout, playback.CapabilityRequestTimeout(hwAccel, hwDevice)) } } diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index 8e037b347..04070dea4 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -435,7 +435,7 @@ func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { oneDevice := nodeCapabilityProbeBudget(func() *config.Config { return single })(clusterNode) twoDevices := nodeCapabilityProbeBudget(func() *config.Config { return pair })(clusterNode) - if want := tonemap.ProbeRequestTimeout("qsv", pair.Playback.HWDevice); twoDevices != want { + if want := playback.CapabilityRequestTimeout("qsv", pair.Playback.HWDevice); twoDevices != want { t.Fatalf("two-device budget = %v, want the node's own advertised %v", twoDevices, want) } if twoDevices <= nodeCapabilityRequestTimeout { diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index dd723ff16..bd298c095 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -302,7 +302,10 @@ func (h *PlaybackHandler) localToneMapCapabilitiesForTransportV3(ctx context.Con func (h *PlaybackHandler) localToneMapProbeTimeoutV3() time.Duration { cfg := h.playbackConfig() - return tonemap.ProbeEndpointTimeout(cfg.HWAccel, cfg.HWDevice) + // The whole read, not its tone-map half: localToneMapCapabilitiesV3 resolves + // the backend first, and on Linux that is a full hardware walk whose cost + // scales with the configured device set. + return playback.CapabilityEndpointTimeout(cfg.HWAccel, cfg.HWDevice) } // remoteToneMapProbeTimeoutV3 returns how long to allow one capability read of diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index c5d04a8f1..eaf95c84c 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -5398,7 +5398,7 @@ func TestPlaybackV3ToneMapBudgetsCoverColdNodeWork(t *testing.T) { return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} } - if got, want := handler.localToneMapProbeTimeoutV3(), tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { + if got, want := handler.localToneMapProbeTimeoutV3(), playback.CapabilityEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { t.Fatalf("local tone-map probe timeout = %s, want %s", got, want) } if got, want := handler.remoteToneMapProbeTimeoutV3("https://unknown.example"), remoteNodeProbeFallbackTimeout; got != want { @@ -5544,7 +5544,7 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { // Still bounded — the value comes off the wire from a worker — but at the // ceiling the probe formula produces rather than a round number that a real // nine-device node already exceeds. - if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), tonemap.MaxProbeRequestTimeout(); got != want { + if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), playback.MaxCapabilityRequestTimeout(); got != want { t.Fatalf("bounded remote probe timeout = %s, want %s", got, want) } } diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index 0a5fd0d57..6f5a18d30 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -495,9 +495,11 @@ func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Durat } cfg := p.config() if cfg == nil { - return tonemap.ProbeRequestTimeout("", "") + return playback.CapabilityRequestTimeout("", "") } - return tonemap.ProbeRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice) + // The whole capability read, not just its tone-map half: the node runs a + // hardware walk first, and that walk scales with the configured device set. + return playback.CapabilityRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice) } // cacheToneMapCapabilityFailure negatively caches an unreachable or invalid diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index 2595dd3de..88483dc7a 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -759,7 +759,7 @@ func TestNodeAwarePreparerUsesTargetNodeProbeBudget(t *testing.T) { preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) preparer.probeClient = &http.Client{Transport: transport} - if got, want := preparer.remoteToneMapProbeTimeout(remote.URL), tonemap.ProbeRequestTimeout(tonemap.BackendQSV, "/central/device"); got != want { + if got, want := preparer.remoteToneMapProbeTimeout(remote.URL), playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/central/device"); got != want { t.Fatalf("unknown-node probe timeout = %s, want configured probe budget %s", got, want) } if _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL); err != nil { @@ -814,7 +814,7 @@ func TestNormalizeRemoteToneMapProbeTimeout(t *testing.T) { // nine-device node legitimately advertises. name: "too large", millis: (24 * time.Hour).Milliseconds(), - want: tonemap.MaxProbeRequestTimeout(), + want: playback.MaxCapabilityRequestTimeout(), }, } { t.Run(test.name, func(t *testing.T) { @@ -843,7 +843,7 @@ func TestNodeAwarePreparerCapabilityFailurePreservesNodeProbeBudget(t *testing.T func TestNodeAwarePreparerDerivesProbeBudgetWithoutCachedAdvertisement(t *testing.T) { preparer := NewNodeAwarePreparer(nil, nil, nil) - if got, want := preparer.remoteToneMapProbeTimeout("https://node.example"), tonemap.ProbeRequestTimeout("", ""); got != want { + if got, want := preparer.remoteToneMapProbeTimeout("https://node.example"), playback.CapabilityRequestTimeout("", ""); got != want { t.Fatalf("uncached probe timeout = %s, want derived budget %s", got, want) } } diff --git a/internal/jellycompat/remote_transcode_reconstruct_test.go b/internal/jellycompat/remote_transcode_reconstruct_test.go index 003c91a51..200d10593 100644 --- a/internal/jellycompat/remote_transcode_reconstruct_test.go +++ b/internal/jellycompat/remote_transcode_reconstruct_test.go @@ -1063,7 +1063,7 @@ func TestRemoteTranscodeStartTimeoutCoversColdProbePreflightAndReadiness(t *test // Still bounded — the advertisement comes off the wire from a worker — but // at the ceiling the probe formula produces, not a round number a real // nine-device node already exceeds. - maxWant := tonemap.MaxProbeRequestTimeout() + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout + maxWant := playback.MaxCapabilityRequestTimeout() + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout if got := handler.remoteTranscodeStartTimeout(request, (24 * time.Hour).Milliseconds()); got != maxWant { t.Fatalf("bounded remote transcode start timeout = %v, want %v", got, maxWant) } diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index e3c4c212d..4b4e7f2fb 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -769,10 +769,15 @@ func resolveDriftNote(stored *string, storedBaseline []byte, drift capabilityDri if stored == nil || strings.TrimSpace(*stored) == "" { return nil, nil } - if !parsed || !hardwareProbesClean(payload) { + if !parsed || !hardwareProbesEvidenced(payload) { // Nothing new was lost, but this report is not evidence of recovery. return stored, marshalDriftBaseline(outstanding) } + if outstanding.empty() && !hardwareProbesClean(payload) { + // A note written before baselines existed names nothing to wait for, so + // a wholly clean report is the only evidence available for it. + return stored, marshalDriftBaseline(outstanding) + } if !outstanding.recoveredBy(payload) { // Recovery is the *originally lost* hardware coming back, not the // inventory merely looking healthy. A surviving sibling probes just as @@ -920,6 +925,30 @@ func hardwareProbesClean(payload []byte) bool { return probed } +// hardwareProbesEvidenced reports whether this report contains probe evidence at +// all: at least one backend that was actually probed rather than skipped. +// +// It is the weaker of the two, and it is what a note with a baseline is judged +// against. Requiring every backend to verify latches a note forever on a node +// that carries one which has never worked — VAAPI failing beside a working QSV +// is an ordinary mixed host — even after the render device the note is about +// comes back and the backend that used it verifies again. Which backends had to +// return is the baseline's question, and recoveredBy answers it precisely; this +// only rules out the empty report, where a loop over no backends finds no +// failure and would otherwise read as recovery. +func hardwareProbesEvidenced(payload []byte) bool { + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false + } + for _, backend := range current.DetectedBackends { + if !backend.Skipped { + return true + } + } + return false +} + // computeCapabilityDrift compares the report a node just served against the one // it replaces. parsed is false when either payload cannot be read, in which case // the drift is unknown rather than empty and callers must not treat it as diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 4f13ed575..5edda1488 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -684,3 +684,57 @@ func TestResolveDriftNoteClearsWhenTheSameCardReturnsByUUID(t *testing.T) { t.Fatalf("capability_drift = %q, want the same uuid in a new slot to clear it", *got) } } + +// A mixed host can carry a backend that has never worked — VAAPI failing beside +// a working QSV is ordinary. Requiring every backend to verify before clearing +// meant a note about a lost render device latched forever, long after that +// device came back and the backend that used it verified again. +func TestResolveDriftNoteClearsDespiteAnUnrelatedFailingBackend(t *testing.T) { + const recovered = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true},{"backend":"vaapi","verified":false,"reason":"no driver"}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"backends":["qsv"],"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(recovered) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want it cleared: the lost device is back and qsv verifies", *got) + } +} + +// The baseline still decides. A backend the note is waiting on that has not come +// back keeps it latched, however healthy the rest of the report looks. +func TestResolveDriftNoteKeepsNoteWhenTheBaselineBackendIsStillFailing(t *testing.T) { + const stillBroken = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":false,"reason":"no driver"},{"backend":"vaapi","verified":true}]}` + + standing := "backends no longer verifying: qsv" + baseline := []byte(`{"backends":["qsv"]}`) + payload := []byte(stillBroken) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared while the backend it names is still failing") + } +} + +// A report with nothing probed is not evidence of anything. Every backend +// skipped means the node could not open its configured devices, which says +// nothing about the hardware the note is waiting on. +func TestResolveDriftNoteKeepsNoteWhenEveryBackendWasSkipped(t *testing.T) { + const allSkipped = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"qsv","skipped":true},{"backend":"vaapi","skipped":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"aliases":["/dev/dri/renderD128"]}]}`) + payload := []byte(allSkipped) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got == nil { + t.Fatal("capability_drift cleared on a report where nothing was probed") + } +} diff --git a/internal/playback/capabilityhash_test.go b/internal/playback/capabilityhash_test.go index 2617ec49e..5a9c1a7fa 100644 --- a/internal/playback/capabilityhash_test.go +++ b/internal/playback/capabilityhash_test.go @@ -149,7 +149,7 @@ func TestComputeCapabilityHashOfEmptyReport(t *testing.T) { // nine-device node's legitimate 311 seconds, so the API canceled that node's // re-probe before its own deadline every time and its inventory never landed. func TestNormalizeProbeRequestTimeoutAdmitsALargeButRealBudget(t *testing.T) { - nineDevices := tonemap.ProbeRequestTimeout(tonemap.BackendQSV, + nineDevices := CapabilityRequestTimeout(tonemap.BackendQSV, "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131,"+ "/dev/dri/renderD132,/dev/dri/renderD133,/dev/dri/renderD134,/dev/dri/renderD135,/dev/dri/renderD136") @@ -161,7 +161,7 @@ func TestNormalizeProbeRequestTimeoutAdmitsALargeButRealBudget(t *testing.T) { // It is still a ceiling: the value comes off the wire from a worker, and a // caller holds a connection open for it. absurd := 24 * time.Hour - if got := NormalizeProbeRequestTimeout(absurd.Milliseconds(), time.Minute); got != tonemap.MaxProbeRequestTimeout() { - t.Fatalf("normalized = %v for an absurd advertisement, want the %v ceiling", got, tonemap.MaxProbeRequestTimeout()) + if got := NormalizeProbeRequestTimeout(absurd.Milliseconds(), time.Minute); got != MaxCapabilityRequestTimeout() { + t.Fatalf("normalized = %v for an absurd advertisement, want the %v ceiling", got, MaxCapabilityRequestTimeout()) } } diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index b8404157e..888aebab6 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -205,7 +205,7 @@ func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Dur if millis < probeRequestMinTimeout.Milliseconds() { return probeRequestMinTimeout } - if ceiling := tonemap.MaxProbeRequestTimeout(); millis > ceiling.Milliseconds() { + if ceiling := MaxCapabilityRequestTimeout(); millis > ceiling.Milliseconds() { return ceiling } return time.Duration(millis) * time.Millisecond @@ -389,6 +389,37 @@ func hwAccelWalkTimeout(candidates hwCandidates) time.Duration { return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack } +// CapabilityEndpointTimeout is how long one capability endpoint may take to +// answer: the hardware detection walk plus the tone-map matrix and the overhead +// around it. +// +// Both halves scale with the configured device set, and they live in different +// packages — tonemap cannot see the walk, because playback imports tonemap and +// not the reverse. Composing them here is what stops the two from drifting: a +// constant standing in for one of them inside the other has to be raised by +// hand whenever it grows, and the first time the walk grew, it was not. +func CapabilityEndpointTimeout(hwAccel, hwDevice string) time.Duration { + return hwAccelWalkTimeout(collectHWCandidates(hwDevice)) + + tonemap.ProbeEndpointTimeout(hwAccel, hwDevice) +} + +// CapabilityRequestTimeout is CapabilityEndpointTimeout plus the transport +// margin a remote caller needs. It is what a node advertises and what a caller +// of that node's capability endpoint must allow. +func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { + return CapabilityEndpointTimeout(hwAccel, hwDevice) + tonemap.ProbeRequestSlack +} + +// MaxCapabilityRequestTimeout is the largest budget a node may advertise and be +// believed, derived from the same composition at the device cap. +func MaxCapabilityRequestTimeout() time.Duration { + devices := make([]string, 0, tonemap.MaxProbedDevices) + for i := range tonemap.MaxProbedDevices { + devices = append(devices, defaultDRIDir+"/renderD"+strconv.Itoa(128+i)) + } + return CapabilityRequestTimeout(hwAccelAuto, strings.Join(devices, ",")) +} + // hwCandidates groups the candidate render devices by the backend each one can // plausibly drive, before any FFmpeg verification. type hwCandidates struct { diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index c0c404f12..07e9d0bd9 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -479,7 +479,7 @@ func TestNormalizeProbeRequestTimeout(t *testing.T) { name: "too large", millis: (24 * time.Hour).Milliseconds(), fallback: 2 * time.Minute, - want: tonemap.MaxProbeRequestTimeout(), + want: MaxCapabilityRequestTimeout(), }, } { t.Run(test.name, func(t *testing.T) { diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 8fd98a4f9..89b6539ec 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -29,7 +29,6 @@ import ( "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" - "github.com/Silo-Server/silo-server/internal/tonemap" ) // Server is the HTTP handler for proxy mode. @@ -279,7 +278,7 @@ func (s *Server) buildCapabilitySnapshotLocked(ctx context.Context) (playback.HW // whole rebuild inside the window a caller was told to allow, and only a // deadline the builder owns bounds the background snapshot, whose context // lives as long as the process. - ctx, cancel := context.WithTimeout(ctx, tonemap.ProbeEndpointTimeout(hwAccel, hwDevice)) + ctx, cancel := context.WithTimeout(ctx, playback.CapabilityEndpointTimeout(hwAccel, hwDevice)) defer cancel() // A walk that ran out of budget is refused rather than published: it marks // unprobed backends Verified=false, which is byte-identical to a real diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index d2e5aca14..5196ef397 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -293,6 +293,11 @@ func ProbeEndpointTimeout(hardwareBackend, hardwareDevice string) time.Duration return ProbeTotalTimeout(backend, hardwareDevice) + probeEndpointSlack } +// ProbeRequestSlack is the transport and response margin a remote caller adds +// on top of a node's endpoint budget. Exported so playback can compose the same +// request budget without duplicating the number. +const ProbeRequestSlack = probeRequestSlack + // MaxProbedDevices is the device count the ceiling on an advertised probe // budget is derived from. // diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 991deecca..cf98c0eb4 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -845,7 +845,7 @@ func resolveToneMapRecipe(ctx context.Context, opts *playback.TranscodeOpts) err if ctx == nil { ctx = context.Background() } - resolveCtx, cancel := context.WithTimeout(ctx, tonemap.ProbeEndpointTimeout(opts.HWAccel, opts.HWDevice)) + resolveCtx, cancel := context.WithTimeout(ctx, playback.CapabilityEndpointTimeout(opts.HWAccel, opts.HWDevice)) defer cancel() if err := resolveCtx.Err(); err != nil { return err @@ -1085,7 +1085,7 @@ func (s *Server) buildCapabilitySnapshotLocked(ctx context.Context) (playback.HW if err != nil { return playback.HWAccelInfo{}, err } - info.ProbeRequestTimeoutMillis = tonemap.ProbeRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() + info.ProbeRequestTimeoutMillis = playback.CapabilityRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), info.Resolved, hwDevice) if err != nil { return playback.HWAccelInfo{}, err @@ -1166,7 +1166,7 @@ func (s *Server) refreshCapabilitySnapshot(ctx context.Context) { } func toneMapCapabilityResolveTimeout(hardwareBackend, hardwareDevice string) time.Duration { - return tonemap.ProbeEndpointTimeout(hardwareBackend, hardwareDevice) + return playback.CapabilityEndpointTimeout(hardwareBackend, hardwareDevice) } func (s *Server) handleChapterThumbnailExtract(w http.ResponseWriter, r *http.Request) { diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go index 5238c5d0c..fe621bf19 100644 --- a/internal/transcodenode/server_test.go +++ b/internal/transcodenode/server_test.go @@ -380,10 +380,15 @@ func TestHandleHWCapabilitiesReturnsServiceUnavailableWhenDeadlineExpires(t *tes } func TestToneMapCapabilityResolveTimeoutCoversConfiguredProbeBudget(t *testing.T) { + // The endpoint budget is both halves — the hardware walk and the tone-map + // matrix — because the endpoint runs both. got := toneMapCapabilityResolveTimeout(tonemap.BackendQSV, "/dev/dri/renderD128") - if want := tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { + if want := playback.CapabilityEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { t.Fatalf("capability resolve timeout = %v, want endpoint budget %v", got, want) } + if tone := tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got <= tone { + t.Fatalf("capability resolve timeout = %v, want more than the tone-map half alone (%v)", got, tone) + } } func TestHandleHWCapabilitiesAdvertisesEffectiveProbeRequestTimeout(t *testing.T) { @@ -405,7 +410,7 @@ func TestHandleHWCapabilitiesAdvertisesEffectiveProbeRequestTimeout(t *testing.T if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil { t.Fatal(err) } - want := float64(tonemap.ProbeRequestTimeout(playback.HWAccelNone, "").Milliseconds()) + want := float64(playback.CapabilityRequestTimeout(playback.HWAccelNone, "").Milliseconds()) if got := response["probe_request_timeout_ms"]; got != want { t.Fatalf("probe request timeout = %v, want %.0fms", got, want) } From 800662d8962455ee43d2aa655170f146c42598f2 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:21:35 -0400 Subject: [PATCH 057/163] fix(nodes): cap the hardware walk, show the whole drift, flag unconfirmed reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings, each a place where two things that should say the same thing did not. The tone-map matrix was capped at MaxProbedDevices last round and the hardware walk was not, so a node with more than sixteen configured devices still probed all of them while every caller clamped its budget to sixteen — cancelling the capability endpoint before its walk could finish. The walk takes the same cap, with the same one-line log; the full device list is still reported, since the inventory is what an operator reads and truncating that would hide hardware that exists. The drift note was built from the latest delta while the baseline accumulated every loss. Two GPUs going one at a time therefore left the operator reading about the second while the note stayed latched — after that one returned — for a first loss nothing on screen ever mentioned. The note is now rendered from the baseline, so the text and the latch name the same hardware. The resolved-backend transition still comes from the delta, because it describes this refresh rather than a standing debt. And a capability report the node has already contradicted looked fresh. isCapabilityReportStale measured the health check, which keeps succeeding every thirty seconds while a refetch fails, so an inventory known to be obsolete showed without its marker indefinitely. Nodes now carry the hash the node advertised on its last check — an observation, derived per sweep rather than persisted — and a mismatch against the stored hash marks the report stale on its own, with a tooltip that says which of the two staleness reasons applies. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 1 + internal/nodepool/health.go | 60 ++++++++++++++++++-- internal/nodepool/health_drift_test.go | 47 +++++++++++++++ internal/nodepool/health_stats_test.go | 10 ++-- internal/nodepool/planner_test.go | 6 +- internal/nodepool/proxy_pool.go | 4 +- internal/nodepool/repository.go | 7 +++ internal/nodepool/transcode_pool.go | 7 ++- internal/playback/gpudetect.go | 24 +++++++- internal/playback/gpudetect_test.go | 40 +++++++++++++ web/src/api/types.ts | 6 ++ web/src/pages/AdminNodes.tsx | 8 ++- web/src/pages/adminNodesPresentation.test.ts | 45 +++++++++++++++ web/src/pages/adminNodesPresentation.ts | 9 +++ 14 files changed, 252 insertions(+), 22 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index f886a2d4d..7913b7e4a 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -28,6 +28,7 @@ Always `200 OK` with a JSON array. | `created_at` | RFC3339 string | When the node was registered. | | `capabilities` | object | The node's last stored capability report — the same body `GET /hw-capabilities` returns on the node. Omitted until one has been stored. | | `capabilities_hash` | string | Identity of that report, as computed by the node. Omitted with `capabilities`. | +| `advertised_capabilities_hash` | string | The hash the node named on its last health check. It differs from `capabilities_hash` while a refetch is outstanding or failing — the one case a recent `last_health_check` cannot rule out, since that check keeps succeeding while the refetch does not. Derived per sweep rather than stored, so it is absent until the first check after an API restart. | | `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | | `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 4b4e7f2fb..0cad72a2e 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -285,7 +285,7 @@ func (hc *HealthChecker) Start(ctx context.Context) { } // applyHealthFunc is a pool's copy-on-write health writer. -type applyHealthFunc func(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) +type applyHealthFunc func(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) // applyCapabilitiesFunc is a pool's copy-on-write capability writer. type applyCapabilitiesFunc func(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) @@ -300,7 +300,7 @@ func (hc *HealthChecker) checkAll(ctx context.Context) { // a Node struct mutated in place (the pool swaps in a copy). Fenced // on the checked URL, like the database write below: the pool can be // reloaded with a different worker on this id while the check runs. - applyHealth(n.ID, n.URL, healthy, activeJobs, egressKbps, lastStats, time.Now()) + applyHealth(n.ID, n.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, time.Now()) if n.Healthy && !healthy { slog.WarnContext(ctx, "stream node unhealthy", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) @@ -761,10 +761,14 @@ func truncateDriftNote(note string) string { // clears it. func resolveDriftNote(stored *string, storedBaseline []byte, drift capabilityDrift, parsed bool, payload []byte) (*string, []byte) { outstanding := mergeDriftBaseline(storedBaseline, drift) - if note := drift.persistedNote(); note != nil { - // A fresh loss extends whatever was already outstanding rather than - // replacing it: two GPUs going one at a time must both have to return. - return note, marshalDriftBaseline(outstanding) + if drift.regressed() { + // The note names everything still outstanding, not just this pass's + // delta. Both accumulate, and if only the baseline did, two GPUs going + // one at a time would leave the operator reading about the second while + // the note stayed latched — after that one returned — for a first loss + // nothing on screen ever mentioned. + note := outstanding.note(drift) + return ¬e, marshalDriftBaseline(outstanding) } if stored == nil || strings.TrimSpace(*stored) == "" { return nil, nil @@ -819,6 +823,50 @@ func (d driftBaselineDevice) matches(candidate renderDeviceAliases) bool { func (b driftBaseline) empty() bool { return len(b.Backends) == 0 && len(b.Devices) == 0 } +// note renders what this baseline is still waiting on, plus the resolution +// change this pass saw. +// +// The hardware half comes from the baseline so the visible text and the latch +// agree: an operator watching the note disappear should be watching the same +// thing the clearing rule is watching. The resolution transition comes from the +// delta, because "qsv -> none" describes this refresh rather than a standing +// debt, and the previous pass's transition is stale once another has happened. +func (b driftBaseline) note(drift capabilityDrift) string { + parts := make([]string, 0, 3) + if len(b.Backends) > 0 { + parts = append(parts, "verified hardware backends lost: "+strings.Join(b.Backends, ", ")) + } + if names := b.deviceNames(); len(names) > 0 { + parts = append(parts, "render devices gone: "+strings.Join(names, ", ")) + } + if drift.previousResolved != drift.resolved { + parts = append(parts, "resolved backend "+drift.previousResolved+" -> "+drift.resolved) + } + return truncateDriftNote(strings.Join(parts, "; ")) +} + +// deviceNames picks one readable name per outstanding device: the render path an +// operator would recognize, else the uuid, else whatever alias there is. +func (b driftBaseline) deviceNames() []string { + names := make([]string, 0, len(b.Devices)) + for _, device := range b.Devices { + name := device.UUID + for _, alias := range device.Aliases { + if strings.HasPrefix(alias, "/dev/") { + name = alias + break + } + if name == "" { + name = alias + } + } + if name != "" { + names = append(names, name) + } + } + return names +} + // recoveredBy reports whether every backend and device the note is waiting on is // accounted for in this report. func (b driftBaseline) recoveredBy(payload []byte) bool { diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index 5edda1488..f66b1184e 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -738,3 +738,50 @@ func TestResolveDriftNoteKeepsNoteWhenEveryBackendWasSkipped(t *testing.T) { t.Fatal("capability_drift cleared on a report where nothing was probed") } } + +// Two GPUs going one at a time: the note and the latch have to name the same +// thing. Built from the delta alone, the second loss replaced the first in the +// text while the baseline still waited for both — so after the visible GPU came +// back the warning stayed, naming hardware the operator could no longer see. +func TestResolveDriftNoteNamesEveryOutstandingLoss(t *testing.T) { + const both = `{"resolved":"qsv","render_devices":["/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + firstLoss := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + // A second card goes while the first is still missing. + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD129","/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"},` + + `{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + drift, parsed := computeCapabilityDrift([]byte(before), []byte(both)) + if !parsed { + t.Fatal("drift not parsed") + } + + note, baseline := resolveDriftNote(&standing, firstLoss, drift, parsed, []byte(both)) + if note == nil { + t.Fatal("capability_drift cleared while two cards are missing") + } + for _, want := range []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} { + if !strings.Contains(*note, want) { + t.Fatalf("capability_drift = %q, want it to name %s — the baseline is still waiting on it", *note, want) + } + } + + // And the note keeps agreeing with the latch: the first card returning is + // not enough, and what remains is still named. + const firstBack = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128","/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"},` + + `{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + drift, parsed = computeCapabilityDrift([]byte(firstBack), []byte(firstBack)) + still, _ := resolveDriftNote(note, baseline, drift, parsed, []byte(firstBack)) + if still == nil { + t.Fatal("capability_drift cleared with the second card still missing") + } + if !strings.Contains(*still, "/dev/dri/renderD129") { + t.Fatalf("capability_drift = %q, want the still-missing card named", *still) + } +} diff --git a/internal/nodepool/health_stats_test.go b/internal/nodepool/health_stats_test.go index b9822c401..e2dbed8c9 100644 --- a/internal/nodepool/health_stats_test.go +++ b/internal/nodepool/health_stats_test.go @@ -147,13 +147,13 @@ func TestApplyHealthClearsStatsWhenACheckCarriesNone(t *testing.T) { pool := NewTranscodePool() pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) - pool.ApplyHealth(1, "http://node", true, 1, 0, []byte(`{"system":{"cpu_pct":41}}`), time.Now()) + pool.ApplyHealth(1, "http://node", true, 1, 0, "", []byte(`{"system":{"cpu_pct":41}}`), time.Now()) stored := pool.Nodes()[0] if len(stored.LastStats) == 0 { t.Fatal("stats were not published to the pool") } - pool.ApplyHealth(1, "http://node", false, 0, 0, nil, time.Now()) + pool.ApplyHealth(1, "http://node", false, 0, 0, "", nil, time.Now()) if got := pool.Nodes()[0].LastStats; got != nil { t.Fatalf("LastStats = %s after a failed check, want nil", got) } @@ -166,7 +166,7 @@ func TestApplyHealthClonesStats(t *testing.T) { pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) buffer := []byte(`{"system":{"cpu_pct":41}}`) - pool.ApplyHealth(1, "http://node", true, 0, 0, buffer, time.Now()) + pool.ApplyHealth(1, "http://node", true, 0, 0, "", buffer, time.Now()) copy(buffer, []byte(`{"system":{"cpu_pct":99}}`)) if got := string(pool.Nodes()[0].LastStats); got != `{"system":{"cpu_pct":41}}` { @@ -183,7 +183,7 @@ func TestApplyHealthIgnoresAResultForAReplacedWorker(t *testing.T) { pool := NewTranscodePool() pool.SetNodes([]*Node{{ID: 1, URL: "http://replacement", Enabled: true}}) - pool.ApplyHealth(1, "http://original", true, 7, 0, []byte(`{"system":{"cpu_pct":41}}`), time.Now()) + pool.ApplyHealth(1, "http://original", true, 7, 0, "", []byte(`{"system":{"cpu_pct":41}}`), time.Now()) stored := pool.Nodes()[0] if stored.ActiveJobs != 0 || len(stored.LastStats) != 0 || stored.LastHealthCheck != nil { @@ -197,7 +197,7 @@ func TestApplyHealthAcceptsATrailingSlashDifference(t *testing.T) { pool := NewTranscodePool() pool.SetNodes([]*Node{{ID: 1, URL: "http://node/", Enabled: true}}) - pool.ApplyHealth(1, "http://node", true, 3, 0, nil, time.Now()) + pool.ApplyHealth(1, "http://node", true, 3, 0, "", nil, time.Now()) if stored := pool.Nodes()[0]; stored.ActiveJobs != 3 { t.Fatalf("active jobs = %d, want the result applied despite the trailing slash", stored.ActiveJobs) diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index 7874ed550..0a871bf02 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -559,7 +559,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // Unlike job reservations, bandwidth bridges ignore health freshness — // a report right after admission would not reflect the streams yet. newer := f.now.Add(5 * time.Second) - f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 0, nil, newer) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 0, "", nil, newer) f.now = f.now.Add(10 * time.Second) if got := f.planner.PlanSession("s4", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("stream should still be rejected during bridge window, got %+v", got) @@ -569,7 +569,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // meter now reports 8 Mbps, so one more 4 Mbps stream still won't fit, // but a 2 Mbps one will. f.now = f.now.Add(bandwidthBridgeAge) - f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 8_000, nil, f.now) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 8_000, "", nil, f.now) if got := f.planner.PlanSession("s5", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("4 Mbps stream should not fit at 8/10 Mbps, got %+v", got) } @@ -621,7 +621,7 @@ func TestUnknownBitrateAdmittedBelowCap(t *testing.T) { t.Fatal("unknown-bitrate stream should be admitted below cap") } - f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 10_000, nil, f.now) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 10_000, "", nil, f.now) if got := f.planner.PlanSession("s2", "", false, 0).ProxyNode; got != nil { t.Fatalf("unknown-bitrate stream should be rejected at cap, got %+v", got) } diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index 6556776b7..4b3d218a9 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -63,10 +63,10 @@ func (p *ProxyPool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *ProxyPool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { +func (p *ProxyPool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, lastStats, checkedAt) + applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, advertisedHash, lastStats, checkedAt) } // ApplyCapabilities records a freshly fetched capability report by swapping the diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 01d1c27b8..f3ceaf67b 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -73,6 +73,13 @@ type Node struct { // for. Non-nil exactly when CapabilityDrift is, except on a note written // before this column existed. CapabilityDriftBaseline json.RawMessage `json:"capability_drift_baseline,omitempty"` + // AdvertisedCapabilitiesHash is the hash the node named on its last health + // check, which is not always the one stored beside it: the sweep refetches + // on a mismatch, and a refetch that keeps failing leaves the two apart while + // the health check goes on succeeding every 30 seconds. Derived per sweep + // rather than persisted — it is an observation about right now, and a + // restarted API re-learns it on its first check. + AdvertisedCapabilitiesHash string `json:"advertised_capabilities_hash,omitempty"` // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from // Capabilities rather than stored: it is a pure function of that payload, so // a column would only be a second copy that can disagree with it. Two nodes diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 573ef44ed..96e8b7b1d 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -86,10 +86,10 @@ func (p *TranscodePool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *TranscodePool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { +func (p *TranscodePool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, lastStats, checkedAt) + applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, advertisedHash, lastStats, checkedAt) } // ApplyCapabilities records a freshly fetched capability report by swapping the @@ -114,7 +114,7 @@ func sameNodeURL(a, b string) bool { // by id alone would then write one worker's health — and the scratch fill // transcode admission reads — onto the replacement, and the database fence // downstream cannot undo that. The pool would stay wrong until a later sweep. -func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte, checkedAt time.Time) { +func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { for i, n := range nodes { if n.ID != id || !sameNodeURL(n.URL, checkedURL) { continue @@ -124,6 +124,7 @@ func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, act clone.ActiveJobs = activeJobs clone.EgressKbps = egressKbps clone.LastHealthCheck = &checkedAt + clone.AdvertisedCapabilitiesHash = advertisedHash // A check that carried no stats clears them rather than keeping the // previous sample: an unreachable node's five-minute-old CPU number // looks live on a dashboard, which is worse than no number at all. The diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 888aebab6..742d8a87a 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -420,6 +420,17 @@ func MaxCapabilityRequestTimeout() time.Duration { return CapabilityRequestTimeout(hwAccelAuto, strings.Join(devices, ",")) } +// hwProbeDevicesTruncatedLogged latches the truncation warning to one line per +// process: a device list is standing configuration, not an event. +var hwProbeDevicesTruncatedLogged sync.Once + +func noteHWProbeDevicesTruncated(configured int) { + hwProbeDevicesTruncatedLogged.Do(func() { + slog.Warn("hardware detection covers only the first configured devices; the rest are not verified", + "component", "playback", "configured", configured, "probed", tonemap.MaxProbedDevices) + }) +} + // hwCandidates groups the candidate render devices by the backend each one can // plausibly drive, before any FFmpeg verification. type hwCandidates struct { @@ -461,7 +472,18 @@ func collectHWCandidates(configuredDevice string) hwCandidates { probeDevices := configured.List() if len(probeDevices) == 0 { probeDevices = candidates.renderDevices - } else { + } + if len(probeDevices) > tonemap.MaxProbedDevices { + // The same ceiling the tone-map matrix is capped at, for the same + // reason: past it the walk costs more than the budget every caller + // allows, so probing further guarantees the capability request is + // canceled rather than finished. The devices that are probed get real + // verdicts; the rest are still reported in the inventory, and the + // omission is logged rather than folded silently into a shorter answer. + noteHWProbeDevicesTruncated(len(probeDevices)) + probeDevices = probeDevices[:tonemap.MaxProbedDevices] + } + if len(configured.List()) > 0 { // A configured device this process cannot open can never pass a smoke // encode, so it is classified for reporting but never probed. This is // the normal state of a proxy node reading the cluster-wide hw_device diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 07e9d0bd9..0cdc5d7dd 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "testing" @@ -1367,3 +1368,42 @@ func TestHWAccelWalkTimeoutScalesWithTheDeviceSet(t *testing.T) { t.Fatalf("empty walk timeout = %v, want a positive bound", got) } } + +// The walk is capped at the same device ceiling the tone-map matrix is, because +// the budget every caller allows is derived from that ceiling. Probing past it +// would guarantee the capability request is canceled before the walk finishes. +func TestCollectHWCandidatesCapsTheProbedDeviceSet(t *testing.T) { + env := setupHWAccelTest(t) + configured := make([]string, 0, tonemap.MaxProbedDevices+3) + for i := range tonemap.MaxProbedDevices + 3 { + name := "renderD" + strconv.Itoa(128+i) + env.addRenderDevice(t, name, "0x8086") + configured = append(configured, env.devicePath(name)) + } + + candidates := collectHWCandidates(strings.Join(configured, ",")) + if got := len(candidates.probeDevicesFor(transcodeHWQSV)); got != tonemap.MaxProbedDevices { + t.Fatalf("qsv probe devices = %d, want the %d cap", got, tonemap.MaxProbedDevices) + } + + // The walk therefore stays inside the budget its callers allow, which is the + // property the cap exists for. + if walk, ceiling := hwAccelWalkTimeout(candidates), MaxCapabilityRequestTimeout(); walk >= ceiling { + t.Fatalf("walk budget %v is not below the %v callers allow", walk, ceiling) + } + + // Every configured device is still reported, capped or not: the inventory is + // what an operator reads, and truncating it would hide hardware that exists. + if got := len(candidates.devicesFor(transcodeHWQSV)); got != tonemap.MaxProbedDevices { + t.Fatalf("qsv reported devices = %d, want the probed set", got) + } + if got := len(candidates.renderDevices); got != len(configured) { + t.Fatalf("render devices = %d, want all %d enumerated", got, len(configured)) + } + + // A set inside the cap is untouched. + small := configured[:3] + if got := len(collectHWCandidates(strings.Join(small, ",")).probeDevicesFor(transcodeHWQSV)); got != 3 { + t.Fatalf("qsv probe devices = %d, want the 3 configured", got) + } +} diff --git a/web/src/api/types.ts b/web/src/api/types.ts index c6df33071..07b4bca90 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3968,6 +3968,12 @@ export interface StreamNode { // until one report has been stored — and on every server predating them. capabilities?: NodeCapabilities | null; capabilities_hash?: string; + /** + * The hash the node named on its last health check. It differs from + * `capabilities_hash` while a refetch is outstanding or failing, which is the + * one case a fresh `last_health_check` cannot rule out. + */ + advertised_capabilities_hash?: string; /** When `capabilities` was fetched: the age of the inventory, not the health check. */ capabilities_refreshed_at?: string; /** Stable per-GPU identities; two nodes sharing one share hardware. */ diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index a49664c57..551b10516 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -195,8 +195,12 @@ function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNod stale diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index a529c949b..eae0634bd 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -1172,3 +1172,48 @@ describe("nodeUsesCUDADevices", () => { expect(nodeUsesCUDADevices(null, HW_ACCEL_INHERIT)).toBe(false); }); }); + +describe("capability report staleness from an unconfirmed hash", () => { + const base = { + id: 1, + name: "gpu-1", + type: "transcode", + url: "http://gpu-1", + enabled: true, + healthy: true, + created_at: "2026-08-28T00:00:00Z", + last_health_check: "2026-08-28T00:00:00Z", + capabilities_refreshed_at: "2026-08-28T00:00:00Z", + capabilities: { resolved: "qsv" }, + } as unknown as StreamNode; + + // A failing refetch leaves the two hashes apart while the health check goes on + // succeeding every 30 seconds, so the timestamp says fresh about an inventory + // the node has already contradicted. + it("marks a report stale when the node advertises a different hash", () => { + const node = { + ...base, + capabilities_hash: "sha256:stored", + advertised_capabilities_hash: "sha256:newer", + } as StreamNode; + const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); + expect(gpu?.stale).toBe(true); + }); + + it("leaves a matching hash alone on a freshly checked node", () => { + const node = { + ...base, + capabilities_hash: "sha256:stored", + advertised_capabilities_hash: "sha256:stored", + } as StreamNode; + const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); + expect(gpu?.stale).toBe(false); + }); + + // A node that never advertises one — an older build — keeps the timestamp rule. + it("falls back to the health-check age when no hash is advertised", () => { + const node = { ...base, capabilities_hash: "sha256:stored" } as StreamNode; + expect(describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z"))?.stale).toBe(false); + expect(describeNodeGPU(node, Date.parse("2026-08-28T00:20:00Z"))?.stale).toBe(true); + }); +}); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index d7e98e854..9c8f03475 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -328,6 +328,15 @@ function isCapabilityReportStale(node: StreamNode, now: number): boolean { if (Number.isNaN(Date.parse(node.capabilities_refreshed_at ?? ""))) { return false; } + // The node says its hardware is something other than what is stored. The + // sweep refetches on that mismatch, so seeing it here means the refetch has + // not landed — and a health check that keeps succeeding every 30 seconds + // would otherwise present a report we already know is obsolete as current. + // This is the one staleness a timestamp cannot see. + const advertised = node.advertised_capabilities_hash?.trim(); + if (advertised && advertised !== node.capabilities_hash?.trim()) { + return true; + } // Measured against the health check, not against the report's own age: the // check is what re-confirms the report, and it is the only one of the two // that moves on a node whose hardware never changes. From f495b0ca956870d3327621d4436e885cc0d6876c Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:26:33 -0400 Subject: [PATCH 058/163] test(web): narrow the GPU presentation union in the staleness tests NodeGPUPresentation is a discriminated union and the "awaiting" variant has no `stale`, so reading it unguarded does not compile. The tests I added last commit did exactly that. They now narrow on `kind === "reported"` the way the rest of the file does. Missed locally because I was checking with `tsc --noEmit`, which does not cover the test files under this project's references; CI runs `tsc -b` via `npm run build`, which does. Verified with that command this time. Co-Authored-By: Claude Opus 5 (1M context) --- web/src/pages/adminNodesPresentation.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index eae0634bd..4eaf062ec 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -1197,7 +1197,7 @@ describe("capability report staleness from an unconfirmed hash", () => { advertised_capabilities_hash: "sha256:newer", } as StreamNode; const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); - expect(gpu?.stale).toBe(true); + expect(gpu.kind === "reported" && gpu.stale).toBe(true); }); it("leaves a matching hash alone on a freshly checked node", () => { @@ -1207,13 +1207,15 @@ describe("capability report staleness from an unconfirmed hash", () => { advertised_capabilities_hash: "sha256:stored", } as StreamNode; const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); - expect(gpu?.stale).toBe(false); + expect(gpu.kind === "reported" && gpu.stale).toBe(false); }); // A node that never advertises one — an older build — keeps the timestamp rule. it("falls back to the health-check age when no hash is advertised", () => { const node = { ...base, capabilities_hash: "sha256:stored" } as StreamNode; - expect(describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z"))?.stale).toBe(false); - expect(describeNodeGPU(node, Date.parse("2026-08-28T00:20:00Z"))?.stale).toBe(true); + const fresh = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); + expect(fresh.kind === "reported" && fresh.stale).toBe(false); + const aged = describeNodeGPU(node, Date.parse("2026-08-28T00:20:00Z")); + expect(aged.kind === "reported" && aged.stale).toBe(true); }); }); From cbb5ad3d7ba41f7eb081a646f2a13ccbd1abed6c Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:41:42 -0400 Subject: [PATCH 059/163] fix(nodes): serve the advertised hash, answer before nudging the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The advertised-hash staleness marker added last commit never worked. The field is set on the pool's copy of a node, and GET /admin/nodes serves freshly scanned database rows — nothing carried it across, so it was always omitted and the Nodes page went on presenting a contradicted inventory as current. The list endpoint now overlays the pools' live observation onto the rows it returns. Separately, HandleUpdateNode did its post-commit work on the request goroutine. writeJSON does not end the response — the handler returning does — so the comments claiming "the response is already written" were wrong about what the client experiences: an operator saving a node waited through the worker nudge, bounded at ten seconds against a host that may be unreachable, and could see the form time out after a database write that had already succeeded. The nudge, capability-cache drop and pool reload now run on their own goroutine with the same detached, bounded context. Ordering between them is unchanged, and an afterNodeUpdate seam lets tests wait on the work rather than on a sleep. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 119 ++++++++++++++++++++-------- internal/api/handlers/nodes_test.go | 98 ++++++++++++++++++++--- 2 files changed, 174 insertions(+), 43 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 23c38d05b..5cf0f6015 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -66,6 +66,10 @@ type NodeHandler struct { // inventory. nil outside integrated mode, where there is no playback handler // holding one. invalidateCapabilityCache func(nodeURL string) + // afterNodeUpdate fires once the post-commit work an update kicks off has + // finished. Tests wait on it rather than on a sleep; production leaves it + // nil. + afterNodeUpdate func() } // SetCapabilityInvalidator wires the planning-cache drop used after a node's @@ -125,7 +129,9 @@ type checkNodeResult struct { // // The stored rows are served as they are: physical_gpu_keys is derived by the // node store when a row is scanned, so the same identities the planner routes -// on are the ones an operator sees. +// on are the ones an operator sees. The one thing added is the hash each node +// advertised on its last health check, which lives only in the pools — see +// overlayAdvertisedHashes. func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { nodes, err := h.repo.List(r.Context()) if err != nil { @@ -136,9 +142,41 @@ func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { if nodes == nil { nodes = []*nodepool.Node{} } + h.overlayAdvertisedHashes(nodes) writeJSON(w, http.StatusOK, nodes) } +// overlayAdvertisedHashes copies each node's last advertised capability hash +// from the pools onto the rows this endpoint serves. +// +// The hash is an observation from the health sweep rather than stored state, so +// it exists only on the pool's copy of a node while the row comes from the +// database. Without this the field is always absent and the admin page cannot +// tell a report the node has already contradicted from a current one — the very +// case a recent last_health_check cannot rule out, because that check keeps +// succeeding while the refetch behind it fails. +func (h *NodeHandler) overlayAdvertisedHashes(nodes []*nodepool.Node) { + advertised := make(map[int]string, len(nodes)) + collect := func(pooled []*nodepool.Node) { + for _, n := range pooled { + if n != nil && n.AdvertisedCapabilitiesHash != "" { + advertised[n.ID] = n.AdvertisedCapabilitiesHash + } + } + } + if h.proxyPool != nil { + collect(h.proxyPool.Nodes()) + } + if h.transcodePool != nil { + collect(h.transcodePool.Nodes()) + } + for _, n := range nodes { + if n != nil { + n.AdvertisedCapabilitiesHash = advertised[n.ID] + } + } +} + // HandleCreateNode handles POST /admin/nodes. func (h *NodeHandler) HandleCreateNode(w http.ResponseWriter, r *http.Request) { var input nodepool.CreateNodeInput @@ -217,40 +255,55 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { // to NVENC on a CUDA index, say) — or repointing the row at a worker that // has been running on what it inherited — gets up to a minute of requests // pairing one side's backend with the other side's device. - if nodePolicyTargetChanged(previous, node) { - // Detached for the same reason reloadPools is: the response is already - // written, so a client that has gone away must not decide whether the - // worker hears about its new policy. - if !h.reloadNodeConfig(context.WithoutCancel(r.Context()), node) { - // The node did not confirm. Its backend now comes from this - // server's pool while its device still comes from its own - // configuration, so until its poll catches up (within 60s) a start - // request can pair the new backend with the old device and fail. - // - // The policy is published anyway. Withholding it would leave an - // override an operator has saved and can see stored never reaching - // dispatch at all — nothing else re-reads the column — which is a - // silent permanent misconfiguration rather than a loud, bounded, - // self-healing one. Closing the window properly means sending the - // effective device alongside the backend so both come from one - // source; that is a change to the node start contract, not - // something to slip into a policy edit. - slog.WarnContext(r.Context(), "node has not adopted its new acceleration policy yet; transcodes dispatched to it may fail until its next config poll", - "component", "api", "node_id", node.ID, "name", node.Name) + // Off the request goroutine, not merely detached from its context. + // + // writeJSON above does not end the response — the handler returning does — + // so anything after it is latency the operator waits through. The node nudge + // alone is bounded at ten seconds against a worker that may be unreachable, + // which is long enough for the admin form to sit in "Saving..." and time out + // after a database write that already succeeded. The row is committed at + // this point; none of what follows can change the answer already written, so + // none of it belongs in front of the client. + policyChanged := nodePolicyTargetChanged(previous, node) + ctx := context.WithoutCancel(r.Context()) + go func() { + if policyChanged { + if !h.reloadNodeConfig(ctx, node) { + // The node did not confirm. Its backend now comes from this + // server's pool while its device still comes from its own + // configuration, so until its poll catches up (within 60s) a + // start request can pair the new backend with the old device + // and fail. + // + // The policy is published anyway. Withholding it would leave an + // override an operator has saved and can see stored never + // reaching dispatch at all — nothing else re-reads the column — + // which is a silent permanent misconfiguration rather than a + // loud, bounded, self-healing one. Closing the window properly + // means sending the effective device alongside the backend so + // both come from one source; that is a change to the node start + // contract, not something to slip into a policy edit. + slog.WarnContext(ctx, "node has not adopted its new acceleration policy yet; transcodes dispatched to it may fail until its next config poll", + "component", "api", "node_id", node.ID, "name", node.Name) + } + // And drop what this server believes the node can do. The v3 + // planning cache holds the tone-map executors and transformation + // inventory the *old* backend advertised, and it stays valid for + // its own TTL — so without this a session started in the next + // minute is planned against the previous backend's filters and then + // rejected by the worker that has already moved on. Dropped before + // the pool reload, for the same reason the worker is nudged first: + // nothing should dispatch under the new policy while stale + // capabilities are still readable. + if h.invalidateCapabilityCache != nil { + h.invalidateCapabilityCache(node.URL) + } } - // And drop what this server believes the node can do. The v3 planning - // cache holds the tone-map executors and transformation inventory the - // *old* backend advertised, and it stays valid for its own TTL — so - // without this a session started in the next minute is planned against - // the previous backend's filters and then rejected by the worker that - // has already moved on. Dropped before the pool reload, for the same - // reason the worker is nudged first: nothing should dispatch under the - // new policy while stale capabilities are still readable. - if h.invalidateCapabilityCache != nil { - h.invalidateCapabilityCache(node.URL) + h.reloadPools(ctx) + if h.afterNodeUpdate != nil { + h.afterNodeUpdate() } - } - h.reloadPools(r.Context()) + }() } // nodePolicyTargetChanged reports whether this update changed which effective diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 082df5904..a2ea30c12 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -136,7 +136,7 @@ func TestHandleUpdateNodeRejectsUnknownHWAccelOverride(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") recorder := httptest.NewRecorder() - handler.HandleUpdateNode(recorder, updateNodeRequest(t, `{"hw_accel_override":"videotoolbox"}`)) + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"videotoolbox"}`) if recorder.Code != http.StatusBadRequest { t.Fatalf("status = %d, want 400; body = %s", recorder.Code, recorder.Body.String()) @@ -183,7 +183,7 @@ func TestHandleUpdateNodeAcceptsAndClearsHWOverrides(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") recorder := httptest.NewRecorder() - handler.HandleUpdateNode(recorder, updateNodeRequest(t, test.body)) + awaitNodeUpdate(t, handler, recorder, test.body) if recorder.Code != http.StatusOK { t.Fatalf("status = %d, want 200; body = %s", recorder.Code, recorder.Body.String()) @@ -247,7 +247,7 @@ func TestHandleUpdateNodeReloadsTheNodeAfterAnOverrideChange(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") recorder := httptest.NewRecorder() - handler.HandleUpdateNode(recorder, updateNodeRequest(t, `{"hw_accel_override":"nvenc"}`)) + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"nvenc"}`) if recorder.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) @@ -315,7 +315,7 @@ func TestHandleUpdateNodeInvalidatesCapabilityCacheAfterAnOverrideChange(t *test invalidated := make(chan string, 4) handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, `{"hw_accel_override":"nvenc"}`)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"hw_accel_override":"nvenc"}`) select { case url := <-invalidated: @@ -368,7 +368,7 @@ func TestHandleUpdateNodePublishesPolicyEvenWhenTheNodeDoesNotConfirm(t *testing handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) recorder := httptest.NewRecorder() - handler.HandleUpdateNode(recorder, updateNodeRequest(t, `{"hw_accel_override":"nvenc"}`)) + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"nvenc"}`) if recorder.Code != http.StatusOK { t.Fatalf("status = %d, want the update to succeed anyway", recorder.Code) @@ -395,7 +395,7 @@ func TestHandleUpdateNodeKeepsCapabilityCacheWithoutAnOverrideChange(t *testing. invalidated := make(chan string, 4) handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, `{"name":"gpu-one"}`)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"name":"gpu-one"}`) select { case url := <-invalidated: @@ -431,7 +431,7 @@ func TestHandleUpdateNodeDoesNotReloadWhenOverridesAreUnchanged(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") body := `{"name":"gpu-1","hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}` - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, body)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), body) select { case <-reloaded: @@ -456,7 +456,7 @@ func TestHandleUpdateNodeDoesNotReloadWithoutAnOverrideChange(t *testing.T) { repo := &stubNodeRepository{updateResult: stored, node: stored} handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, `{"name":"gpu-one"}`)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"name":"gpu-one"}`) select { case <-reloaded: @@ -555,7 +555,7 @@ func TestHandleUpdateNodeReloadsTheReplacementWhenAURLMoves(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") body := `{"name":"gpu-1","url":"` + replacement.URL + `","hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}` - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, body)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), body) select { case path := <-reloaded: @@ -608,7 +608,7 @@ func TestHandleUpdateNodeReloadsOnAURLOnlyRepoint(t *testing.T) { handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") // Only the url: no acceleration field in the body at all. - handler.HandleUpdateNode(httptest.NewRecorder(), updateNodeRequest(t, `{"url":"`+replacement.URL+`"}`)) + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"url":"`+replacement.URL+`"}`) select { case path := <-reloaded: @@ -619,3 +619,81 @@ func TestHandleUpdateNodeReloadsOnAURLOnlyRepoint(t *testing.T) { t.Fatal("a url-only repoint left the replacement worker on its inherited policy") } } + +// awaitNodeUpdate runs one update and waits for the post-commit work it starts. +// +// HandleUpdateNode answers as soon as the row is committed and does the node +// nudge, cache drop and pool reload on their own goroutine — the nudge alone is +// bounded at ten seconds against a worker that may be unreachable, and an +// operator should not wait through that. Tests therefore wait on the handler's +// own completion signal rather than on the call returning. +func awaitNodeUpdate(t *testing.T, handler *NodeHandler, recorder *httptest.ResponseRecorder, body string) { + t.Helper() + done := make(chan struct{}) + handler.afterNodeUpdate = func() { close(done) } + handler.HandleUpdateNode(recorder, updateNodeRequest(t, body)) + if recorder.Code < 200 || recorder.Code > 299 { + // A rejected update commits nothing and starts no post-commit work, so + // there is no signal coming; the test is asserting on the refusal. + return + } + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("timed out waiting for the post-commit node update work") + } +} + +// The advertised hash lives only on the pools — it is an observation from the +// health sweep, not stored state — while this endpoint serves database rows. If +// it is not carried across, the field is always absent and the admin page has +// no way to tell a report the node has already contradicted from a current one. +func TestHandleListNodesCarriesTheAdvertisedHashFromThePools(t *testing.T) { + stored := "sha256:stored" + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + {ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", CapabilitiesHash: &stored}, + {ID: 2, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1"}, + {ID: 3, Name: "quiet", Type: nodepool.NodeTypeTranscode, URL: "http://quiet"}, + }} + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{ + {ID: 1, URL: "http://gpu-1", Enabled: true}, + {ID: 3, URL: "http://quiet", Enabled: true}, + }) + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ID: 2, URL: "http://proxy-1", Enabled: true}}) + // The sweep learns each node's advertised hash on its health check. + transcodes.ApplyHealth(1, "http://gpu-1", true, 0, 0, "sha256:newer", nil, time.Now()) + proxies.ApplyHealth(2, "http://proxy-1", true, 0, 0, "sha256:proxy", nil, time.Now()) + + handler := NewNodeHandler(repo, proxies, transcodes, nil, nil, nil, "secret") + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } + var listed []struct { + ID int `json:"id"` + Hash string `json:"capabilities_hash"` + Advertised string `json:"advertised_capabilities_hash"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &listed); err != nil { + t.Fatalf("decode: %v (%s)", err, recorder.Body.String()) + } + byID := map[int]string{} + for _, node := range listed { + byID[node.ID] = node.Advertised + } + if byID[1] != "sha256:newer" { + t.Fatalf("transcode advertised hash = %q, want the sweep's observation", byID[1]) + } + if byID[2] != "sha256:proxy" { + t.Fatalf("proxy advertised hash = %q, want the sweep's observation", byID[2]) + } + // A node that has not been checked since this process started carries none, + // and the field is omitted rather than reported as a mismatch. + if byID[3] != "" { + t.Fatalf("unchecked node advertised hash = %q, want it absent", byID[3]) + } +} From 29b0da347f96e4fccbbe6f1e0b336cd50c53f5b6 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:59:14 -0400 Subject: [PATCH 060/163] fix(playback): make the budget ceiling host-independent, advertise it on proxies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MaxCapabilityRequestTimeout priced a synthetic device list, and pricing a device means classifying it, which reads this host's sysfs. The host doing the clamping is an API replica that does not have the remote node's cards, so the fabricated render paths resolved to no vendor and counted as VAAPI alone — three commands per device short of an Intel classification. The ceiling therefore came out below what a node with a dozen Intel devices legitimately advertises, and the clamp cancelled that node before its own matrix could finish. It is now computed from the command counts directly: a ceiling that depends on where it is evaluated is not a ceiling. The proxy's capability snapshot never set ProbeRequestTimeoutMillis at all, so remote callers fell back to their own default and could cancel a proxy still inside its own endpoint budget — a proxy runs the same hardware walk a transcode node does. It advertises the budget now, before the hash is taken, since the hash covers it. Co-Authored-By: Claude Opus 5 (1M context) --- internal/playback/gpudetect.go | 32 ++++++++++++++++--- internal/playback/gpudetect_test.go | 31 ++++++++++++++++++ internal/proxy/capability_snapshot_test.go | 37 ++++++++++++++++++++++ internal/proxy/server.go | 6 ++++ 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 742d8a87a..66ad9ce25 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -411,13 +411,35 @@ func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { } // MaxCapabilityRequestTimeout is the largest budget a node may advertise and be -// believed, derived from the same composition at the device cap. +// believed. +// +// It is computed from the command counts rather than by pricing a synthetic +// device list, because classifying devices reads *this* host's sysfs — and the +// host doing the clamping is an API replica that does not have the remote +// node's cards. Fabricated render paths there resolve to no vendor and count as +// VAAPI alone, so the ceiling came out below what a node with a dozen Intel +// devices legitimately advertises, and the clamp then canceled that node +// before its own matrix could finish. A ceiling that depends on where it is +// evaluated is not a ceiling. func MaxCapabilityRequestTimeout() time.Duration { - devices := make([]string, 0, tonemap.MaxProbedDevices) - for i := range tonemap.MaxProbedDevices { - devices = append(devices, defaultDRIDir+"/renderD"+strconv.Itoa(128+i)) + return maxHWAccelWalkTimeout() + tonemap.MaxProbeRequestTimeout() +} + +// maxHWAccelWalkTimeout prices the largest walk the cap allows: every device +// classified as Intel, which is the only vendor that draws two backends, plus +// the single NVENC probe that runs regardless of the device list. +func maxHWAccelWalkTimeout() time.Duration { + perDevice := 0 + for _, backend := range []string{transcodeHWQSV, transcodeHWVAAPI} { + if probe, ok := hwBackendProbeFor(backend); ok { + perDevice += probe.commandCount + } + } + commands := perDevice * tonemap.MaxProbedDevices + if probe, ok := hwBackendProbeFor(transcodeHWNVENC); ok { + commands += probe.commandCount } - return CapabilityRequestTimeout(hwAccelAuto, strings.Join(devices, ",")) + return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack } // hwProbeDevicesTruncatedLogged latches the truncation warning to one line per diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 0cdc5d7dd..fa8cfdd6a 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -1407,3 +1407,34 @@ func TestCollectHWCandidatesCapsTheProbedDeviceSet(t *testing.T) { t.Fatalf("qsv probe devices = %d, want the 3 configured", got) } } + +// The ceiling is clamped against on an API replica, which does not have the +// remote node's cards. Pricing a synthetic device list there classified the +// fabricated paths as VAAPI-only — no sysfs vendor to read — so the ceiling came +// out below what a node with a dozen Intel devices legitimately advertises, and +// the clamp then canceled that node before its own matrix could finish. +func TestMaxCapabilityRequestTimeoutDoesNotDependOnTheLocalHost(t *testing.T) { + env := setupHWAccelTest(t) + bare := MaxCapabilityRequestTimeout() + + // Same process, now with Intel cards present: classification would change if + // the ceiling consulted sysfs at all. + for i := range 4 { + env.addRenderDevice(t, "renderD"+strconv.Itoa(128+i), "0x8086") + } + if got := MaxCapabilityRequestTimeout(); got != bare { + t.Fatalf("ceiling moved from %v to %v when local hardware appeared", bare, got) + } + + // And it covers the largest matrix the cap allows: a full set of Intel + // devices, which is the classification that draws two backends per device. + devices := make([]string, 0, tonemap.MaxProbedDevices) + for i := range tonemap.MaxProbedDevices { + name := "renderD" + strconv.Itoa(200+i) + env.addRenderDevice(t, name, "0x8086") + devices = append(devices, env.devicePath(name)) + } + if advertised := CapabilityRequestTimeout(hwAccelAuto, strings.Join(devices, ",")); advertised > bare { + t.Fatalf("a full Intel set advertises %v, above the %v ceiling that clamps it", advertised, bare) + } +} diff --git a/internal/proxy/capability_snapshot_test.go b/internal/proxy/capability_snapshot_test.go index fb3307023..14dcdfc16 100644 --- a/internal/proxy/capability_snapshot_test.go +++ b/internal/proxy/capability_snapshot_test.go @@ -160,3 +160,40 @@ func newCapabilityProxyServer(t *testing.T, secret string) *Server { w.SetConfigForTest(cfg) return NewServer(w, nil) } + +// A proxy runs the same hardware walk a transcode node does, so with enough +// configured devices its cold capability read outlives every caller's fallback. +// Without advertising a budget, a caller cancels mid-walk and the proxy's stored +// inventory falls as far behind as it would after a failure. +func TestProxyCapabilitiesAdvertiseTheProbeBudget(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + cfg := server.watcher.Config().Playback + want := playback.CapabilityRequestTimeout(cfg.HWAccel, cfg.HWDevice).Milliseconds() + if info.ProbeRequestTimeoutMillis != want { + t.Fatalf("probe_request_timeout_ms = %d, want the proxy's own %d", + info.ProbeRequestTimeoutMillis, want) + } + + // It is inside the hash, so a build that needs longer reaches the sweep + // rather than sitting behind an unchanged identity. + served := info + served.ProbeRequestTimeoutMillis = want + 1_000 + served.CapabilityHash = "" + if playback.ComputeCapabilityHash(served) == info.CapabilityHash { + t.Fatal("the advertised budget does not move the capability hash") + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 89b6539ec..bdaebb595 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -293,6 +293,12 @@ func (s *Server) buildCapabilitySnapshotLocked(ctx context.Context) (playback.HW return playback.HWAccelInfo{}, err } info.Transformations = registry.Advertised() + // Advertised before the hash is taken, because it is part of what the hash + // covers. A proxy runs the same hardware walk a transcode node does, so + // with enough configured devices its cold read outlives every caller's + // fallback — and a caller that cancels mid-walk leaves the proxy's stored + // inventory as far behind as a failure would. + info.ProbeRequestTimeoutMillis = playback.CapabilityRequestTimeout(hwAccel, hwDevice).Milliseconds() info.CapabilityHash = playback.ComputeCapabilityHash(info) return info, nil } From dcf62fa60a0b5eed50a358f0d1c1ba9e672aa6f3 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:18:48 -0400 Subject: [PATCH 061/163] fix(nodes): serialize config reloads and publish manual checks to the pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways a node's live state could disagree with what the API just confirmed. Reloads arrive from three places — the 60s poll, the settings-changed event, and ForceReload on a request goroutine — and only the first two share a goroutine. A poll that sampled server_settings before an operator's edit could return after ForceReload had already applied that edit and put the pre-edit snapshot back. The node answered the endpoint 204, the API reloaded its pool believing the worker had adopted the new backend and device, and the worker went on transcoding with the old ones. Reload now holds a mutex from the read through the swap and its callbacks, so a later swap is always built on a later read. A manual node check wrote only the database row. The pool the planner reads kept its previous stats, so an operator who checked a node and found its scratch volume full watched work keep going there until the next sweep, and the Nodes page paired the fresh row timestamp with the pool's older advertised hash — which it renders as a reconfirmed inventory. The result now goes through the matching pool's ApplyHealth, exactly as the background checker's does. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 28 +++++++++++ internal/api/handlers/nodes_test.go | 54 ++++++++++++++++++++++ internal/nodeconfig/watcher.go | 41 ++++++++++++++-- internal/nodeconfig/watcher_test.go | 72 +++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 5 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 5cf0f6015..9820500b9 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -442,6 +442,13 @@ func (h *NodeHandler) HandleCheckNode(w http.ResponseWriter, r *http.Request) { if err := h.repo.UpdateHealth(r.Context(), id, node.URL, healthy, activeJobs, egressKbps, lastStats); err != nil { slog.ErrorContext(r.Context(), "persisting health check result", "component", "api", "node_id", id, "error", err) } + // The pools get it too, exactly as the background sweep would. A manual + // check that only wrote the row would leave the planner admitting work to a + // node whose scratch volume this check just found full, and would pair a + // fresh last_health_check in the database with the pool's older advertised + // hash — which is the combination the Nodes page reads as "this inventory + // was reconfirmed", for up to the next 30 seconds. + h.applyHealthToPools(node, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats) writeJSON(w, http.StatusOK, checkNodeResult{ Healthy: healthy, @@ -451,6 +458,27 @@ func (h *NodeHandler) HandleCheckNode(w http.ResponseWriter, r *http.Request) { }) } +// applyHealthToPools publishes one check's result to whichever pool holds the +// node, so an operator-triggered check lands everywhere the sweep's would. +// +// Fenced on the node's URL by the pools themselves, like every other health +// write: the row can be repointed while a check is in flight. +func (h *NodeHandler) applyHealthToPools( + node *nodepool.Node, healthy bool, activeJobs, egressKbps int, capabilitiesHash string, lastStats []byte, +) { + checkedAt := time.Now() + switch node.Type { + case nodepool.NodeTypeProxy: + if h.proxyPool != nil { + h.proxyPool.ApplyHealth(node.ID, node.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, checkedAt) + } + case nodepool.NodeTypeTranscode: + if h.transcodePool != nil { + h.transcodePool.ApplyHealth(node.ID, node.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, checkedAt) + } + } +} + // HandleForceReloadNodes handles POST /admin/nodes/force-reload — sends a // force-reload signal to every enabled node in parallel. func (h *NodeHandler) HandleForceReloadNodes(w http.ResponseWriter, r *http.Request) { diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index a2ea30c12..41b3fa616 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -697,3 +697,57 @@ func TestHandleListNodesCarriesTheAdvertisedHashFromThePools(t *testing.T) { t.Fatalf("unchecked node advertised hash = %q, want it absent", byID[3]) } } + +// A manual check is a health check like the sweep's: whatever it learns has to +// reach the pool the planner and the Nodes page read, not just the row. +func TestHandleCheckNodePublishesResultToThePool(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "active_jobs": 3, + "egress_kbps": 4200, + "capabilities_hash": "sha256:after", + "system": map[string]any{"scratch_free_gb": 0.5}, + }) + })) + defer node.Close() + + stale := "sha256:before" + pooled := &nodepool.Node{ + ID: 7, Name: "gpu-7", Type: nodepool.NodeTypeTranscode, URL: node.URL, Enabled: true, + Healthy: true, ActiveJobs: 99, CapabilitiesHash: &stale, AdvertisedCapabilitiesHash: stale, + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{pooled}) + + repo := &stubNodeRepository{node: pooled} + handler := NewNodeHandler(repo, nil, pool, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/admin/nodes/7/check", nil) + routeContext := chi.NewRouteContext() + routeContext.URLParams.Add("id", "7") + handler.HandleCheckNode(recorder, request.WithContext(context.WithValue(request.Context(), chi.RouteCtxKey, routeContext))) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + + updated := pool.Nodes() + if len(updated) != 1 { + t.Fatalf("pool holds %d nodes, want 1", len(updated)) + } + if got := updated[0].AdvertisedCapabilitiesHash; got != "sha256:after" { + t.Errorf("pool advertised hash = %q, want the hash this check just read", got) + } + if got := updated[0].ActiveJobs; got != 3 { + t.Errorf("pool active jobs = %d, want 3", got) + } + // The stats decide whether the planner keeps admitting work; a check that + // found the scratch volume nearly full has to reach the planner's copy. + if len(updated[0].LastStats) == 0 { + t.Error("pool node kept no stats from the manual check") + } + if updated[0].LastHealthCheck == nil { + t.Error("pool node kept no check timestamp, so the row and the pool disagree on freshness") + } +} diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index a067f8f6b..61ef4abea 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -82,6 +82,14 @@ type Watcher struct { // nodeRowID is the row this worker has resolved to, kept so a later rename // or repoint cannot sever the association. Zero until a lookup succeeds. nodeRowID int + + // reloadMu makes one reload atomic from the read of server_settings through + // the config swap and its callbacks; see reload. + reloadMu sync.Mutex + // fetchSettings, when set, replaces the read of server_settings. Only a + // test sets it — nothing in production needs to read settings from + // anywhere but the database. + fetchSettingsFn func(context.Context) (map[string]string, error) } // rememberNodeRowID records the row a lookup resolved to. @@ -175,7 +183,7 @@ func (w *Watcher) Start(ctx context.Context) error { // ForceReload triggers an immediate config reload from the database. func (w *Watcher) ForceReload(ctx context.Context) error { - if w == nil || w.pool == nil { + if w == nil || (w.pool == nil && w.fetchSettingsFn == nil) { // A watcher with no database is one a test constructed, or a mode that // runs entirely off bootstrap overrides. Either way there is nothing to // re-read, and an error is a far better answer than the nil dereference @@ -186,9 +194,8 @@ func (w *Watcher) ForceReload(ctx context.Context) error { } // RequestReload asks the poll goroutine to reload soon. Non-blocking and -// coalescing — safe to call from request handlers. Unlike ForceReload, the -// reload runs on the poll goroutine, so concurrent requests can never swap a -// stale snapshot over a newer one. +// coalescing — safe to call from request handlers. Unlike ForceReload it does +// not wait for the reload, and does not report whether it succeeded. func (w *Watcher) RequestReload() { select { case w.reloadCh <- struct{}{}: @@ -207,8 +214,32 @@ func (w *Watcher) SetConfigForTest(cfg *config.Config) { // reload fetches all settings from the database, builds a new Config, // applies bootstrap overrides, and atomically swaps the config pointer. +// +// Read and swap are one critical section. Reloads arrive from three places — +// the 60s poll, the settings-changed event, and ForceReload on a request +// goroutine — and only the first two share a goroutine. Letting them overlap +// would make the winner the one that finished last rather than the one that +// read last: a poll that sampled server_settings before an operator's edit can +// return after ForceReload has already applied the edit, and put its pre-edit +// snapshot back. The node would then answer 204 to the endpoint, the API would +// reload its pool believing the node adopted the new backend and device, and +// the node would go on transcoding with the old ones until something reloaded +// it again. +// +// Serializing the whole operation, rather than just the swap, is what fixes +// that: a reload's fetch cannot begin until the previous reload's swap is +// done, so a later swap is always built on a later read. The callbacks run +// inside the section too, so an older reload can't announce its config as the +// current one after a newer swap. func (w *Watcher) reload(ctx context.Context) error { - m, err := w.fetchSettings(ctx) + w.reloadMu.Lock() + defer w.reloadMu.Unlock() + + fetch := w.fetchSettings + if w.fetchSettingsFn != nil { + fetch = w.fetchSettingsFn + } + m, err := fetch(ctx) if err != nil { return err } diff --git a/internal/nodeconfig/watcher_test.go b/internal/nodeconfig/watcher_test.go index 8013546e9..f382b9936 100644 --- a/internal/nodeconfig/watcher_test.go +++ b/internal/nodeconfig/watcher_test.go @@ -2,7 +2,11 @@ package nodeconfig import ( "context" + "slices" + "sync" + "sync/atomic" "testing" + "time" "github.com/Silo-Server/silo-server/internal/config" ) @@ -128,3 +132,71 @@ func TestOnLoadNormalizersApplyOnEveryLoad(t *testing.T) { t.Fatalf("after reload Playback.FFmpegPath = %q, want normalized value", got) } } + +// staleReloadWindow is how long the poll's fetch waits for a concurrent forced +// reload to overtake it. It is a bound on a violation, not a wait for +// completion: serialized, nothing can overtake and the wait expires; unserialized, +// the forced reload finishes far inside it and the poll's stale snapshot lands +// last. +const staleReloadWindow = 250 * time.Millisecond + +// The poll, the settings-changed event, and ForceReload all reload, and only +// the first two share a goroutine. A poll that read server_settings before an +// operator's edit must not be able to put that pre-edit snapshot back after +// ForceReload has already applied the edit — the node would answer the endpoint +// 204 while running the old policy. +func TestReloadDoesNotLetAnEarlierSnapshotSupersedeALaterOne(t *testing.T) { + w := &Watcher{} + + var mu sync.Mutex + var events []string + record := func(event string) { + mu.Lock() + defer mu.Unlock() + events = append(events, event) + } + w.OnChange(func(_, updated *config.Config) { record("apply " + updated.Server.LogLevel) }) + + polling := make(chan struct{}) + forced := make(chan struct{}) + var fetches atomic.Int32 + w.fetchSettingsFn = func(context.Context) (map[string]string, error) { + if fetches.Add(1) == 1 { + record("fetch info") + close(polling) + select { + case <-forced: + case <-time.After(staleReloadWindow): + } + return map[string]string{"server.log_level": "info"}, nil + } + record("fetch error") + return map[string]string{"server.log_level": "error"}, nil + } + + polled := make(chan error, 1) + go func() { polled <- w.reload(context.Background()) }() + <-polling + + go func() { + if err := w.ForceReload(context.Background()); err != nil { + t.Errorf("ForceReload() error = %v", err) + } + close(forced) + }() + + if err := <-polled; err != nil { + t.Fatalf("poll reload error = %v", err) + } + <-forced + + if got := w.Config().Server.LogLevel; got != "error" { + t.Errorf("Server.LogLevel = %q, want the value the later read saw", got) + } + mu.Lock() + defer mu.Unlock() + want := []string{"fetch info", "apply info", "fetch error", "apply error"} + if !slices.Equal(events, want) { + t.Errorf("reloads interleaved:\n got %v\nwant %v", events, want) + } +} From 02605faeb40d0a5d0d94dffcd7c285479f9078ba Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:33:26 -0400 Subject: [PATCH 062/163] fix(nodemetrics): only correct CPU where something actually caps it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unconstrained cgroup publishes an effective cpuset holding every online CPU, because it inherits the root's — every container started without --cpuset-cpus and every systemd service without AllowedCPUs= looks, to a reader that only counts entries, exactly like a process pinned to a subset. cgroupCPU then took that count as binding, moved the reading to this cgroup's own usage, and a nearly idle Silo on a saturated shared host reported a few percent as the machine's load. A cpuset spanning the whole host is now no cpuset at all. Ignoring it exposed the general form of the same mistake: with no cap in force the leaf's usage was still the numerator, so the same host reported Silo's share rather than the machine's. Uncapped, the CPU a neighbor burns is CPU this node cannot have — which is what /proc/stat already reports, and what lxcfs narrows to the container where that applies. So the correction now applies only where a quota or a real cpuset binds, and the file header no longer claims a root-cgroup fallback the self-cgroup walk had made untrue. TestCPUWithoutCgroupQuotaUsesHostCores asserted the behavior being removed here; it is rewritten as TestCPUWithoutCgroupQuotaReportsTheHost, with the process at a quarter of the machine and the machine at three quarters so the two answers cannot be confused. Co-Authored-By: Claude Opus 5 (1M context) --- internal/nodemetrics/cgroupcpu.go | 32 +++++++++-- internal/nodemetrics/sampler.go | 29 ++++++---- internal/nodemetrics/sampler_test.go | 85 +++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 24 deletions(-) diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index a893d40f0..8f24b9a58 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -19,9 +19,11 @@ import ( // cumulative usage is the busy signal, and its quota is what that usage is // normalized against. // -// A host with no cgroup limit reads its root cgroup, which accounts for every -// process on the machine, so an unconstrained deployment reports what it always -// did. +// The correction applies only where something actually caps CPU. A cgroup +// imposes no limit far more often than it imposes one — every unconstrained +// container and systemd service has one — and its usage then describes Silo +// alone rather than the machine, so an uncapped deployment keeps reading +// /proc/stat and reports the load it is really competing with. // cgroupCPUPath locates one cgroup version's CPU accounting. type cgroupCPUPath struct { @@ -90,15 +92,33 @@ var cgroupCPUSetPaths = []string{ // The effective file already accounts for ancestors, so unlike the quota this // needs no walk of its own; where only the pre-intersection file exists, the // per-level candidates cover the same ground. -func cgroupCPUSetCores(paths []string) int { +// +// hostCores is how many CPUs the machine has, and a cpuset that spans all of +// them is not a restriction — it is what every unconstrained container and +// service publishes, because the effective set is inherited from a root that +// holds every online CPU. Counting it as a cap would be worse than ignoring it: +// cgroupCPU treats any cpuset as binding when nothing else caps CPU, so the +// reading would move to this cgroup's own usage, and a nearly idle Silo on a +// saturated shared host would report a few percent instead of the host's load. +// A host size of 0 means /proc/stat could not be counted, and an unknown host +// is no reason to discard a cpuset that may well be real. +func cgroupCPUSetCores(paths []string, hostCores int) int { for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { continue } - if count := countCPUSetEntries(string(raw)); count > 0 { - return count + count := countCPUSetEntries(string(raw)) + if count <= 0 { + continue + } + if hostCores > 0 && count >= hostCores { + // The effective set is the intersection with every ancestor, so no + // file later in the list can narrow what this one just said was the + // whole machine. + return 0 } + return count } return 0 } diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 95c14221b..7d57441d9 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -266,11 +266,12 @@ func (s *Sampler) sampleSystem(ctx context.Context, now time.Time) *SystemStats // cpuStats reports busy percentage and the core count that percentage is // normalized against. // -// Under a cgroup both come from the cgroup, not the host: /proc/stat is -// host-wide even in a container, so a node capped at two cores of a busy -// 64-core machine would otherwise report the machine's load instead of its own -// — and a node pinned at its quota, which is the state worth alerting on, would -// look nearly idle. +// Under a cgroup that caps CPU both come from the cgroup, not the host: +// /proc/stat is host-wide even in a container, so a node capped at two cores of +// a busy 64-core machine would otherwise report the machine's load instead of +// its own — and a node pinned at its quota, which is the state worth alerting +// on, would look nearly idle. Where no cap is in force the host figure stands, +// because then the machine's load is the load this node competes with. func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { host, hostCores := readCPUTimes(s.procDirFor("stat")) busyPct, _ = cpuBusyPercent(s.prevCPU, host) @@ -287,10 +288,19 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { // host. Whichever cap is tighter is the one it actually runs under, and // cgroupCPU decides that, because the answer also settles which cgroup's // usage the reading comes from. - sample, quota := s.cgroupCPU(now, cgroupCPUSetCores(s.cgroupCPUSetPaths)) - if quota > 0 { - cores = cgroupQuotaCores(quota, hostCores) + sample, quota := s.cgroupCPU(now, cgroupCPUSetCores(s.cgroupCPUSetPaths, hostCores)) + if quota <= 0 { + // Nothing caps this process, so its own cgroup is the wrong domain to + // measure: the leaf accounts for Silo alone, and Silo idling beside a + // saturated neighbor is not an idle machine. Uncapped, the CPU a + // neighbor burns is CPU this node cannot have, which is exactly what + // /proc/stat reports — and under lxcfs that file is already narrowed to + // the container, so this stays right there too. The next pass to find a + // cap starts a fresh pair rather than differencing across the switch. + s.prevCgroupCPU = cgroupCPUSample{} + return busyPct, cores } + cores = cgroupQuotaCores(quota, hostCores) if !sample.valid { return busyPct, cores } @@ -304,9 +314,6 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { if hostCores > 0 && budget > float64(hostCores) { budget = float64(hostCores) } - if budget <= 0 { - budget = float64(cores) - } // Once the cgroup can be read it is the only honest source, so its answer // stands even when this pass cannot derive one (the first sample, or a // counter reset). Falling back to the host figure would silently mix two diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 608719c4d..67e0db890 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -388,12 +388,15 @@ func TestCPUCorrectedByCgroupQuotaAndUsage(t *testing.T) { } } -// An unconstrained cgroup still measures this process's domain, but there is no -// quota to normalize against, so the host's core count is the right divisor. -func TestCPUWithoutCgroupQuotaUsesHostCores(t *testing.T) { +// An unconstrained cgroup measures this process alone, which is not what a node +// competes for: uncapped, the CPU a neighbor burns is CPU this node cannot have. +// So with no quota the host's own busyness stands, even though the cgroup could +// be read — here Silo uses a quarter of the machine while the machine is at +// three quarters. +func TestCPUWithoutCgroupQuotaReportsTheHost(t *testing.T) { tree := newProcTree(t) clock := newFakeClock() - tree.write("stat", "cpu 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") + tree.write("stat", "cpu 100 0 0 900 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") tree.write("loadavg", "0 0 0 0/0 0\n") tree.write("meminfo", "MemTotal: 1024 kB\n") tree.write("net/dev", "") @@ -416,16 +419,19 @@ func TestCPUWithoutCgroupQuotaUsesHostCores(t *testing.T) { }} s.sample(context.Background()) - // 5 seconds of CPU over 5 seconds of wall time across 2 cores: half busy. + // 5 seconds of CPU over 5 seconds of wall time across 2 cores: half of what + // this process could use, and a quarter of the two-core machine. if err := os.WriteFile(usage, []byte("usage_usec 5000000\n"), 0o644); err != nil { t.Fatal(err) } + // The host meanwhile spent 750 of the interval's 1000 ticks busy. + tree.write("stat", "cpu 850 0 0 1150 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") clock.advance(5 * time.Second) s.sample(context.Background()) system := s.Snapshot().System - if system.CPUPct != 50 { - t.Fatalf("CPUPct = %d, want 50", system.CPUPct) + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 rather than this process's 25", system.CPUPct) } if system.Cores != 2 { t.Fatalf("Cores = %d, want the host's 2 with no quota set", system.Cores) @@ -1113,3 +1119,68 @@ func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { t.Fatalf("CPUPct = %d, want 100 — the node has spent every CPU it can", system.CPUPct) } } + +// Every unconstrained container and systemd service publishes an effective +// cpuset holding every online CPU, because it inherits the root's. Reading that +// as a two-CPU-style restriction would move the CPU reading to this cgroup's own +// usage on a host where nothing caps it, and Silo idling beside a saturated +// neighbor would report Silo's few percent as the machine's load. +func TestCPUIgnoresACpusetSpanningTheWholeHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 900)) + + dir := t.TempDir() + // All eight of the host's CPUs — what an unrestricted cgroup reports. + cpuset := filepath.Join(dir, "cpuset.cpus.effective") + if err := os.WriteFile(cpuset, []byte("0-7\n"), 0o644); err != nil { + t.Fatalf("write cpuset: %v", err) + } + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + if err := os.WriteFile(quota, []byte("max 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + if got := cgroupCPUSetCores([]string{cpuset}, 8); got != 0 { + t.Fatalf("cgroupCPUSetCores() = %d for a cpuset holding every host CPU, want 0", got) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.cgroupCPUSetPaths = []string{cpuset} + s.sample(context.Background()) + + // The host spends 750 of the interval's 1000 ticks busy; Silo uses 2 of its + // 40 available core-seconds, which is 5%. + writeUsage(2_000_000) + tree.write("stat", hostStat(850, 1150)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 — nothing caps this process", system.CPUPct) + } + if system.Cores != 8 { + t.Fatalf("Cores = %d, want the host's 8", system.Cores) + } +} From e11eaff4a975546092f3891bed910aed1f0db99f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 02:05:16 -0400 Subject: [PATCH 063/163] fix(nodes): price cold node probes from the node, not the cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review findings, three of which share a root: something about a node was being derived from something that is not that node. A capability read is slowest when it is cold — every probe cache on the node is empty, so the whole hardware walk and tone-map matrix run — and that is exactly when both remote paths guessed low. Protocol-v3 planning fell back to a flat two minutes, which its own comment records as short of the ~136 seconds a two-device node legitimately asks for; the download preparer priced the cluster-wide device setting, which says nothing about a node overridden onto four devices. Guessing low here does not slow the read down, it cancels it: the node drops out of the capability map mid-matrix and playback plans without it, or the download falls back locally and fails outright where local fallback is off. Both now resolve the pooled node by URL and price it from what it advertised in its stored report, then from its own effective override, then from the caller's fallback — playback.ColdCapabilityRequestTimeout, which also absorbs the parse the reprobe handler had grown separately. The Prometheus surface exported the GPU engine gauges but not whole-GPU utilization, though the sampler collects it and the JSON surfaces show it. On a shared card those are the two numbers that disagree, and the missing one is the one worth alerting on: a transcode planned onto a GPU another tenant has saturated. streamapp_node_gpu_busy_percent now ships wherever a source reports one. Finally, an advertised capability hash was a string, so "the node named no hash" and "nobody has checked this node yet" were the same value — and the admin page, which marks a report stale when the node contradicts it, ignored the first case entirely. A node downgraded past capability reports would have kept presenting a stored inventory as current for as long as its health checks kept succeeding. The field is now a pointer with three states, absent means unchecked, and the page distinguishes contradicted from unreported from unconfirmed in what it says. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 2 +- docs/wiki/admin/monitoring-nodes.md | 18 +++-- internal/api/handlers/nodes.go | 19 ++--- internal/api/handlers/nodes_test.go | 48 ++++++++++++- internal/api/handlers/playback_v3.go | 37 +++++++++- internal/api/handlers/playback_v3_test.go | 75 +++++++++++++++++++- internal/downloads/remote_preparer.go | 35 +++++++-- internal/downloads/remote_preparer_test.go | 61 ++++++++++++++++ internal/nodemetrics/collector.go | 12 ++++ internal/nodemetrics/collector_test.go | 24 +++++++ internal/nodepool/repository.go | 36 +++++++++- internal/nodepool/transcode_pool.go | 5 +- internal/playback/gpudetect.go | 50 +++++++++++++ web/src/api/types.ts | 9 ++- web/src/pages/AdminNodes.tsx | 35 ++++++--- web/src/pages/adminNodesPresentation.test.ts | 46 +++++++++--- web/src/pages/adminNodesPresentation.ts | 48 +++++++++---- 17 files changed, 487 insertions(+), 73 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index 7913b7e4a..d60063333 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -28,7 +28,7 @@ Always `200 OK` with a JSON array. | `created_at` | RFC3339 string | When the node was registered. | | `capabilities` | object | The node's last stored capability report — the same body `GET /hw-capabilities` returns on the node. Omitted until one has been stored. | | `capabilities_hash` | string | Identity of that report, as computed by the node. Omitted with `capabilities`. | -| `advertised_capabilities_hash` | string | The hash the node named on its last health check. It differs from `capabilities_hash` while a refetch is outstanding or failing — the one case a recent `last_health_check` cannot rule out, since that check keeps succeeding while the refetch does not. Derived per sweep rather than stored, so it is absent until the first check after an API restart. | +| `advertised_capabilities_hash` | string | The hash the node named on its last health check. It differs from `capabilities_hash` while a refetch is outstanding or failing — the one case a recent `last_health_check` cannot rule out, since that check keeps succeeding while the refetch does not. Derived per sweep rather than stored, so it is **absent** until the first check after an API restart; **present and empty** when the node answered but named no hash at all, as a build predating capability reports does. Absent says nothing about the stored report; empty says the node is no longer confirming it. | | `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | | `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index 1de9b8e2e..52ec8b6e5 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -306,12 +306,18 @@ idle on a card that may be busy and merely unobservable. Its session count still ships, since that comes from Silo's own accounting rather than from a driver. The GPU gauges (`streamapp_node_gpu_video_busy_percent`, -`streamapp_node_gpu_render_busy_percent`, `streamapp_node_gpu_sessions`, -`streamapp_node_gpu_vram_used_bytes`, `streamapp_node_gpu_vram_total_bytes`) are -labeled by `device`. On Intel and AMD they measure Silo's own FFmpeg processes -only, so a card shared with anything outside Silo reads as less busy than it is; -on NVIDIA they come from `nvidia-smi` and are whole-GPU. A device that nothing -could measure this interval publishes no VRAM series rather than a zero. +`streamapp_node_gpu_render_busy_percent`, `streamapp_node_gpu_busy_percent`, +`streamapp_node_gpu_sessions`, `streamapp_node_gpu_vram_used_bytes`, +`streamapp_node_gpu_vram_total_bytes`) are labeled by `device`. On Intel and AMD +the engine gauges measure Silo's own FFmpeg processes only, so a card shared with +anything outside Silo reads as less busy than it is; on NVIDIA they come from +`nvidia-smi` and are whole-GPU. A device that nothing could measure this interval +publishes no VRAM series rather than a zero. + +`streamapp_node_gpu_busy_percent` is the card's own utilization, other tenants +included, and it ships wherever a source reports one — `nvidia-smi` today. It is +the series to alert on for a shared GPU: the engine gauges can show Silo idle +while the card it is planned onto is saturated by something else. If `nvidia-smi` fails five samples running, Silo stops calling it — a host without the NVIDIA toolkit would otherwise spawn a doomed subprocess every few diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 9820500b9..3e376756c 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -156,10 +156,10 @@ func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { // case a recent last_health_check cannot rule out, because that check keeps // succeeding while the refetch behind it fails. func (h *NodeHandler) overlayAdvertisedHashes(nodes []*nodepool.Node) { - advertised := make(map[int]string, len(nodes)) + advertised := make(map[int]*string, len(nodes)) collect := func(pooled []*nodepool.Node) { for _, n := range pooled { - if n != nil && n.AdvertisedCapabilitiesHash != "" { + if n != nil && n.AdvertisedCapabilitiesHash != nil { advertised[n.ID] = n.AdvertisedCapabilitiesHash } } @@ -623,19 +623,10 @@ const nodeReprobeFallbackTimeout = 150 * time.Second // The node itself publishes the budget its probe matrix needs in every // capability report (probe_request_timeout_ms), which is exactly what a cold // capability fetch is given; a node with different hardware or a slower ffmpeg -// therefore gets its own number rather than a cluster-wide guess. The stored -// report is parsed minimally here for the same reason nodepool parses its own -// narrow views: this layer has no business decoding the whole inventory. +// therefore gets its own number rather than a cluster-wide guess. func nodeReprobeTimeout(n *nodepool.Node) time.Duration { - var advertised struct { - ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms"` - } - if n != nil && len(n.Capabilities) > 0 { - // A report that cannot be parsed leaves the zero value, which is the - // fallback; an unreadable report is not a reason to fail the action. - _ = json.Unmarshal(n.Capabilities, &advertised) - } - return playback.NormalizeProbeRequestTimeout(advertised.ProbeRequestTimeoutMillis, nodeReprobeFallbackTimeout) + return playback.NormalizeProbeRequestTimeout( + playback.AdvertisedProbeBudgetMillis(n.StoredCapabilities()), nodeReprobeFallbackTimeout) } // nodeReprobeWriteSlack covers the repository round trips and the JSON write diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go index 41b3fa616..928f1e09d 100644 --- a/internal/api/handlers/nodes_test.go +++ b/internal/api/handlers/nodes_test.go @@ -714,7 +714,7 @@ func TestHandleCheckNodePublishesResultToThePool(t *testing.T) { stale := "sha256:before" pooled := &nodepool.Node{ ID: 7, Name: "gpu-7", Type: nodepool.NodeTypeTranscode, URL: node.URL, Enabled: true, - Healthy: true, ActiveJobs: 99, CapabilitiesHash: &stale, AdvertisedCapabilitiesHash: stale, + Healthy: true, ActiveJobs: 99, CapabilitiesHash: &stale, AdvertisedCapabilitiesHash: &stale, } pool := nodepool.NewTranscodePool() pool.SetNodes([]*nodepool.Node{pooled}) @@ -736,8 +736,8 @@ func TestHandleCheckNodePublishesResultToThePool(t *testing.T) { if len(updated) != 1 { t.Fatalf("pool holds %d nodes, want 1", len(updated)) } - if got := updated[0].AdvertisedCapabilitiesHash; got != "sha256:after" { - t.Errorf("pool advertised hash = %q, want the hash this check just read", got) + if got := updated[0].AdvertisedCapabilitiesHash; got == nil || *got != "sha256:after" { + t.Errorf("pool advertised hash = %v, want the hash this check just read", got) } if got := updated[0].ActiveJobs; got != 3 { t.Errorf("pool active jobs = %d, want 3", got) @@ -751,3 +751,45 @@ func TestHandleCheckNodePublishesResultToThePool(t *testing.T) { t.Error("pool node kept no check timestamp, so the row and the pool disagree on freshness") } } + +// Three states, not two: a node that answers health checks with no hash is not +// the same as a node nobody has checked yet. The first is no longer standing +// behind the inventory stored for it — a build downgraded past capability +// reports — while the second is every node until the first sweep after a +// restart, and says nothing at all. +func TestHandleListNodesDistinguishesUncheckedFromUnreportedHashes(t *testing.T) { + stored := "sha256:stored" + none := "" + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + {ID: 1, Name: "checked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", CapabilitiesHash: &stored}, + {ID: 2, Name: "unchecked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-2", CapabilitiesHash: &stored}, + }} + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{ + {ID: 1, Name: "checked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", AdvertisedCapabilitiesHash: &none}, + {ID: 2, Name: "unchecked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-2"}, + }) + handler := NewNodeHandler(repo, nil, pool, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var items []map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &items); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(items) != 2 { + t.Fatalf("returned %d nodes, want 2", len(items)) + } + // Present and empty: the node was asked and named nothing. + advertised, ok := items[0]["advertised_capabilities_hash"] + if !ok || advertised != "" { + t.Errorf("checked node advertised %v (present=%v), want an empty string", advertised, ok) + } + // Absent: nothing has asked, so the field must not claim the node said so. + if _, ok := items[1]["advertised_capabilities_hash"]; ok { + t.Errorf("unchecked node carried an advertised hash: %v", items[1]) + } +} diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index bd298c095..40f957346 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -326,7 +326,42 @@ func (h *PlaybackHandler) remoteToneMapProbeTimeoutV3(nodeURL string) time.Durat if budget > 0 { return budget } - return remoteNodeProbeFallbackTimeout + // Nothing learned from this node yet, which is every node after an API + // restart and any node registered since. The durable report holds what it + // last advertised, and its override holds what it would advertise; the flat + // fallback below is shorter than a two-device node's matrix legitimately + // takes, so reaching for it first would cancel exactly the multi-GPU nodes + // this path most wants to keep. + return h.coldNodeProbeTimeoutV3(nodeURL) +} + +// coldNodeProbeTimeoutV3 prices one capability read of a node this process has +// not read successfully yet, from that node rather than from a cluster-wide +// guess. Without a pooled record — a planner that cannot look nodes up, or a +// URL that is no longer in the pool — the cluster's own policy is the closest +// description available. +func (h *PlaybackHandler) coldNodeProbeTimeoutV3(nodeURL string) time.Duration { + cfg := h.playbackConfig() + var node *nodepool.Node + if lookup, ok := h.NodePlanner.(transcodeNodeLookupV3); ok { + if found, ok := lookup.TranscodeNodeByURL(nodeURL); ok { + node = found + } + } + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(cfg.HWAccel), + node.EffectiveHWDevice(cfg.HWDevice), + remoteNodeProbeFallbackTimeout, + ) +} + +// transcodeNodeLookupV3 resolves the pooled record behind a transcode node URL, +// which carries that node's stored capability report and its acceleration +// override. Optional, like the planner itself: without it this path falls back +// to the cluster-wide setting. *nodepool.Planner implements it. +type transcodeNodeLookupV3 interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) } // rememberNodeProbeBudgetV3 records what a node says its capability read costs. diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index eaf95c84c..5bb3538c3 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -5401,8 +5401,12 @@ func TestPlaybackV3ToneMapBudgetsCoverColdNodeWork(t *testing.T) { if got, want := handler.localToneMapProbeTimeoutV3(), playback.CapabilityEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { t.Fatalf("local tone-map probe timeout = %s, want %s", got, want) } - if got, want := handler.remoteToneMapProbeTimeoutV3("https://unknown.example"), remoteNodeProbeFallbackTimeout; got != want { - t.Fatalf("remote tone-map probe timeout = %s, want %s", got, want) + // A node this handler cannot resolve is priced at the cluster's own policy, + // which is what such a node runs unless it carries an override — closer than + // a flat guess, and the flat fallback is reserved for having no policy either. + coldUnknown := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/dev/dri/renderD128") + if got := handler.remoteToneMapProbeTimeoutV3("https://unknown.example"); got != coldUnknown { + t.Fatalf("remote tone-map probe timeout = %s, want %s", got, coldUnknown) } if got := handler.toneMapPlanningTimeoutV3(true); got != v3NodeCapabilityPlanTimeout { t.Fatalf("planning timeout with local fallback = %s, want %s", got, v3NodeCapabilityPlanTimeout) @@ -5417,7 +5421,7 @@ func TestPlaybackV3ToneMapBudgetsCoverColdNodeWork(t *testing.T) { RequireReady: true, HWAccel: tonemap.BackendQSV, } - want := remoteNodeProbeFallbackTimeout + playback.ManifestStartupTimeout + + want := coldUnknown + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout if got := handler.remotePlaybackTransportTimeout("https://unknown.example", request); got != want { t.Fatalf("remote tone-map start timeout = %s, want %s", got, want) @@ -5548,3 +5552,68 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { t.Fatalf("bounded remote probe timeout = %s, want %s", got, want) } } + +// v3NodeLookupPlanner plans nothing and only resolves nodes by URL, which is all +// a cold probe budget needs from a planner. +type v3NodeLookupPlanner struct { + node *nodepool.Node +} + +func (p *v3NodeLookupPlanner) PlanSession(string, string, bool, int) nodepool.Plan { + return nodepool.Plan{} +} + +func (p *v3NodeLookupPlanner) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if p.node == nil || p.node.URL != nodeURL { + return nil, false + } + return p.node, true +} + +// The first read of a node after an API restart is also its most expensive: no +// probe cache on the node survives, so the whole matrix runs. The flat fallback +// is shorter than a two-device node legitimately takes, and paying it would +// cancel exactly the multi-GPU nodes this path exists to plan onto — so the +// durable report the node last advertised is what prices it. +func TestPlaybackV3ColdNodeProbeBudgetComesFromTheStoredReport(t *testing.T) { + const nodeURL = "https://gpu-1.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, + Capabilities: json.RawMessage(`{"resolved":"qsv","probe_request_timeout_ms":163000}`), + }} + + if got, want := handler.remoteToneMapProbeTimeoutV3(nodeURL), 163*time.Second; got != want { + t.Fatalf("cold probe timeout = %s, want the node's advertised %s", got, want) + } + if remoteNodeProbeFallbackTimeout >= 163*time.Second { + t.Fatal("fixture is inert: the advertised budget must exceed the flat fallback") + } +} + +// A node registered since the last capability fetch has no stored report to read +// a budget from, but it does have its own acceleration override — and that is +// what decides how long its walk takes. +func TestPlaybackV3ColdNodeProbeBudgetFollowsTheOverrideWithoutAReport(t *testing.T) { + const nodeURL = "https://gpu-2.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130" + backend := tonemap.BackendQSV + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 2, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != want { + t.Fatalf("cold probe timeout = %s, want the override's %s", got, want) + } + if cluster := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); want <= cluster { + t.Fatalf("fixture is inert: the override budget %s must exceed the cluster's %s", want, cluster) + } +} diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index 6f5a18d30..f4372034d 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -493,13 +493,38 @@ func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Durat if timeout > 0 { return timeout } - cfg := p.config() - if cfg == nil { - return playback.CapabilityRequestTimeout("", "") + // Cold. The cluster setting describes the cluster, not this node: a node + // overridden onto four devices walks four, and pricing it at the cluster's + // one cancels its matrix before its own deadline — which drops it from the + // capability map and sends the download local, or fails it outright where + // local fallback is off. Its own stored report and its own override are + // what describe it. + var node *nodepool.Node + if lookup, ok := p.planner.(transcodeNodeLookup); ok { + if found, ok := lookup.TranscodeNodeByURL(nodeURL); ok { + node = found + } + } + hwAccel, hwDevice := "", "" + if cfg := p.config(); cfg != nil { + hwAccel, hwDevice = cfg.Playback.HWAccel, cfg.Playback.HWDevice } // The whole capability read, not just its tone-map half: the node runs a - // hardware walk first, and that walk scales with the configured device set. - return playback.CapabilityRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice) + // hardware walk first, and that walk scales with the device set it walks. + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(hwAccel), + node.EffectiveHWDevice(hwDevice), + playback.CapabilityRequestTimeout(hwAccel, hwDevice), + ) +} + +// transcodeNodeLookup resolves the pooled record behind a transcode node URL, +// which carries that node's stored capability report and its acceleration +// override. Optional, like the planner's other capabilities: without it this +// path falls back to the cluster-wide setting. *nodepool.Planner implements it. +type transcodeNodeLookup interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) } // cacheToneMapCapabilityFailure negatively caches an unreachable or invalid diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index 88483dc7a..10a6c03dc 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -1027,3 +1027,64 @@ func TestNodeAwarePreparerDoesNotFallBackWhenRecoveryProbeIsIndeterminate(t *tes t.Fatalf("local calls = %d, want 0", local.calls) } } + +// nodeLookupPlanner is a planner that can resolve a node by URL, as the real +// one does. It reserves nothing: these tests only price probes. +type nodeLookupPlanner struct { + node *nodepool.Node +} + +func (p *nodeLookupPlanner) ReserveTranscodeWork(string) (*nodepool.Node, func()) { + return nil, func() {} +} + +func (p *nodeLookupPlanner) TranscodeNode(int) (*nodepool.Node, bool) { return nil, false } + +func (p *nodeLookupPlanner) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if p.node == nil || p.node.URL != nodeURL { + return nil, false + } + return p.node, true +} + +// The cluster setting describes the cluster. A node overridden onto four render +// devices walks four of them cold, and pricing that walk at the cluster's single +// device cancels it partway — which drops the node from the capability map and +// sends the download local, or fails it where local fallback is off. +func TestNodeAwarePreparerColdProbeBudgetFollowsTheNodeOverride(t *testing.T) { + const nodeURL = "https://node.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + cfg := &config.Config{} + cfg.Playback.HWAccel = tonemap.BackendQSV + cfg.Playback.HWDevice = "/dev/dri/renderD128" + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != want { + t.Fatalf("cold probe timeout = %s, want the node's four-device budget %s", got, want) + } + if cluster := playback.CapabilityRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice); want <= cluster { + t.Fatalf("fixture is inert: the override budget %s must exceed the cluster's %s", want, cluster) + } +} + +// What the node last advertised beats what its policy would be priced at: it is +// the node's own measurement of its own matrix, and it survives an API restart +// because it is stored with the report. +func TestNodeAwarePreparerColdProbeBudgetPrefersTheStoredAdvertisement(t *testing.T) { + const nodeURL = "https://node.example" + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, + Capabilities: json.RawMessage(`{"resolved":"qsv","probe_request_timeout_ms":161000}`), + }} + cfg := &config.Config{} + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + + if got, want := preparer.remoteToneMapProbeTimeout(nodeURL), 161*time.Second; got != want { + t.Fatalf("cold probe timeout = %s, want the advertised %s", got, want) + } +} diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go index 597e0751a..3a3a5f682 100644 --- a/internal/nodemetrics/collector.go +++ b/internal/nodemetrics/collector.go @@ -64,6 +64,10 @@ var ( "streamapp_node_gpu_render_busy_percent", "GPU render engine busy percentage. From DRM fdinfo this covers only this node's own transcodes.", []string{gpuDeviceLabel}, nil) + descGPUTotalBusy = prometheus.NewDesc( + "streamapp_node_gpu_busy_percent", + "Whole-GPU utilization including workloads from other tenants, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) descGPUSessions = prometheus.NewDesc( "streamapp_node_gpu_sessions", "Active GPU workloads this node has pinned to a device.", @@ -154,6 +158,14 @@ func (c collector) Collect(ch chan<- prometheus.Metric) { if gpu.RenderBusyPct != nil { gauge(descGPURenderBusy, float64(*gpu.RenderBusyPct), gpu.Device) } + // The engine readings above describe this node's own work when they come + // from fdinfo; this one describes the card. A shared GPU saturated by + // another tenant is the case where they disagree, and it is the one an + // operator most needs to alert on — a transcode that will not get the + // silicon it was planned onto. + if gpu.TotalBusyPct != nil { + gauge(descGPUTotalBusy, float64(*gpu.TotalBusyPct), gpu.Device) + } // Sessions always ships: it comes from this process's own workload // accounting, not from a driver, so it is exact whatever the driver can // or cannot tell us — and a busy GPU with no engine reading is precisely diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go index 2d60333aa..869285915 100644 --- a/internal/nodemetrics/collector_test.go +++ b/internal/nodemetrics/collector_test.go @@ -58,6 +58,7 @@ func TestCollectorExposesSnapshot(t *testing.T) { "streamapp_node_network_rx_bps", "streamapp_node_network_tx_bps", "streamapp_node_gpu_video_busy_percent", + "streamapp_node_gpu_busy_percent", "streamapp_node_gpu_sessions", "streamapp_node_gpu_vram_used_bytes", "streamapp_node_gpu_vram_total_bytes", @@ -263,11 +264,34 @@ func TestCollectorOmitsUnmeasuredGPUEngineGauges(t *testing.T) { if _, present := values["streamapp_node_gpu_render_busy_percent"]; present { t.Fatal("an unmeasured GPU exported a render busy percentage") } + if _, present := values["streamapp_node_gpu_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a whole-GPU utilization") + } if got := values["streamapp_node_gpu_sessions"]; got != 2 { t.Fatalf("gpu sessions = %v, want the workload count exported regardless", got) } } +// Whole-GPU utilization is what nvidia-smi can see and fdinfo cannot: the card's +// own busyness, other tenants included. Without it an operator watching only the +// engine gauges sees this node's idle transcoder and no sign that the card it is +// planned onto is saturated by someone else. +func TestCollectorExportsWholeGPUUtilization(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "cuda:0", Vendor: vendorNVIDIA, Sessions: 0, + VideoBusyPct: ptr(3), TotalBusyPct: ptr(94), Source: SourceNVIDIASMI, + }}, + }) + + values := gatherNames(t, sampler) + if got := values["streamapp_node_gpu_busy_percent"]; got != 94 { + t.Fatalf("whole-GPU busy = %v, want the card's 94 rather than this node's 3", got) + } +} + // A measured source exports both engines, including a genuine zero — which does // mean idle. func TestCollectorExportsMeasuredGPUEngineGauges(t *testing.T) { diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index f3ceaf67b..f4e558471 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -79,7 +79,14 @@ type Node struct { // the health check goes on succeeding every 30 seconds. Derived per sweep // rather than persisted — it is an observation about right now, and a // restarted API re-learns it on its first check. - AdvertisedCapabilitiesHash string `json:"advertised_capabilities_hash,omitempty"` + // + // A pointer for three states, not two. nil is "nobody has checked this node + // yet", which every node reads as until the first sweep after a restart, and + // which says nothing about the stored report. A pointer to "" is the health + // check answering with no hash at all — a node downgraded to a build that + // predates capability reports — and that is a node no longer standing behind + // what is stored for it, which a reader must be able to tell from silence. + AdvertisedCapabilitiesHash *string `json:"advertised_capabilities_hash,omitempty"` // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from // Capabilities rather than stored: it is a pure function of that payload, so // a column would only be a second copy that can disagree with it. Two nodes @@ -110,6 +117,33 @@ func (n *Node) EffectiveHWAccel(clusterHWAccel string) string { return clusterHWAccel } +// StoredCapabilities returns this node's last stored capability report, nil-safe +// like the Effective* accessors so a caller whose lookup came up empty prices a +// missing node and a missing report through one path. +func (n *Node) StoredCapabilities() json.RawMessage { + if n == nil { + return nil + } + return n.Capabilities +} + +// EffectiveHWDevice is the device set this node runs under, resolved the same +// way as EffectiveHWAccel. +// +// Its readers care about the size of the set, not the paths: how many devices a +// node walks is what decides how long its cold capability probe takes, and a +// node overridden onto four devices needs several times the budget the cluster +// setting would price for it. +func (n *Node) EffectiveHWDevice(clusterHWDevice string) string { + if n == nil || n.HWDeviceOverride == nil { + return clusterHWDevice + } + if override := strings.TrimSpace(*n.HWDeviceOverride); override != "" { + return override + } + return clusterHWDevice +} + // CreateNodeInput holds the fields for creating a new node. type CreateNodeInput struct { Name string `json:"name"` diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 96e8b7b1d..6c4f0b3cf 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -124,7 +124,10 @@ func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, act clone.ActiveJobs = activeJobs clone.EgressKbps = egressKbps clone.LastHealthCheck = &checkedAt - clone.AdvertisedCapabilitiesHash = advertisedHash + // Always a pointer once a check has happened, empty string included: the + // distinction between "this node reports no hash" and "no one has asked" + // is the whole reason the field is one. + clone.AdvertisedCapabilitiesHash = &advertisedHash // A check that carried no stats clears them rather than keeping the // previous sample: an unreachable node's five-minute-old CPU number // looks live on a dashboard, which is worse than no number at all. The diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 66ad9ce25..3b7204c31 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -2,6 +2,7 @@ package playback import ( "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -410,6 +411,55 @@ func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { return CapabilityEndpointTimeout(hwAccel, hwDevice) + tonemap.ProbeRequestSlack } +// ColdCapabilityRequestTimeout is how long to allow one capability read of a +// node this process has not read successfully yet. +// +// Cold is when the read is slowest — every probe cache on the node is empty and +// the whole matrix runs — so it is exactly the wrong moment to guess low, and +// there are three progressively worse sources to guess from: +// +// - The budget the node itself advertised in the report stored for it. That +// is the node's own measurement of its own matrix, it survives an API +// restart because it is persisted with the report, and it is right even +// when this replica has never spoken to the node. +// - The node's effective acceleration policy — its own override where it has +// one, the cluster setting otherwise — priced here. This is what a node +// registered a minute ago has, and the override matters: a cluster +// configured for one device says nothing about a node overridden onto four. +// - The caller's fallback, for a node with neither. +// +// Getting this low does not slow anything down, it cancels the read: the node +// is dropped from the capability map mid-matrix and playback plans without it. +func ColdCapabilityRequestTimeout(storedReport json.RawMessage, hwAccel, hwDevice string, fallback time.Duration) time.Duration { + if millis := AdvertisedProbeBudgetMillis(storedReport); millis > 0 { + return NormalizeProbeRequestTimeout(millis, fallback) + } + if budget := CapabilityRequestTimeout(hwAccel, hwDevice); budget > 0 { + return budget + } + return fallback +} + +// AdvertisedProbeBudgetMillis reads the probe budget out of a stored capability +// report, or 0 when it names none. +// +// The report is parsed for this one field rather than decoded whole: callers +// that want a budget have no business depending on the shape of an inventory, +// and a report they cannot parse is not a reason to fail — it reads as "no +// budget advertised" and the caller falls back. +func AdvertisedProbeBudgetMillis(storedReport json.RawMessage) int64 { + if len(storedReport) == 0 { + return 0 + } + var advertised struct { + ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms"` + } + if err := json.Unmarshal(storedReport, &advertised); err != nil { + return 0 + } + return advertised.ProbeRequestTimeoutMillis +} + // MaxCapabilityRequestTimeout is the largest budget a node may advertise and be // believed. // diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 07b4bca90..14ead0510 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3969,9 +3969,12 @@ export interface StreamNode { capabilities?: NodeCapabilities | null; capabilities_hash?: string; /** - * The hash the node named on its last health check. It differs from - * `capabilities_hash` while a refetch is outstanding or failing, which is the - * one case a fresh `last_health_check` cannot rule out. + * The hash the node named on its last health check, present only once a check + * has happened. It differs from `capabilities_hash` while a refetch is + * outstanding or failing, which is the one case a fresh `last_health_check` + * cannot rule out; it is present and empty when the node answers with no hash + * at all, as a build predating capability reports does. Absent means nothing + * has asked yet, which says nothing about the stored report. */ advertised_capabilities_hash?: string; /** When `capabilities` was fetched: the age of the inventory, not the health check. */ diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 551b10516..43aa3d8d1 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -36,7 +36,11 @@ import { ConfirmDialog } from "@/components/ConfirmDialog"; import { formatDateTime } from "@/lib/datetime"; import { toggleHWDevice } from "@/lib/hwDevices"; import { cn } from "@/lib/utils"; -import type { NodeHWDeviceRow, ResourceMetric } from "./adminNodesPresentation"; +import type { + NodeCapabilityStaleReason, + NodeHWDeviceRow, + ResourceMetric, +} from "./adminNodesPresentation"; import { HW_ACCEL_INHERIT, HW_ACCEL_OVERRIDE_OPTIONS, @@ -156,6 +160,26 @@ function NodeDriftBadge({ node }: { node: StreamNode }) { ); } +// Each reason names a different thing to go look at, so none of them is worth +// collapsing into a generic "out of date": a contradicted report means the +// refetch is failing, an unreported one means the node no longer speaks +// capabilities at all, and an unconfirmed one means the health check stopped +// landing. +function staleInventoryTitle(reason: NodeCapabilityStaleReason, node: StreamNode): string { + const refreshed = `The inventory below was last refreshed ${formatDateTime(node.capabilities_refreshed_at ?? "")}.`; + switch (reason) { + case "contradicted": + return `This node reports hardware different from what is stored, and the refetch has not landed. ${refreshed}`; + case "unreported": + return `This node no longer reports a hardware inventory, so nothing confirms the one stored for it. ${refreshed}`; + case "unconfirmed": + return ( + `No health check has confirmed this inventory since ${formatDateTime(node.last_health_check ?? "")}. ` + + `It was last refreshed ${formatDateTime(node.capabilities_refreshed_at ?? "")}.` + ); + } +} + function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNode[] }) { const gpu = describeNodeGPU(node); if (gpu.kind === "awaiting") { @@ -194,14 +218,7 @@ function NodeGPUCell({ node, allNodes }: { node: StreamNode; allNodes: StreamNod {gpu.stale && ( stale diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index 4eaf062ec..037165d0c 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -83,7 +83,7 @@ describe("describeNodeGPU", () => { title: "QSV verified by FFmpeg probe on /dev/dri/renderD128.", }, failures: [], - stale: false, + stale: null, }); }); @@ -278,9 +278,9 @@ describe("describeNodeGPU", () => { last_health_check: new Date(NOW - CAPABILITY_STALE_AFTER_MS - 1000).toISOString(), }); - expect(describeNodeGPU(node, NOW)).toMatchObject({ stale: true }); + expect(describeNodeGPU(node, NOW)).toMatchObject({ stale: "unconfirmed" }); // The same node read earlier was still being checked: the clock decides. - expect(describeNodeGPU(node, NOW - CAPABILITY_STALE_AFTER_MS)).toMatchObject({ stale: false }); + expect(describeNodeGPU(node, NOW - CAPABILITY_STALE_AFTER_MS)).toMatchObject({ stale: null }); }); // The sweep refetches only when a node advertises a changed hash, so an @@ -296,7 +296,7 @@ describe("describeNodeGPU", () => { NOW, ); - expect(presentation).toMatchObject({ stale: false }); + expect(presentation).toMatchObject({ stale: null }); }); it("does not call an unhealthy node's report stale", () => { @@ -310,12 +310,12 @@ describe("describeNodeGPU", () => { NOW, ); - expect(presentation).toMatchObject({ stale: false }); + expect(presentation).toMatchObject({ stale: null }); }); it("is not stale when the server sent no refresh timestamp", () => { expect(describeNodeGPU(makeNode({ capabilities: { resolved: "qsv" } }), NOW)).toMatchObject({ - stale: false, + stale: null, }); }); @@ -329,7 +329,7 @@ describe("describeNodeGPU", () => { NOW, ); - expect(presentation).toMatchObject({ stale: false }); + expect(presentation).toMatchObject({ stale: null }); }); it("reports no live devices for a node whose server sends no last_stats", () => { @@ -1197,7 +1197,31 @@ describe("capability report staleness from an unconfirmed hash", () => { advertised_capabilities_hash: "sha256:newer", } as StreamNode; const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); - expect(gpu.kind === "reported" && gpu.stale).toBe(true); + expect(gpu.kind === "reported" && gpu.stale).toBe("contradicted"); + }); + + // A node downgraded to a build that predates capability reports answers health + // checks with no hash at all. It is not confirming the stored inventory any + // more than a mismatching node is, and the timestamp rule alone would present + // that inventory as current for as long as the node keeps answering. + it("marks a report stale when a checked node advertises no hash", () => { + const node = { + ...base, + capabilities_hash: "sha256:stored", + advertised_capabilities_hash: "", + } as StreamNode; + const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); + expect(gpu.kind === "reported" && gpu.stale).toBe("unreported"); + }); + + // Absent is not empty: until the first sweep after a restart every node reads + // that way, and marking them all stale would be a warning about the API rather + // than about any node. + it("leaves an unchecked node to the timestamp rule", () => { + const node = { ...base, capabilities_hash: "sha256:stored" } as StreamNode; + delete (node as { advertised_capabilities_hash?: string }).advertised_capabilities_hash; + const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); + expect(gpu.kind === "reported" && gpu.stale).toBe(null); }); it("leaves a matching hash alone on a freshly checked node", () => { @@ -1207,15 +1231,15 @@ describe("capability report staleness from an unconfirmed hash", () => { advertised_capabilities_hash: "sha256:stored", } as StreamNode; const gpu = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); - expect(gpu.kind === "reported" && gpu.stale).toBe(false); + expect(gpu.kind === "reported" && gpu.stale).toBe(null); }); // A node that never advertises one — an older build — keeps the timestamp rule. it("falls back to the health-check age when no hash is advertised", () => { const node = { ...base, capabilities_hash: "sha256:stored" } as StreamNode; const fresh = describeNodeGPU(node, Date.parse("2026-08-28T00:00:10Z")); - expect(fresh.kind === "reported" && fresh.stale).toBe(false); + expect(fresh.kind === "reported" && fresh.stale).toBe(null); const aged = describeNodeGPU(node, Date.parse("2026-08-28T00:20:00Z")); - expect(aged.kind === "reported" && aged.stale).toBe(true); + expect(aged.kind === "reported" && aged.stale).toBe("unconfirmed"); }); }); diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index 9c8f03475..607cb9ac9 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -56,8 +56,8 @@ export type NodeGPUPresentation = deviceSummary: string | null; /** Full device paths, one per line, for the summary's tooltip. */ deviceTitle: string | null; - /** No health check has re-confirmed this report recently. */ - stale: boolean; + /** Why this report is no longer trustworthy as current; null when it is. */ + stale: NodeCapabilityStaleReason | null; /** * Live per-device readings from the node's last health check, matched * against the capability inventory. Empty when the node reports no @@ -67,6 +67,18 @@ export type NodeGPUPresentation = live: NodeGPULiveDevice[]; }; +/** + * Why a stored capability report is not being treated as current. + * + * - `contradicted` — the node reports a different hash than the one stored, so + * the stored inventory describes hardware the node itself disagrees with. + * - `unreported` — the node answers health checks but names no hash at all, as + * a build predating capability reports does. It is not standing behind the + * stored inventory either; it simply cannot say. + * - `unconfirmed` — no health check has re-confirmed the report recently enough. + */ +export type NodeCapabilityStaleReason = "contradicted" | "unreported" | "unconfirmed"; + /** One GPU's live reading, as rendered next to the capability inventory. */ export interface NodeGPULiveDevice { /** Stable key for list rendering; the device id as the node reported it. */ @@ -112,7 +124,7 @@ export function describeNodeGPU(node: StreamNode, now: number = Date.now()): Nod failures: otherFailures(resolved, capabilities.detected_backends ?? []), deviceSummary: devices.summary, deviceTitle: devices.title, - stale: isCapabilityReportStale(node, now), + stale: capabilityReportStaleness(node, now), live: describeLiveGPUs(node), }; } @@ -319,32 +331,38 @@ function failureReason(entry: NodeDetected): string { return entry.reason?.trim() || "no reason reported"; } -function isCapabilityReportStale(node: StreamNode, now: number): boolean { +function capabilityReportStaleness( + node: StreamNode, + now: number, +): NodeCapabilityStaleReason | null { // An unhealthy node cannot refresh its report; calling that stale would blame // the inventory for the outage the Health column already shows. if (!node.healthy) { - return false; + return null; } if (Number.isNaN(Date.parse(node.capabilities_refreshed_at ?? ""))) { - return false; + return null; } - // The node says its hardware is something other than what is stored. The - // sweep refetches on that mismatch, so seeing it here means the refetch has - // not landed — and a health check that keeps succeeding every 30 seconds - // would otherwise present a report we already know is obsolete as current. - // This is the one staleness a timestamp cannot see. + // What the node said about itself on its last health check. Absent means no + // check has happened here yet — every node reads that way until the first + // sweep after a restart — which is silence, not disagreement. Present and + // empty is the node answering with no hash at all, and that is a node not + // standing behind the stored report any more than a mismatching one does: + // both leave an inventory nothing currently confirms while the health check + // goes on succeeding every 30 seconds. This is the staleness a timestamp + // cannot see. const advertised = node.advertised_capabilities_hash?.trim(); - if (advertised && advertised !== node.capabilities_hash?.trim()) { - return true; + if (advertised !== undefined && advertised !== (node.capabilities_hash?.trim() ?? "")) { + return advertised === "" ? "unreported" : "contradicted"; } // Measured against the health check, not against the report's own age: the // check is what re-confirms the report, and it is the only one of the two // that moves on a node whose hardware never changes. const lastCheck = Date.parse(node.last_health_check ?? ""); if (Number.isNaN(lastCheck)) { - return false; + return null; } - return now - lastCheck > CAPABILITY_STALE_AFTER_MS; + return now - lastCheck > CAPABILITY_STALE_AFTER_MS ? "unconfirmed" : null; } function summarizeRenderDevices(capabilities: NodeCapabilities): { From 1cacc21eb0cda68e8737f7fecfac04bab15a7a3b Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 02:25:55 -0400 Subject: [PATCH 064/163] fix(playback): floor a learned probe budget at the node's current policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A budget learned from a node is preserved across invalidations on purpose: an invalidation is the moment the next read is coldest, so dropping the budget with the inventory would fall back to a figure short of what the node needs. But a per-node policy edit invalidates through the same path, and widening hw_device_override is exactly the change that makes the learned number too small. The node reloads, walks four devices instead of one, and gets canceled at the one-device deadline — on every retry, with no way out, because a budget is only ever learned from a read that completes. Protocol-v3 planning and prepared downloads would both drop that node indefinitely: playback plans without it, and the download falls back locally or fails outright where local fallback is off. Both paths now take the larger of what was learned and what the node currently describes, and ColdCapabilityRequestTimeout composes its own two sources the same way. Neither dominates: the node's advertisement is its own measurement of its own matrix and survives a restart, but it is as old as the report it came with, while the policy price moves the instant an operator edits the node and is a floor rather than the truth — it is computed on an API replica that has none of the node's cards, so device classification there falls back to the cheapest backend. The cost of being too generous is holding a dead node's fetch open a few seconds longer, bounded by MaxCapabilityRequestTimeout and by the planning timeout above it. The cost of being too small is losing the node. TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget priced a two-device cluster above the node's own 137s, so under the new floor it was asserting the floor rather than the node; it now uses a single device and fails if that stops being true. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/playback_v3.go | 25 +++++---- internal/api/handlers/playback_v3_test.go | 62 +++++++++++++++++++++- internal/downloads/remote_preparer.go | 30 +++++++---- internal/downloads/remote_preparer_test.go | 58 ++++++++++++++++++-- internal/playback/gpudetect.go | 44 +++++++++------ 5 files changed, 177 insertions(+), 42 deletions(-) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 40f957346..65f330c59 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -323,16 +323,21 @@ func (h *PlaybackHandler) remoteToneMapProbeTimeoutV3(nodeURL string) time.Durat h.v3NodeCapabilitiesMu.Lock() budget := h.v3NodeProbeBudgets[nodeURL] h.v3NodeCapabilitiesMu.Unlock() - if budget > 0 { - return budget - } - // Nothing learned from this node yet, which is every node after an API - // restart and any node registered since. The durable report holds what it - // last advertised, and its override holds what it would advertise; the flat - // fallback below is shorter than a two-device node's matrix legitimately - // takes, so reaching for it first would cancel exactly the multi-GPU nodes - // this path most wants to keep. - return h.coldNodeProbeTimeoutV3(nodeURL) + // Never less than what this node currently describes, whatever was learned + // from it before. + // + // A learned budget is kept across invalidations on purpose — an invalidation + // is the moment the next read is coldest and slowest, so dropping the budget + // with the inventory would fall back to a figure short of what the node + // needs. But a per-node policy edit invalidates through the same path, and + // widening hw_device_override is precisely a change that makes the learned + // number too small: the node reloads, walks four devices instead of one, and + // gets canceled at the one-device deadline. Nothing recovers from that on its + // own, because a budget is only ever learned from a read that completes. + if cold := h.coldNodeProbeTimeoutV3(nodeURL); cold > budget { + return cold + } + return budget } // coldNodeProbeTimeoutV3 prices one capability read of a node this process has diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 5bb3538c3..fcf25568b 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -5524,12 +5524,18 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { defer remote.Close() handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) handler.PlaybackConfig = func() config.PlaybackConfig { - return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/central/device/one,/central/device/two"} + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/central/device/one"} } if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), remote.URL, false); err != nil { t.Fatal(err) } + // The node's own figure is what this path exists to use. It has to exceed + // what the cluster setting prices, or the floor below it would be answering + // and this would pass without the target node being consulted at all. + if cluster := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/central/device/one"); cluster >= 137*time.Second { + t.Fatalf("fixture is inert: the cluster price %s must stay under the node's 137s", cluster) + } if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), 137*time.Second; got != want { t.Fatalf("remote probe timeout = %s, want target node budget %s", got, want) } @@ -5617,3 +5623,57 @@ func TestPlaybackV3ColdNodeProbeBudgetFollowsTheOverrideWithoutAReport(t *testin t.Fatalf("fixture is inert: the override budget %s must exceed the cluster's %s", want, cluster) } } + +// A per-node policy edit invalidates the inventory through the same path a +// hardware change does, and the learned budget deliberately survives that. But +// widening hw_device_override is exactly the change that makes the learned +// number too small: the node reloads, walks four devices instead of one, and is +// canceled at the one-device deadline on every retry — with no way back, since a +// budget is only learned from a read that completes. +func TestPlaybackV3RepricesALearnedBudgetAfterTheDeviceSetGrows(t *testing.T) { + const nodeURL = "https://gpu-3.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 3, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + // Learned while the node was still on the cluster's single device. + learned := playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128") + handler.v3NodeCapabilitiesMu.Lock() + handler.rememberNodeProbeBudgetLockedV3(nodeURL, learned) + handler.v3NodeCapabilitiesMu.Unlock() + + handler.RefreshNodeCapabilitiesV3(nodeURL) + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != want { + t.Fatalf("probe timeout after the override grew = %s, want the four-device %s", got, want) + } + if want <= learned { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the learned %s", want, learned) + } +} + +// The reverse must still hold: a node whose own measurement exceeds what this +// replica can price keeps its measurement, which is the whole reason the learned +// budget survives an invalidation. +func TestPlaybackV3KeepsALearnedBudgetLargerThanThePolicyPrice(t *testing.T) { + const nodeURL = "https://gpu-4.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ID: 4, URL: nodeURL}} + learned := playback.MaxCapabilityRequestTimeout() + handler.v3NodeCapabilitiesMu.Lock() + handler.rememberNodeProbeBudgetLockedV3(nodeURL, learned) + handler.v3NodeCapabilitiesMu.Unlock() + + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != learned { + t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) + } +} diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index f4372034d..cfded198b 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -490,15 +490,21 @@ func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Durat p.capabilityMu.Lock() timeout := p.capabilities[nodeURL].probeRequestTimeout p.capabilityMu.Unlock() - if timeout > 0 { - return timeout - } - // Cold. The cluster setting describes the cluster, not this node: a node - // overridden onto four devices walks four, and pricing it at the cluster's - // one cancels its matrix before its own deadline — which drops it from the - // capability map and sends the download local, or fails it outright where - // local fallback is off. Its own stored report and its own override are - // what describe it. + // The larger of what was learned from this node and what it currently + // describes; neither dominates. + // + // A learned budget is preserved across failures on purpose, so a cold retry + // is not cut short by a fallback — but it describes the node as it was, and + // an operator who widens hw_device_override leaves one behind that prices a + // smaller device set than the node now walks. Every retry would be canceled + // at that deadline, and since a budget is only ever learned from a read that + // completes, nothing would replace it. + // + // What the node currently describes is its own stored report and its own + // override — not the cluster setting, which says nothing about a node + // overridden onto four devices. Pricing four at the cluster's one cancels the + // matrix mid-walk, which drops the node from the capability map and sends the + // download local, or fails it outright where local fallback is off. var node *nodepool.Node if lookup, ok := p.planner.(transcodeNodeLookup); ok { if found, ok := lookup.TranscodeNodeByURL(nodeURL); ok { @@ -511,12 +517,16 @@ func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Durat } // The whole capability read, not just its tone-map half: the node runs a // hardware walk first, and that walk scales with the device set it walks. - return playback.ColdCapabilityRequestTimeout( + cold := playback.ColdCapabilityRequestTimeout( node.StoredCapabilities(), node.EffectiveHWAccel(hwAccel), node.EffectiveHWDevice(hwDevice), playback.CapabilityRequestTimeout(hwAccel, hwDevice), ) + if cold > timeout { + return cold + } + return timeout } // transcodeNodeLookup resolves the pooled record behind a transcode node URL, diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index 10a6c03dc..830c84f87 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -1072,10 +1072,10 @@ func TestNodeAwarePreparerColdProbeBudgetFollowsTheNodeOverride(t *testing.T) { } } -// What the node last advertised beats what its policy would be priced at: it is -// the node's own measurement of its own matrix, and it survives an API restart -// because it is stored with the report. -func TestNodeAwarePreparerColdProbeBudgetPrefersTheStoredAdvertisement(t *testing.T) { +// What the node last advertised is its own measurement of its own matrix, and it +// survives an API restart because it is stored with the report — so where it +// exceeds what this replica can price, it is the answer. +func TestNodeAwarePreparerColdProbeBudgetTakesTheStoredAdvertisement(t *testing.T) { const nodeURL = "https://node.example" planner := &nodeLookupPlanner{node: &nodepool.Node{ ID: 1, URL: nodeURL, @@ -1088,3 +1088,53 @@ func TestNodeAwarePreparerColdProbeBudgetPrefersTheStoredAdvertisement(t *testin t.Fatalf("cold probe timeout = %s, want the advertised %s", got, want) } } + +// A budget learned before an operator widened the node's device set describes +// the node as it was. Keeping it would cancel every cold retry at the old +// one-device deadline — and a budget is only ever learned from a read that +// completes, so nothing would replace it. +func TestNodeAwarePreparerRepricesALearnedBudgetAfterTheDeviceSetGrows(t *testing.T) { + const nodeURL = "https://node.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + cfg := &config.Config{} + cfg.Playback.HWAccel = tonemap.BackendQSV + cfg.Playback.HWDevice = "/dev/dri/renderD128" + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + // What the node advertised while it was still on one device. + learned := playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128") + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + probeRequestTimeout: learned, + expiresAt: time.Now().Add(-time.Second), + } + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != want { + t.Fatalf("probe timeout after the override grew = %s, want the four-device %s", got, want) + } + if want <= learned { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the learned %s", want, learned) + } +} + +// The other direction: a node whose own measurement exceeds what this replica +// can price for it keeps that measurement. An API replica has none of the node's +// cards, so its pricing is a floor rather than the truth. +func TestNodeAwarePreparerKeepsALearnedBudgetLargerThanThePolicyPrice(t *testing.T) { + const nodeURL = "https://node.example" + planner := &nodeLookupPlanner{node: &nodepool.Node{ID: 1, URL: nodeURL}} + cfg := &config.Config{} + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + learned := playback.MaxCapabilityRequestTimeout() + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + probeRequestTimeout: learned, + expiresAt: time.Now().Add(-time.Second), + } + + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != learned { + t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 3b7204c31..039b25402 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -415,29 +415,39 @@ func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { // node this process has not read successfully yet. // // Cold is when the read is slowest — every probe cache on the node is empty and -// the whole matrix runs — so it is exactly the wrong moment to guess low, and -// there are three progressively worse sources to guess from: +// the whole matrix runs — so it is exactly the wrong moment to guess low. +// Getting it low does not slow the read down, it cancels it: the node drops out +// of the capability map mid-matrix and playback plans without it. // -// - The budget the node itself advertised in the report stored for it. That -// is the node's own measurement of its own matrix, it survives an API -// restart because it is persisted with the report, and it is right even -// when this replica has never spoken to the node. -// - The node's effective acceleration policy — its own override where it has -// one, the cluster setting otherwise — priced here. This is what a node -// registered a minute ago has, and the override matters: a cluster -// configured for one device says nothing about a node overridden onto four. -// - The caller's fallback, for a node with neither. +// Two sources describe the same node and neither dominates, so the answer is +// the larger: // -// Getting this low does not slow anything down, it cancels the read: the node -// is dropped from the capability map mid-matrix and playback plans without it. +// - What the node advertised in the report stored for it. That is the node's +// own measurement of its own matrix, it survives an API restart because it +// is persisted with the report, and it is right even when this replica has +// never spoken to the node. It is also as old as the report: an operator who +// has just widened the node's device set has invalidated it. +// - What the node's effective acceleration policy prices — its own override +// where it has one, the cluster setting otherwise. This moves the moment an +// operator edits the node, before any refetch can land, which is exactly +// when the stored figure is wrong. But it is priced *here*, on an API +// replica that does not have the node's cards, so device classification +// falls back to the cheapest backend and it reads as a floor rather than as +// the truth. +// +// The fallback covers a node with neither: no stored report and no policy. func ColdCapabilityRequestTimeout(storedReport json.RawMessage, hwAccel, hwDevice string, fallback time.Duration) time.Duration { + budget := time.Duration(0) if millis := AdvertisedProbeBudgetMillis(storedReport); millis > 0 { - return NormalizeProbeRequestTimeout(millis, fallback) + budget = NormalizeProbeRequestTimeout(millis, fallback) + } + if priced := CapabilityRequestTimeout(hwAccel, hwDevice); priced > budget { + budget = priced } - if budget := CapabilityRequestTimeout(hwAccel, hwDevice); budget > 0 { - return budget + if budget <= 0 { + return fallback } - return fallback + return budget } // AdvertisedProbeBudgetMillis reads the probe budget out of a stored capability From 28a37f28503e4a14fe0958a259345c96826a78df Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 02:45:49 -0400 Subject: [PATCH 065/163] fix(nodes): reprice re-probes, and see NVIDIA-only cards disappear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings. A re-probe deliberately discards every cache on the node, so it runs the full cold matrix for the device set that node is configured for now — but its budget came from the report stored for it, which an operator who widened hw_device_override has already made obsolete. The API then cancels the very request that would have replaced that report, and unlike the planning and download paths there is nothing to learn from: a re-probe is the request that never completes. It now goes through ColdCapabilityRequestTimeout like the others, with the node's own override resolved against the cluster policy the router hands in. That also settles what the fallback in that helper means. Every source is a lower bound on what the read may need, so the answer is the largest of them, the fallback included — a node that has never been inventoried is the one most likely to be slow, and a policy that happens to price lower must not shorten what the caller was willing to spend on it. Second: the ordinary NVENC container has /dev/nvidia* and the toolkit and no /dev/dri, so its cards exist only in nvidia_gpu_uuids. A card disappearing from such a node moved nothing drift looked at — render_devices was empty to begin with — and the backend comparison did not cover it either, because NVENC stops being a candidate the moment the device nodes go away and an absent backend is deliberately not a lost one. A four-GPU node losing one went unreported. Those uuids now join the device comparison, deduplicated against render_device_details by uuid so a card with both identities is one card. The suppression that comes with it is the same "identity strength is not constant" problem one level up: nvidia-smi sits behind a circuit breaker and can be missing from an image outright, so an empty uuid list is not evidence a card is gone. NVENC is only probed where /dev/nvidia* opens, so a report still carrying that backend describes a node whose cards are present and whose query tool is not, and its uuid losses are ignored. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 48 ++++++++++-- internal/api/handlers/nodes_reprobe_test.go | 48 ++++++++++-- internal/api/router.go | 5 ++ internal/nodepool/health.go | 82 ++++++++++++++++++--- internal/nodepool/health_drift_test.go | 68 +++++++++++++++++ internal/playback/gpudetect.go | 21 ++++-- 6 files changed, 243 insertions(+), 29 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 3e376756c..38657d705 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -16,6 +16,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" @@ -70,6 +71,31 @@ type NodeHandler struct { // finished. Tests wait on it rather than on a sleep; production leaves it // nil. afterNodeUpdate func() + // clusterPlayback reports the cluster-wide acceleration policy, which is + // what a node without an override of its own runs. Read live rather than + // snapshotted, because it is hot-reloadable. nil where nothing wired it, and + // then a node's own override is all this handler can price from. + clusterPlayback func() config.PlaybackConfig +} + +// SetClusterPlaybackPolicy wires the live cluster-wide acceleration policy, +// which the re-probe budget needs to resolve a node that carries no override of +// its own. Set after construction like the other collaborators here. +func (h *NodeHandler) SetClusterPlaybackPolicy(policy func() config.PlaybackConfig) { + if h == nil { + return + } + h.clusterPlayback = policy +} + +// playbackPolicy is the cluster-wide acceleration policy, or the zero policy +// where none was wired — which prices the default device set rather than +// nothing. +func (h *NodeHandler) playbackPolicy() config.PlaybackConfig { + if h == nil || h.clusterPlayback == nil { + return config.PlaybackConfig{} + } + return h.clusterPlayback() } // SetCapabilityInvalidator wires the planning-cache drop used after a node's @@ -624,9 +650,21 @@ const nodeReprobeFallbackTimeout = 150 * time.Second // capability report (probe_request_timeout_ms), which is exactly what a cold // capability fetch is given; a node with different hardware or a slower ffmpeg // therefore gets its own number rather than a cluster-wide guess. -func nodeReprobeTimeout(n *nodepool.Node) time.Duration { - return playback.NormalizeProbeRequestTimeout( - playback.AdvertisedProbeBudgetMillis(n.StoredCapabilities()), nodeReprobeFallbackTimeout) +// +// Floored at what the node's current policy prices, for the same reason the +// planning and download paths are: a report stored before an operator widened +// hw_device_override advertises a budget for the smaller device set, and a +// re-probe deliberately discards every cache on the node, so it runs the larger +// matrix and gets canceled at the old deadline. Nothing here learns its way out +// of that — a re-probe is exactly the request that never completes. +func (h *NodeHandler) nodeReprobeTimeout(n *nodepool.Node) time.Duration { + cluster := h.playbackPolicy() + return playback.ColdCapabilityRequestTimeout( + n.StoredCapabilities(), + n.EffectiveHWAccel(cluster.HWAccel), + n.EffectiveHWDevice(cluster.HWDevice), + nodeReprobeFallbackTimeout, + ) } // nodeReprobeWriteSlack covers the repository round trips and the JSON write @@ -695,7 +733,7 @@ func (h *NodeHandler) HandleReprobeNode(w http.ResponseWriter, r *http.Request) return } - extendReprobeWriteDeadline(w, r, nodeReprobeTimeout(node)) + extendReprobeWriteDeadline(w, r, h.nodeReprobeTimeout(node)) result := ReprobeNodeResult{NodeID: node.ID, NodeName: node.Name, Status: "ok"} reprobed, err := h.reprobeNode(r.Context(), node) @@ -736,7 +774,7 @@ type nodeReprobeResponse struct { // reprobeNode performs the bearer-authenticated re-probe call against one node, // mirroring the force-reload client. func (h *NodeHandler) reprobeNode(ctx context.Context, node *nodepool.Node) (nodeReprobeResponse, error) { - timeout := nodeReprobeTimeout(node) + timeout := h.nodeReprobeTimeout(node) client := &http.Client{Timeout: timeout} ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() diff --git a/internal/api/handlers/nodes_reprobe_test.go b/internal/api/handlers/nodes_reprobe_test.go index 046af26ba..195d13860 100644 --- a/internal/api/handlers/nodes_reprobe_test.go +++ b/internal/api/handlers/nodes_reprobe_test.go @@ -10,8 +10,10 @@ import ( "testing" "time" + "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/go-chi/chi/v5" ) @@ -245,18 +247,54 @@ func TestHandleReprobeNodeUnknownNode(t *testing.T) { // slower hardware is not abandoned on a cluster-wide guess. A node that has // never been inventoried falls back to the generous constant. func TestNodeReprobeTimeoutPrefersTheNodeAdvertisedBudget(t *testing.T) { - advertised := playback.HWAccelInfo{ProbeRequestTimeoutMillis: 111_000} + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + // Above the fallback, which is a floor: a node that says it needs less than + // the constant this handler is willing to spend does not shorten it. + advertised := playback.HWAccelInfo{ProbeRequestTimeoutMillis: 200_000} payload, err := json.Marshal(advertised) if err != nil { t.Fatal(err) } - if got := nodeReprobeTimeout(&nodepool.Node{Capabilities: payload}); got != 111*time.Second { - t.Fatalf("timeout = %s, want the node-advertised 111s", got) + if nodeReprobeFallbackTimeout >= 200*time.Second { + t.Fatalf("fixture is inert: the fallback %s must stay under the advertised 200s", nodeReprobeFallbackTimeout) + } + if got := handler.nodeReprobeTimeout(&nodepool.Node{Capabilities: payload}); got != 200*time.Second { + t.Fatalf("timeout = %s, want the node-advertised 200s", got) } - if got := nodeReprobeTimeout(&nodepool.Node{}); got != nodeReprobeFallbackTimeout { + if got := handler.nodeReprobeTimeout(&nodepool.Node{}); got != nodeReprobeFallbackTimeout { t.Fatalf("timeout = %s, want the fallback %s for a node with no report", got, nodeReprobeFallbackTimeout) } - if got := nodeReprobeTimeout(&nodepool.Node{Capabilities: json.RawMessage(`not json`)}); got != nodeReprobeFallbackTimeout { + if got := handler.nodeReprobeTimeout(&nodepool.Node{Capabilities: json.RawMessage(`not json`)}); got != nodeReprobeFallbackTimeout { t.Fatalf("timeout = %s, want the fallback for an unreadable report", got) } } + +// A re-probe discards every cache on the node, so it runs the full matrix for +// whatever device set the node is configured for *now*. A report stored before +// an operator widened hw_device_override advertises the old, smaller budget, and +// honoring it would cancel the very request that would have replaced it. +func TestNodeReprobeTimeoutRepricesAWidenedDeviceOverride(t *testing.T) { + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + handler.SetClusterPlaybackPolicy(func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + }) + stored, err := json.Marshal(playback.HWAccelInfo{ + ProbeRequestTimeoutMillis: playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128").Milliseconds(), + }) + if err != nil { + t.Fatal(err) + } + + want := playback.CapabilityRequestTimeout(backend, devices) + got := handler.nodeReprobeTimeout(&nodepool.Node{ + Capabilities: stored, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }) + if got != want { + t.Fatalf("re-probe timeout = %s, want the four-device %s", got, want) + } + if want <= nodeReprobeFallbackTimeout { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the fallback %s", want, nodeReprobeFallbackTimeout) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 12c28b3e7..f14624d4d 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3173,6 +3173,11 @@ func NewRouter(deps Dependencies) chi.Router { // moment it lands; the same invalidation the // health sweep uses drops it. nodeHandler.SetCapabilityInvalidator(playbackHandler.RefreshNodeCapabilitiesV3) + // A node with no override of its own runs the + // cluster's acceleration policy, and how many + // devices that names is what decides how long its + // re-probe may take. + nodeHandler.SetClusterPlaybackPolicy(playbackHandler.PlaybackConfig) r.Route("/nodes", func(r chi.Router) { r.Get("/", nodeHandler.HandleListNodes) r.Post("/", nodeHandler.HandleCreateNode) diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 0cad72a2e..100b436e9 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -546,16 +546,9 @@ func storedCapabilitiesHash(n *Node) string { // nodepool must not depend on playback, and drift only needs to know which // backends were verified and which render devices existed. type capabilityDriftView struct { - Resolved string `json:"resolved"` - DetectedBackends []struct { - Backend string `json:"backend"` - Verified bool `json:"verified"` - // Skipped reports that no probe was attempted because the node cannot - // open any of the backend's candidate devices. That is a statement - // about access, not about hardware, so it never counts as a failure. - Skipped bool `json:"skipped"` - } `json:"detected_backends"` - RenderDevices []string `json:"render_devices"` + Resolved string `json:"resolved"` + DetectedBackends []driftBackendView `json:"detected_backends"` + RenderDevices []string `json:"render_devices"` // RenderDeviceDetails carries each device's stable identity. Comparing // enumeration paths alone reports a GPU as gone whenever DRM hands out a // different renderD number, which a reboot is free to do; the uuid and the @@ -565,6 +558,47 @@ type capabilityDriftView struct { PCIAddress string `json:"pci_address"` GPUUUID string `json:"gpu_uuid"` } `json:"render_device_details"` + // NVIDIAGPUUUIDs is where an NVIDIA card's identity lives when it has no + // readable DRM node — the ordinary NVENC container: /dev/nvidia* and the + // toolkit, no /dev/dri. Such a node reports no render devices at all, so + // without this a card disappearing from it moves nothing this comparison + // looks at, and the backend comparison does not cover it either: NVENC stops + // being a candidate the moment /dev/nvidia* goes away, and an absent backend + // is deliberately not a lost one. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` +} + +// driftBackendView is one backend's probe outcome as drift reads it. +type driftBackendView struct { + Backend string `json:"backend"` + Verified bool `json:"verified"` + // Skipped reports that no probe was attempted because the node cannot open + // any of the backend's candidate devices. That is a statement about access, + // not about hardware, so it never counts as a failure. + Skipped bool `json:"skipped"` +} + +// nvidiaBackend is the backend name whose presence in a report means the node +// still has NVIDIA device nodes; detection drops it as a candidate when they go +// away. +const nvidiaBackend = "nvenc" + +// nvidiaIdentityBlind reports that this report cannot name the node's NVIDIA +// cards even though the node still has them. +// +// nvidia-smi is queried behind a circuit breaker and can be absent from an +// image entirely, so an empty uuid list is not by itself evidence that anything +// is gone — the same "identity strength is not constant" problem renderDeviceAliases +// exists for. What separates the two is the backend list: NVENC is only probed +// where /dev/nvidia* can be opened, so a report that still carries NVENC and no +// uuids is a node whose cards are present and whose query tool is not. +func (v capabilityDriftView) nvidiaIdentityBlind() bool { + if len(v.NVIDIAGPUUUIDs) > 0 { + return false + } + return slices.ContainsFunc(v.DetectedBackends, func(backend driftBackendView) bool { + return backend.Backend == nvidiaBackend + }) } // renderDeviceAliases lists every stable name each device in a report answers @@ -588,6 +622,10 @@ type renderDeviceAliases struct { // machine and outlive the card in them. uuid string aliases []string + // nvidiaOnly marks a card known only through nvidia-smi, with no render + // node behind it. Its identity depends on a tool that comes and goes, so a + // report that lost it has to be read differently — see nvidiaIdentityBlind. + nvidiaOnly bool } // sameDevice reports whether two reports describe one card. @@ -607,8 +645,9 @@ func (a renderDeviceAliases) sameDevice(b renderDeviceAliases) bool { } func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { - devices := make([]renderDeviceAliases, 0, len(view.RenderDevices)) + devices := make([]renderDeviceAliases, 0, len(view.RenderDevices)+len(view.NVIDIAGPUUUIDs)) covered := make(map[string]bool, len(view.RenderDeviceDetails)) + coveredUUIDs := make(map[string]bool, len(view.RenderDeviceDetails)+len(view.NVIDIAGPUUUIDs)) for _, device := range view.RenderDeviceDetails { entry := renderDeviceAliases{path: device.Path, uuid: device.GPUUUID} for _, alias := range []string{device.GPUUUID, device.PCIAddress, device.Path} { @@ -620,6 +659,9 @@ func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { continue } covered[device.Path] = true + if device.GPUUUID != "" { + coveredUUIDs[device.GPUUUID] = true + } devices = append(devices, entry) } // A report that lists paths without details (a node predating them) still @@ -630,6 +672,17 @@ func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { } devices = append(devices, renderDeviceAliases{path: path, aliases: []string{path}}) } + // A card nvidia-smi named and no render node covers. Deduplicated against + // the details above by uuid, because a card with both a DRM node and an + // nvidia-smi entry is one card, not two. It gets no path: the uuid is the + // only name it has, and lostRenderDevices falls back to naming it by that. + for _, uuid := range view.NVIDIAGPUUUIDs { + if uuid == "" || coveredUUIDs[uuid] { + continue + } + coveredUUIDs[uuid] = true + devices = append(devices, renderDeviceAliases{uuid: uuid, aliases: []string{uuid}, nvidiaOnly: true}) + } return devices } @@ -639,11 +692,18 @@ func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { // when it comes back under a different one. func lostRenderDeviceEntries(previous, current capabilityDriftView) []renderDeviceAliases { currentDevices := renderDeviceAliasSets(current) + blind := current.nvidiaIdentityBlind() var lost []renderDeviceAliases for _, device := range renderDeviceAliasSets(previous) { if slices.ContainsFunc(currentDevices, device.sameDevice) { continue } + if blind && device.nvidiaOnly { + // The node still has its NVIDIA devices; only the tool that names + // them is missing. Latching a note here would demand a uuid come + // back that nothing on the node can currently produce. + continue + } lost = append(lost, device) } return lost diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go index f66b1184e..83f1aa050 100644 --- a/internal/nodepool/health_drift_test.go +++ b/internal/nodepool/health_drift_test.go @@ -785,3 +785,71 @@ func TestResolveDriftNoteNamesEveryOutstandingLoss(t *testing.T) { t.Fatalf("capability_drift = %q, want the still-missing card named", *still) } } + +// The ordinary NVENC container has /dev/nvidia* and the toolkit and no /dev/dri, +// so its cards exist only in nvidia_gpu_uuids. Losing one moves nothing in +// render_devices, and the backend comparison does not cover it either: NVENC +// stops being a candidate the moment the device nodes go away, and an absent +// backend is deliberately not a lost one. +func TestCapabilityDriftNotesAnNVIDIAOnlyCardGoingAway(t *testing.T) { + const before = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-aaa","GPU-bbb"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + // One card left. NVENC still verifies on it, so nothing else in the report + // says anything was lost. + const after = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-bbb"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if !drift.regressed() { + t.Fatal("a card that disappeared produced no drift") + } + if got := drift.lostDevices; len(got) != 1 || got[0] != "GPU-aaa" { + t.Fatalf("lost devices = %v, want the missing GPU-aaa", got) + } + if len(drift.lostBackends) != 0 { + t.Fatalf("lost backends = %v, want none — NVENC still verifies on the remaining card", drift.lostBackends) + } +} + +// nvidia-smi sits behind a circuit breaker and can be missing from an image +// outright, so an empty uuid list is not evidence a card is gone. NVENC is only +// probed where /dev/nvidia* opens, so a report still carrying it describes a +// node whose cards are present and whose query tool is not — latching drift +// there would demand a uuid come back that nothing on the node can produce. +func TestCapabilityDriftIgnoresNVIDIAUUIDsLostWithTheQueryTool(t *testing.T) { + const before = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-aaa"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + const after = `{"resolved":"nvenc","detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if drift.regressed() { + t.Fatalf("nvidia-smi going quiet was read as hardware loss: %+v", drift) + } +} + +// A card with both a render node and an nvidia-smi entry is one card. Counting +// it twice would report a device gone whenever the two sources disagree about +// which of them can currently see it. +func TestCapabilityDriftCountsOneCardOnceAcrossBothIdentitySources(t *testing.T) { + const before = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","gpu_uuid":"GPU-aaa"}],` + + `"nvidia_gpu_uuids":["GPU-aaa"],"detected_backends":[{"backend":"nvenc","verified":true}]}` + // nvidia-smi went quiet; the render node still reports the same card. + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if drift.regressed() { + t.Fatalf("one card read as two: %+v", drift) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 039b25402..14f6a5892 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -419,8 +419,12 @@ func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { // Getting it low does not slow the read down, it cancels it: the node drops out // of the capability map mid-matrix and playback plans without it. // -// Two sources describe the same node and neither dominates, so the answer is -// the larger: +// Every source is a lower bound on what the read may need, so the answer is the +// largest of them — the caller's fallback included, which is why it is a floor +// and not only a last resort. Overshooting holds a dead node's fetch open a +// little longer; undershooting loses a live one. +// +// Two of those sources describe the node, and neither dominates: // // - What the node advertised in the report stored for it. That is the node's // own measurement of its own matrix, it survives an API restart because it @@ -435,18 +439,19 @@ func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { // falls back to the cheapest backend and it reads as a floor rather than as // the truth. // -// The fallback covers a node with neither: no stored report and no policy. +// The fallback is what each caller is willing to spend on a node it knows +// nothing about, and a node that has never been inventoried is the one most +// likely to be slow — so a policy that happens to price lower does not lower it. func ColdCapabilityRequestTimeout(storedReport json.RawMessage, hwAccel, hwDevice string, fallback time.Duration) time.Duration { - budget := time.Duration(0) + budget := fallback if millis := AdvertisedProbeBudgetMillis(storedReport); millis > 0 { - budget = NormalizeProbeRequestTimeout(millis, fallback) + if advertised := NormalizeProbeRequestTimeout(millis, fallback); advertised > budget { + budget = advertised + } } if priced := CapabilityRequestTimeout(hwAccel, hwDevice); priced > budget { budget = priced } - if budget <= 0 { - return fallback - } return budget } From 568027d071498f1ea925ad8966d9744d91f60ae5 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:06:21 -0400 Subject: [PATCH 066/163] fix(nodes): re-read GPU identities per walk, and the scratch dir per sample MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both a value captured once and then outliving what it describes. The NVIDIA uuid listing was cached for the process, reset only by a manual re-probe. On an NVIDIA-only node — /dev/nvidia* and the toolkit, no /dev/dri — that uuid is a card's whole identity, so a card hot-removed after the first build kept being published by every scheduled snapshot: the hash never moved on its account, drift never saw the loss, and physical_gpu_keys went on grouping the node with hardware it no longer had. The listing now lives for one detection walk. That asymmetry against the probe cache is deliberate — a probe is several ffmpeg execs and this is one cheap query, and it is the query that answers "is this card still here". The explicit reset on re-probe stays: that entry point is exported and cannot require a walk to follow, and the sampler reads identities between walks. The resource sampler captured playback.transcode_dir at startup. It is hot-reloadable, and in integrated mode this host is the one transcoding, so an operator repointing it left the Server resources card and the disk metrics measuring a volume nothing writes to — reporting headroom while the volume actually filling went unwatched, on the one mount transcode admission reads. The proxy had the same shape. Options.ScratchDir is now a provider, read once per pass so the three places that ask which mount is the scratch one cannot disagree mid-pass. The transcode node passes a constant: it writes every session under the directory it resolved at startup, so for it the old behavior was already right. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 11 +++- internal/nodemetrics/disk_test.go | 61 +++++++++++++++++++-- internal/nodemetrics/sampler.go | 57 ++++++++++++------- internal/playback/gpudetect.go | 30 ++++++---- internal/playback/gpudetect_publish_test.go | 37 +++++++++++++ internal/proxy/server.go | 15 +++-- internal/transcodenode/server.go | 5 +- 7 files changed, 175 insertions(+), 41 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index eacbe9e9b..9761441dc 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1266,7 +1266,16 @@ func main() { // it is the process that knows what the library is, and its own view of // a media mount is the one that is authoritative. resourceSampler := nodemetrics.NewSampler(nodemetrics.Options{ - ScratchDir: cfg.Playback.TranscodeDir, + // Read per sample rather than captured: in integrated mode this host + // is the one transcoding, and playback.transcode_dir is + // hot-reloadable. A startup snapshot would keep reporting headroom on + // the volume it used to write to while the new one fills unwatched. + ScratchDir: func() string { + if live := configWatcher.Config(); live != nil { + return live.Playback.TranscodeDir + } + return cfg.Playback.TranscodeDir + }, MediaRoots: libraryPathProvider(catalog.NewFolderRepository(pool)), DeviceSessions: playback.HWDeviceLoadSnapshot, DeviceIdentities: playback.SamplerDeviceIdentities, diff --git a/internal/nodemetrics/disk_test.go b/internal/nodemetrics/disk_test.go index abb3e7419..e15a192f4 100644 --- a/internal/nodemetrics/disk_test.go +++ b/internal/nodemetrics/disk_test.go @@ -64,7 +64,7 @@ func newDiskFixture(t *testing.T, paths ...string) *diskFixture { scratch, roots = roots[0], roots[1:] } f.sampler = newTestSampler(t, tree, clock, Options{ - ScratchDir: scratch, + ScratchDir: staticScratch(scratch), MediaRoots: func(context.Context) []string { return roots }, }) f.sampler.diskProbeDone = f.done @@ -366,7 +366,7 @@ func TestDiskStatsForgetsPathsNoLongerConfigured(t *testing.T) { done := make(chan string, 16) s := newTestSampler(t, tree, clock, Options{ - ScratchDir: "/transcode", + ScratchDir: staticScratch("/transcode"), MediaRoots: func(context.Context) []string { return roots }, }) s.diskProbeDone = done @@ -502,7 +502,7 @@ func TestDiskProbesAreBoundedAcrossReconfiguration(t *testing.T) { func TestRefreshDisksRotatesProbesPastAWedgedRetiredMount(t *testing.T) { tree := newProcTree(t) clock := newFakeClock() - s := newTestSampler(t, tree, clock, Options{ScratchDir: "/transcode"}) + s := newTestSampler(t, tree, clock, Options{ScratchDir: staticScratch("/transcode")}) done := make(chan string, 64) s.diskProbeDone = done @@ -557,7 +557,10 @@ func TestRefreshDisksRotatesProbesPastAWedgedRetiredMount(t *testing.T) { func TestRefreshDisksAlwaysOffersScratchFirst(t *testing.T) { tree := newProcTree(t) clock := newFakeClock() - s := newTestSampler(t, tree, clock, Options{ScratchDir: "/transcode"}) + s := newTestSampler(t, tree, clock, Options{ScratchDir: staticScratch("/transcode")}) + // Which mount is the scratch one is resolved once per pass, so the ordering + // below is asking a question a pass has to have asked first. + s.diskPaths(context.Background()) candidates := []*diskEntry{{path: "/transcode"}, {path: "/a"}, {path: "/b"}, {path: "/c"}} for pass := range 6 { @@ -605,3 +608,53 @@ func TestFSCapacityExcludesBlocksReservedFromThisProcess(t *testing.T) { t.Fatalf("UsedBytes = %d for free > total, want 0", got.UsedBytes) } } + +// staticScratch is the provider for a host whose scratch dir never moves, which +// is every test but the one covering a hot-reloaded one. +func staticScratch(path string) func() string { + return func() string { return path } +} + +// playback.transcode_dir is hot-reloadable, and in integrated mode this host is +// the one transcoding. A sampler that captured the directory at startup would +// keep reporting headroom on a volume nothing writes to while the new one fills +// unwatched — and the scratch role is what transcode admission reads. +func TestScratchDirFollowsAHotReloadedTranscodeDir(t *testing.T) { + f := newDiskFixture(t, "/transcode-old") + f.answer("/transcode-old", fsStats{UsedBytes: 10 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/transcode-new", fsStats{UsedBytes: 490 << 30, TotalBytes: 500 << 30, FSID: "b:2"}) + current := "/transcode-old" + f.sampler.scratchDirFn = func() string { return current } + + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + scratch := scratchDisk(t, f.sampler) + if scratch.Path != "/transcode-old" { + t.Fatalf("scratch = %q, want the configured /transcode-old", scratch.Path) + } + + // An operator repoints the setting at a nearly full volume. + current = "/transcode-new" + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + scratch = scratchDisk(t, f.sampler) + if scratch.Path != "/transcode-new" { + t.Fatalf("scratch = %q, want the reloaded /transcode-new", scratch.Path) + } + if scratch.UsedGB < 400 { + t.Fatalf("scratch used = %v GiB, want the new volume's near-full reading", scratch.UsedGB) + } +} + +// scratchDisk returns the disk carrying the scratch role, which is the one +// transcode admission reads. +func scratchDisk(t *testing.T, s *Sampler) DiskStats { + t.Helper() + for _, disk := range s.Snapshot().System.Disks { + if disk.Role == ScratchDiskRole { + return disk + } + } + t.Fatalf("no disk carried the scratch role: %+v", s.Snapshot().System.Disks) + return DiskStats{} +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index 7d57441d9..fbaad95c7 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -8,6 +8,7 @@ import ( "slices" "sort" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -24,9 +25,16 @@ const DefaultInterval = 5 * time.Second // means that dimension simply is not reported, which is the correct behavior // for a proxy node with no scratch dir or a host with no GPU. type Options struct { - // ScratchDir is the transcode working directory. It is sampled first + // ScratchDir returns the transcode working directory. It is sampled first // because it is the volume whose filling up silently kills transcodes. - ScratchDir string + // + // A provider rather than a value because playback.transcode_dir is + // hot-reloadable: a host that captured it at startup would keep measuring + // the volume it used to transcode onto, reporting ample headroom on a disk + // nothing writes to while the one filling up goes unwatched. Called on the + // sampling goroutine each pass, so it must be cheap; a nil provider means + // this host has no scratch dir, which is the right answer for a proxy. + ScratchDir func() string // MediaRoots returns library folder paths to sample alongside the scratch // dir. It is called on the sampling goroutine each pass, so it must be cheap // and must respect ctx; a nil provider samples the scratch dir only. @@ -63,14 +71,19 @@ type Options struct { // either atomic (the published snapshot) or explicitly guarded (the disk // entries, which detached probe goroutines also write). type Sampler struct { - interval time.Duration - now func() time.Time - goos string - scratchDir string - mediaRoots func(ctx context.Context) []string - sessions func() map[string]int - identities func() []DeviceIdentity - ffmpegPIDs func() []int + interval time.Duration + now func() time.Time + goos string + // scratchDirFn is the live source; scratchDir is what it answered for the + // pass in flight, resolved once in diskPaths so the three readers below + // cannot disagree about which mount is the scratch one mid-pass. Owned by + // the sampling goroutine. + scratchDirFn func() string + scratchDir string + mediaRoots func(ctx context.Context) []string + sessions func() map[string]int + identities func() []DeviceIdentity + ffmpegPIDs func() []int // Path seams. Production values point at the real filesystem; tests point // them at a fake /proc tree. @@ -143,16 +156,16 @@ func NewSampler(opts Options) *Sampler { ffmpegPIDs = func() []int { return defaultFFmpegChildren(procDir, pid) } } s := &Sampler{ - interval: interval, - now: now, - goos: runtime.GOOS, - scratchDir: opts.ScratchDir, - mediaRoots: opts.MediaRoots, - sessions: opts.DeviceSessions, - identities: opts.DeviceIdentities, - ffmpegPIDs: ffmpegPIDs, - procDir: procDir, - hostProcDir: hostProcDir, + interval: interval, + now: now, + goos: runtime.GOOS, + scratchDirFn: opts.ScratchDir, + mediaRoots: opts.MediaRoots, + sessions: opts.DeviceSessions, + identities: opts.DeviceIdentities, + ffmpegPIDs: ffmpegPIDs, + procDir: procDir, + hostProcDir: hostProcDir, // Each cgroup read is tried at this process's own cgroup before the // mount root, so a systemd unit with CPUQuota= or MemoryMax= is // measured against its limit rather than against the whole machine. @@ -338,6 +351,10 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { // The scratch dir is always first and so is never the entry dropped: it is the // one mount admission control reads. func (s *Sampler) diskPaths(ctx context.Context) []string { + s.scratchDir = "" + if s.scratchDirFn != nil { + s.scratchDir = strings.TrimSpace(s.scratchDirFn()) + } var paths []string if s.scratchDir != "" { paths = append(paths, s.scratchDir) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 14f6a5892..bd2ce17b2 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -250,6 +250,11 @@ var ErrHardwareDetectionIncomplete = errors.New("hardware detection did not comp // the node capability endpoints and their background snapshots — must use this // form and refuse to publish on ErrHardwareDetectionIncomplete. func DetectHWAccelWithFFmpegContextResult(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) (HWAccelInfo, error) { + // One listing per walk: the identities below are re-read here, while the + // probe verdicts they accompany stay cached. That asymmetry is deliberate — + // a probe is several ffmpeg execs and a listing is one cheap query, and it + // is the listing that answers "is this card still here". + resetNVIDIAGPUUUIDs() candidates := collectHWCandidates(hwDevice) resolved := HWAccelNone var detected []DetectedBackend @@ -1088,10 +1093,10 @@ func InvalidateHWProbeCache() { hwProbeCache.generation++ hwProbeCache.entries = make(map[string]hwProbeCacheEntry) hwProbeCache.verifiedDevices = make(map[string][]string) - // The GPU identity listing goes too. A re-probe is exactly when nvidia-smi - // may have become available, or a card in the same slot may have been - // replaced, and both of those change identities the drift comparison and - // shared-GPU placement read. + // The GPU identity listing goes too. A detection walk drops it on its own, + // but this is exported and nothing here can require that a walk follows — + // the sampler reads identities every few seconds between walks, and an + // operator who re-probes has asked for that to be current now. resetNVIDIAGPUUUIDs() // VideoToolbox keeps its own cache, keyed by the FFmpeg binary's identity // rather than by generation, so it has to be cleared rather than superseded. @@ -1717,13 +1722,16 @@ var nvidiaSMIQuery = runNVIDIASMIQuery // probe caches a second query could only cost a subprocess to learn the same // answer, since GPU identities do not change under a running kernel. // -// It is not cached for the process lifetime, though, which a sync.Once would -// make it. Two things invalidate it and both are exactly what the operator -// re-probe exists for: an nvidia-smi that was missing or broken at first call -// would otherwise never be asked again, leaving every NVIDIA card without its -// permanent id; and a card swapped into the same PCI slot would keep answering -// to its predecessor's uuid. Both feed drift detection and shared-GPU -// placement, so a stale answer here is a wrong answer there. +// It lives for one detection walk, not for the process, which a sync.Once would +// make it. Everything that reads it is asking what hardware this host has right +// now: drift detection compares one walk's answer against the last one, and +// shared-GPU placement groups nodes by it. A listing that outlives its walk +// makes all of those describe a machine that no longer exists — an nvidia-smi +// missing or broken at first call is never asked again, a card swapped into the +// same slot keeps answering to its predecessor's uuid, and a card hot-removed +// from an NVIDIA-only node goes on being reported by every scheduled snapshot +// until someone re-probes by hand. On such a node that uuid is the card's only +// identity, so nothing else in the report would show it gone. var nvidiaGPUUUIDs struct { mu sync.Mutex loaded bool diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go index b54aa5b10..1ccad984b 100644 --- a/internal/playback/gpudetect_publish_test.go +++ b/internal/playback/gpudetect_publish_test.go @@ -783,3 +783,40 @@ func TestHWProbesInFlightCountsADetachedVideoToolboxProbe(t *testing.T) { released.Do(func() { close(release) }) awaitNoProbesInFlight(t) } + +// A scheduled capability snapshot is how a long-running node notices its +// hardware changing, and on an NVIDIA-only node — /dev/nvidia* and the toolkit, +// no /dev/dri — a card's uuid is the only trace of it in the report. A listing +// cached past the walk that took it would republish a hot-removed card in every +// snapshot until someone re-probed by hand. +func TestCapabilityWalkRequeriesNVIDIAIdentities(t *testing.T) { + previous := nvidiaSMIQuery + answer := "GPU-aaa, 00000000:03:00.0\n" + queries := 0 + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + queries++ + return []byte(answer), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + first, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, "", "") + if !slices.Contains(first.NVIDIAGPUUUIDs, "GPU-aaa") { + t.Fatalf("first walk reported %v, want the card nvidia-smi named", first.NVIDIAGPUUUIDs) + } + + // The card is hot-removed. Nothing else in this node's report mentions it. + answer = "" + second, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, "", "") + if slices.Contains(second.NVIDIAGPUUUIDs, "GPU-aaa") { + t.Fatalf("second walk still reported %v for a card that is gone", second.NVIDIAGPUUUIDs) + } + if queries < 2 { + t.Fatalf("nvidia-smi queried %d times across two walks, want one per walk", queries) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index bdaebb595..1fa6a8525 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -408,11 +408,18 @@ func (s *Server) StartMetricsSampler(ctx context.Context) { if s == nil || ctx == nil { return } - scratchDir := "" - if s.watcher != nil { - if cfg := s.watcher.Config(); cfg != nil { - scratchDir = strings.TrimSpace(cfg.Playback.TranscodeDir) + // Read per sample, not captured here: playback.transcode_dir is + // hot-reloadable, and a proxy that snapshotted it at startup would go on + // measuring a volume nothing writes to. + scratchDir := func() string { + if s.watcher == nil { + return "" } + cfg := s.watcher.Config() + if cfg == nil { + return "" + } + return cfg.Playback.TranscodeDir } s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ ScratchDir: scratchDir, diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index cf98c0eb4..23d7d06b3 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -1017,7 +1017,10 @@ func (s *Server) StartMetricsSampler(ctx context.Context) { return } s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ - ScratchDir: s.transcodeDir, + // Fixed for this process: a node writes every session under the + // directory it resolved at startup, so a later config edit does not move + // the volume this one is filling. + ScratchDir: func() string { return s.transcodeDir }, DeviceSessions: playback.HWDeviceLoadSnapshot, DeviceIdentities: playback.SamplerDeviceIdentities, }) From 3a2270e160db044a15b082e6d0192cd7841838fc Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:22:42 -0400 Subject: [PATCH 067/163] fix(playback): bind the device ceiling to selection, and inherit device syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings. The probe matrix stops at a ceiling, but workload selection balanced across the whole configured list, so a 17th device could be handed a transcode with no verdict behind it — failing after the session started, on a node whose published capabilities say nothing about that device either way. It was only accidentally safe: verifiedHWDevices narrows to devices a walk recorded, which excludes the unprobed ones, but only in a process that has walked. The truncation is now UsableHWDevices, applied to the parsed list in configured order on both sides, so the probe matrix and the balancer keep exactly the same devices. Second, the per-node editor decided render-path versus CUDA device syntax from what the node resolves today. That is the backend being given up when an operator selects "Cluster default": a node overriding QSV under an NVENC cluster kept the render-path picker while inheriting, with no way to type the CUDA identity NVENC needs, and could save hw_accel_override = null beside a /dev/dri/… device override — a policy that cannot work. Inheritance now reads the cluster backend, falling back to the node's own resolution only where the cluster names none, since that is what an auto cluster inherits. Moving between the two syntaxes clears the device override, because a render path is not a CUDA identity and neither is settable as the other; on first render it clears nothing, which would have wiped a valid override just for opening the form. Co-Authored-By: Claude Opus 5 (1M context) --- internal/playback/gpudetect.go | 35 ++++++++++++------ internal/playback/hwdevice.go | 6 +++- internal/playback/hwdevice_test.go | 37 ++++++++++++++++++++ web/src/pages/AdminNodes.tsx | 23 ++++++++++-- web/src/pages/adminNodesPresentation.test.ts | 30 +++++++++++++++- web/src/pages/adminNodesPresentation.ts | 17 +++++++-- 6 files changed, 132 insertions(+), 16 deletions(-) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index bd2ce17b2..8deed4317 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -512,6 +512,30 @@ func maxHWAccelWalkTimeout() time.Duration { return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack } +// UsableHWDevices truncates a configured device list to the ceiling this host +// will actually use, which is the same ceiling the probe matrix is capped at. +// +// Past it the walk costs more than the budget every caller allows, so probing +// further guarantees the capability request is canceled rather than finished. +// The cap therefore has to bind selection too, not only probing: a device the +// matrix never reached has no verdict behind it, and dispatching a transcode +// there means finding out whether it works after the session has started, on a +// node whose published capabilities say nothing about it. Truncating in one +// place and balancing over the full list in another is only accidentally safe — +// it holds while a walk has recorded verified devices to narrow against, and +// stops holding in a process that has not walked yet. +// +// The devices past the ceiling are still reported in the inventory, so an +// operator can see what was configured, and the omission is logged rather than +// folded silently into a shorter answer. +func UsableHWDevices(devices []string) []string { + if len(devices) <= tonemap.MaxProbedDevices { + return devices + } + noteHWProbeDevicesTruncated(len(devices)) + return devices[:tonemap.MaxProbedDevices] +} + // hwProbeDevicesTruncatedLogged latches the truncation warning to one line per // process: a device list is standing configuration, not an event. var hwProbeDevicesTruncatedLogged sync.Once @@ -565,16 +589,7 @@ func collectHWCandidates(configuredDevice string) hwCandidates { if len(probeDevices) == 0 { probeDevices = candidates.renderDevices } - if len(probeDevices) > tonemap.MaxProbedDevices { - // The same ceiling the tone-map matrix is capped at, for the same - // reason: past it the walk costs more than the budget every caller - // allows, so probing further guarantees the capability request is - // canceled rather than finished. The devices that are probed get real - // verdicts; the rest are still reported in the inventory, and the - // omission is logged rather than folded silently into a shorter answer. - noteHWProbeDevicesTruncated(len(probeDevices)) - probeDevices = probeDevices[:tonemap.MaxProbedDevices] - } + probeDevices = UsableHWDevices(probeDevices) if len(configured.List()) > 0 { // A configured device this process cannot open can never pass a smoke // encode, so it is classified for reporting but never probed. This is diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index e81473d50..e53a9bf48 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -284,7 +284,11 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, w } // Select and reserve in one critical section so concurrent workload starts // observe each other's reservations instead of piling onto one device. - present := verifiedHWDevices(resolvedHWAccel, presentHWDevices(set.List())) + // The same ceiling the probe matrix stops at: nothing past it has a verdict + // behind it, so nothing past it takes work. Applied to the parsed list in + // the order it was configured, which is the order the walk probed, so both + // sides keep exactly the same devices. + present := verifiedHWDevices(resolvedHWAccel, presentHWDevices(UsableHWDevices(set.List()))) if len(present) > 1 && avoidDevice != "" { eligible := make([]string, 0, len(present)-1) for _, device := range present { diff --git a/internal/playback/hwdevice_test.go b/internal/playback/hwdevice_test.go index 321c93e58..b66f2f40e 100644 --- a/internal/playback/hwdevice_test.go +++ b/internal/playback/hwdevice_test.go @@ -2,8 +2,12 @@ package playback import ( "os" + "strconv" + "strings" "sync" "testing" + + "github.com/Silo-Server/silo-server/internal/tonemap" ) // fakeDeviceStat installs a stat function that reports only the given paths as @@ -436,3 +440,36 @@ func TestAcquireHWDeviceConcurrentStartsBalanceExactly(t *testing.T) { t.Fatalf("concurrent workload split = %v, want exact %d/%d", counts, workloads/2, workloads/2) } } + +// The probe matrix stops at a ceiling, so past it there is no verdict to +// dispatch on. Balancing over the full configured list would hand a share of the +// node's transcodes to a device the walk never reached, and the failure would +// land after the session started — while the published capabilities say nothing +// about that device either way. +func TestAcquireHWDeviceNeverSelectsPastTheProbeCeiling(t *testing.T) { + resetDeviceLoad(t) + devices := make([]string, 0, tonemap.MaxProbedDevices+1) + for i := range tonemap.MaxProbedDevices + 1 { + devices = append(devices, "/dev/dri/renderD"+strconv.Itoa(128+i)) + } + fakeDeviceStat(t, devices...) + beyond := devices[tonemap.MaxProbedDevices] + configured := strings.Join(devices, ",") + + // Every probed device has to be handed a workload before the one past the + // ceiling could come up, so run enough starts to cover the whole list twice. + var releases []func() + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range len(devices) * 2 { + selected, release := AcquireHWDevice(configured, "qsv") + releases = append(releases, release) + if selected == beyond { + t.Fatalf("workload dispatched to %q, which is past the %d-device probe ceiling", + beyond, tonemap.MaxProbedDevices) + } + } +} diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 43aa3d8d1..242833029 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -1,4 +1,5 @@ -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; +import { useAdminServerSettings } from "@/hooks/queries/admin/settings"; import type { FormEvent, ReactNode } from "react"; import type { StreamNode, CreateNodeRequest, UpdateNodeRequest } from "@/api/types"; import { @@ -543,7 +544,25 @@ function NodeForm({ // to be settable on a node this server has not heard from yet. // NVENC names GPUs by CUDA index or UUID, so the render-path picker is // meaningless for it — the same rule the cluster-wide Playback form applies. - const usesCUDADevices = nodeUsesCUDADevices(node, hwAccelOverride); + // The cluster setting is read because "inherit" means running what the + // cluster names, which this node's current resolution does not describe. + const { data: serverSettings } = useAdminServerSettings(); + const clusterHWAccel = serverSettings?.["playback.hw_accel"]; + const usesCUDADevices = nodeUsesCUDADevices(node, hwAccelOverride, clusterHWAccel); + // A device override written for one syntax means nothing in the other: a + // render path is not a CUDA identity and neither is settable as the other. So + // when the selection moves between them the stored value is dropped rather + // than carried into a policy it cannot express — but only on a change made + // here, never on the first render, which would wipe a valid override just for + // opening the form. + const previousUsesCUDA = useRef(usesCUDADevices); + useEffect(() => { + if (previousUsesCUDA.current === usesCUDADevices) { + return; + } + previousUsesCUDA.current = usesCUDADevices; + setHwDeviceOverride(""); + }, [usesCUDADevices]); const hasDeviceInventory = nodeHasHWDeviceInventory(node) && !usesCUDADevices; const deviceRows = buildNodeHWDeviceRows(node, hwDeviceOverride); const devicePaths = nodeHWDevicePaths(node); diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index 037165d0c..ebef72f9e 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -1153,10 +1153,38 @@ describe("nodeUsesCUDADevices", () => { expect(nodeUsesCUDADevices(makeNode({}), "nvenc")).toBe(true); }); - it("follows what the node resolves to when the backend is inherited", () => { + it("follows what the node resolves to when the cluster names no backend", () => { const node = makeNode({ capabilities: { resolved: "nvenc" } }); expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT)).toBe(true); expect(nodeUsesCUDADevices(node, "auto")).toBe(true); + expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT, "auto")).toBe(true); + }); + + // Inheriting means running what the cluster names. A node overriding QSV + // under an NVENC cluster still resolves qsv today, and following that would + // keep the render-path picker while inheritance is selected — leaving no way + // to type the CUDA identity and letting /dev/dri/… be saved as an NVENC + // policy that cannot work. + it("follows the cluster backend when inheritance is selected", () => { + const node = makeNode({ + capabilities: { resolved: "qsv" }, + hw_accel_override: "qsv", + hw_device_override: "/dev/dri/renderD128", + }); + expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT, "nvenc")).toBe(true); + }); + + it("uses render paths when the cluster names a render-device backend", () => { + const node = makeNode({ capabilities: { resolved: "nvenc" } }); + expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT, "qsv")).toBe(false); + expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT, "vaapi")).toBe(false); + }); + + // The node's own selection still wins: it is what that node will run. + it("keeps an explicit override ahead of the cluster backend", () => { + const node = makeNode({ capabilities: { resolved: "qsv" } }); + expect(nodeUsesCUDADevices(node, "nvenc", "qsv")).toBe(true); + expect(nodeUsesCUDADevices(node, "qsv", "nvenc")).toBe(false); }); // An explicit render-device backend wins over whatever the stale report says. diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index 607cb9ac9..ee8a83fd5 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -672,12 +672,21 @@ export const HW_ACCEL_OVERRIDE_OPTIONS: readonly { value: string; label: string * the backend actually needs. * * `override` is the backend selected in the editor, which may be the inherit - * sentinel — in that case what matters is the backend the node currently - * resolves to, exactly as the cluster form falls back to its detection result. + * sentinel. Inheriting means running whatever the cluster names, so that is + * what decides the device syntax — not what this node resolves today, which + * still reflects the override being given up. A node overriding QSV under an + * NVENC cluster would otherwise keep the render-path picker while inheritance + * is selected, leaving no way to type the CUDA identity the backend needs and + * letting `/dev/dri/…` be saved as an NVENC policy that cannot work. + * + * Only when the cluster itself names no backend — unset or `auto` — does the + * node's own resolution decide, because then its detection result is what it + * will inherit. */ export function nodeUsesCUDADevices( node: StreamNode | null | undefined, override: string | null | undefined, + clusterHWAccel?: string | null, ): boolean { const selected = override?.trim().toLowerCase() ?? ""; if (selected === "nvenc") { @@ -686,6 +695,10 @@ export function nodeUsesCUDADevices( if (selected !== "" && selected !== HW_ACCEL_INHERIT && selected !== "auto") { return false; } + const cluster = clusterHWAccel?.trim().toLowerCase() ?? ""; + if (cluster !== "" && cluster !== "auto") { + return cluster === "nvenc"; + } return node?.capabilities?.resolved?.trim().toLowerCase() === "nvenc"; } From 9bc72bbf7af468e0f6294ff6d7c4a3bc57f317eb Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:34:24 -0400 Subject: [PATCH 068/163] fix(nodes): reserve the node's own refresh bound on a re-probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-probe route holds one connection open across two long calls: the node's own matrix, then the capability refresh that stores its result. It reserved the node's probe budget for the first and the exported five-minute floor for the second — but that floor is not the refresh's bound. The backstop is derived from the node's advertised probe budget, so past about four minutes of budget the refresh outlives what the connection reserved: the write deadline fires after the re-probe succeeded and before its response is written, and the UI reports a failure for an action that has already changed the node. The bound is now asked of the thing that enforces it. HealthChecker exposes CapabilityRefreshBound and the handler reads it through an optional interface, so there is one number rather than the same rule derived twice on two sides of a package boundary. Without a refresher wired there is no refresh to wait for, and the floor is all the handler can promise. nodeCapabilityProbeBudget went through the same ladder while it was here. It open-coded the override resolution that Node.EffectiveHWAccel/EffectiveHWDevice already express, and it ignored the budget a node advertises in its stored report — which every other caller honors. The sweep, the re-probe, and the two planning paths now allow one node's matrix the same amount of time. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 19 +++---- internal/api/handlers/nodes.go | 31 ++++++++++-- internal/api/handlers/nodes_reprobe_test.go | 56 +++++++++++++++++++++ internal/nodepool/health.go | 26 ++++++++-- 4 files changed, 116 insertions(+), 16 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 9761441dc..cc3b6fc51 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -267,15 +267,16 @@ func nodeCapabilityProbeBudget(live func() *config.Config) func(*nodepool.Node) hwAccel, hwDevice = current.Playback.HWAccel, current.Playback.HWDevice } } - if node != nil { - if node.HWAccelOverride != nil && *node.HWAccelOverride != "" { - hwAccel = *node.HWAccelOverride - } - if node.HWDeviceOverride != nil && *node.HWDeviceOverride != "" { - hwDevice = *node.HWDeviceOverride - } - } - return max(nodeCapabilityRequestTimeout, playback.CapabilityRequestTimeout(hwAccel, hwDevice)) + // The same ladder every other caller prices a node's capability read + // with — its own advertisement, then its own policy, then this floor — + // so the sweep, the re-probe, and the two planning paths cannot end up + // allowing different amounts of time for one node's matrix. + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(hwAccel), + node.EffectiveHWDevice(hwDevice), + nodeCapabilityRequestTimeout, + ) } } diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 38657d705..e8af5ccc5 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -667,6 +667,29 @@ func (h *NodeHandler) nodeReprobeTimeout(n *nodepool.Node) time.Duration { ) } +// capabilityRefreshBound is how long the refresh after a re-probe may run for +// this node, asked of the thing that will enforce it. +// +// The bound is derived from the node's own advertised probe budget, so it is +// node-specific and routinely past the exported floor — a node with a large +// device set asks for well over five minutes. Computing it here from a second +// rule would be the same number derived twice, and the two would disagree +// exactly where it matters. Without a refresher wired there is no refresh to +// wait for, so the floor is all this can promise. +func (h *NodeHandler) capabilityRefreshBound(n *nodepool.Node) time.Duration { + if bounder, ok := h.capabilities.(nodeCapabilityRefreshBounder); ok { + return bounder.CapabilityRefreshBound(n) + } + return nodepool.CapabilityRefreshTimeout +} + +// nodeCapabilityRefreshBounder reports how long a refresh of one node may take. +// Optional on NodeCapabilityRefresher, like the other collaborators here; +// *nodepool.HealthChecker implements it. +type nodeCapabilityRefreshBounder interface { + CapabilityRefreshBound(n *nodepool.Node) time.Duration +} + // nodeReprobeWriteSlack covers the repository round trips and the JSON write // that bracket the two long calls this handler makes. const nodeReprobeWriteSlack = 15 * time.Second @@ -683,8 +706,10 @@ const nodeReprobeWriteSlack = 15 * time.Second // the UI toasts a failure for an action that succeeded, and the obvious response // is to run the whole cold FFmpeg matrix again. The diagnostics upload route // extends its deadlines for exactly the same reason. -func extendReprobeWriteDeadline(w http.ResponseWriter, r *http.Request, probeBudget time.Duration) { - budget := probeBudget + nodepool.CapabilityRefreshTimeout + nodeReprobeWriteSlack +func (h *NodeHandler) extendReprobeWriteDeadline( + w http.ResponseWriter, r *http.Request, n *nodepool.Node, probeBudget time.Duration, +) { + budget := probeBudget + h.capabilityRefreshBound(n) + nodeReprobeWriteSlack if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(budget)); err != nil { // A ResponseWriter that cannot carry a deadline (a test recorder, a // wrapper that does not unwrap) is not a reason to refuse the action. @@ -733,7 +758,7 @@ func (h *NodeHandler) HandleReprobeNode(w http.ResponseWriter, r *http.Request) return } - extendReprobeWriteDeadline(w, r, h.nodeReprobeTimeout(node)) + h.extendReprobeWriteDeadline(w, r, node, h.nodeReprobeTimeout(node)) result := ReprobeNodeResult{NodeID: node.ID, NodeName: node.Name, Status: "ok"} reprobed, err := h.reprobeNode(r.Context(), node) diff --git a/internal/api/handlers/nodes_reprobe_test.go b/internal/api/handlers/nodes_reprobe_test.go index 195d13860..284bdd449 100644 --- a/internal/api/handlers/nodes_reprobe_test.go +++ b/internal/api/handlers/nodes_reprobe_test.go @@ -298,3 +298,59 @@ func TestNodeReprobeTimeoutRepricesAWidenedDeviceOverride(t *testing.T) { t.Fatalf("fixture is inert: the four-device budget %s must exceed the fallback %s", want, nodeReprobeFallbackTimeout) } } + +// boundedCapabilityRefresher is a refresher that also reports how long its +// refresh may take, as the real health checker does. +type boundedCapabilityRefresher struct { + stubCapabilityRefresher + bound time.Duration +} + +func (b *boundedCapabilityRefresher) CapabilityRefreshBound(*nodepool.Node) time.Duration { + return b.bound +} + +// The connection has to stay open across both long calls: the node's re-probe +// and the capability refresh that stores its result. The refresh's bound is +// derived from the node's own advertised probe budget, so on a node with many +// devices it runs well past the exported five-minute floor — and reserving the +// floor would let the write deadline fire after the re-probe succeeded but +// before its response was written, telling an operator an action failed that +// has already changed the node. +func TestReprobeWriteDeadlineReservesTheNodesOwnRefreshBound(t *testing.T) { + node := &nodepool.Node{ID: 1, Name: "gpu-1", URL: "http://gpu-1", Type: nodepool.NodeTypeTranscode} + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + + // No refresher: the floor is all this handler can promise. + if got := handler.capabilityRefreshBound(node); got != nodepool.CapabilityRefreshTimeout { + t.Fatalf("bound without a refresher = %s, want the exported floor %s", got, nodepool.CapabilityRefreshTimeout) + } + + // A refresher that will hold the fetch open past the floor. + bound := nodepool.CapabilityRefreshTimeout + 4*time.Minute + handler.SetCapabilityRefresher(&boundedCapabilityRefresher{bound: bound}) + if got := handler.capabilityRefreshBound(node); got != bound { + t.Fatalf("bound = %s, want the refresher's own %s", got, bound) + } + + // And it is what the deadline reserves, on top of the probe budget. + recorder := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + probeBudget := 3 * time.Minute + before := time.Now() + handler.extendReprobeWriteDeadline(recorder, reprobeRequest(t), node, probeBudget) + want := probeBudget + bound + nodeReprobeWriteSlack + if reserved := recorder.deadline.Sub(before); reserved < want { + t.Fatalf("reserved %s, want at least the probe budget plus the refresh bound (%s)", reserved, want) + } +} + +// deadlineRecorder captures the write deadline the handler sets. +type deadlineRecorder struct { + *httptest.ResponseRecorder + deadline time.Time +} + +func (d *deadlineRecorder) SetWriteDeadline(at time.Time) error { + d.deadline = at + return nil +} diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 100b436e9..79ff3261b 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -168,12 +168,30 @@ const capabilityFetchTimeout = 5 * time.Minute // fires first and the failure an operator sees names the probe rather than this. const capabilityFetchSlack = time.Minute -// CapabilityRefreshTimeout is the bound RefreshNodeCapabilities puts on the -// fetch it performs. It is exported for the one caller that has to hold an HTTP -// connection open across that fetch and must therefore size its own write -// deadline to include it. +// CapabilityRefreshTimeout is the floor under that bound, and all a caller can +// assume when it has no health checker to ask. It is exported for the one caller +// that has to hold an HTTP connection open across a refresh and must therefore +// size its own write deadline to include it — see CapabilityRefreshBound, which +// is the number that caller should actually use. const CapabilityRefreshTimeout = capabilityFetchTimeout +// CapabilityRefreshBound is how long RefreshNodeCapabilities may take for this +// node, which is the same backstop the sweep's own fetches run under. +// +// It is exported because a caller holding an HTTP connection open across a +// refresh has to reserve the real number, not the floor: the backstop is derived +// from the node's advertised probe budget, and a node with a large device set +// asks for well past five minutes. Reserving the floor there means the +// connection's write deadline can fire after the refresh succeeded but before +// its response is written, and the operator is told an action failed that has +// already changed the node. +func (hc *HealthChecker) CapabilityRefreshBound(n *Node) time.Duration { + if hc == nil { + return capabilityFetchTimeout + } + return hc.capabilityFetchBackstop(n) +} + // HealthChecker runs periodic health checks on all nodes in both pools, // updating in-memory state and optionally persisting to the database. type HealthChecker struct { From cf2617157bebb7f6deb95425e96c3c97eadb20f4 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:52:19 -0400 Subject: [PATCH 069/163] fix(nodes): keep Auto node-owned, stop hydration erasing an override, budget the walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings. An explicit Auto override reached the cluster-backend branch added last round, but Auto is not inheritance — it tells this node to detect against its own hardware, so what the cluster names says nothing about what it will run. Under an NVENC cluster a node whose Auto resolves QSV switched to CUDA syntax and could throw away a valid render-path override. Only the inherit sentinel defers to the cluster now; an Auto cluster is still inherited as that same instruction, which is why the node's own resolution remains the fallback there. The clearing itself watched the effective syntax through an effect, and that value moves without anyone touching the dialog: the cluster setting is absent on the first render and arrives with the query, so opening the form under an NVENC cluster could erase a valid CUDA override, and saving an unrelated field then sent hw_device_override: null. The decision is now hwDeviceSyntaxChanges, taking the selection being left and the one being chosen, called from the backend select's own handler. Hydration cannot reach it. Third, /admin/system/hw-accel runs a full hardware walk on the request goroutine. That walk scales with the configured device set — eight Intel render devices draw five ffmpeg commands each at three seconds — and passes the API listener's 120-second write timeout, so the settings page could lose its response while every probe was still inside its own bound. The route now lifts its write deadline from playback.HWAccelWalkTimeout, the same thing the re-probe route does for the same reason. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/system.go | 31 +++++++++-- internal/api/handlers/system_test.go | 42 +++++++++++++++ internal/playback/gpudetect.go | 14 +++++ web/src/pages/AdminNodes.tsx | 30 ++++++----- web/src/pages/adminNodesPresentation.test.ts | 42 +++++++++++++++ web/src/pages/adminNodesPresentation.ts | 56 +++++++++++++++----- 6 files changed, 185 insertions(+), 30 deletions(-) diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index 15b623945..c1ff741b5 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -131,7 +131,7 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if len(healthy) == 0 { - writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: h.localHWAccel()}) + writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: h.localHWAccel(w, r)}) return } @@ -173,7 +173,7 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if !primaried { - inventory.HWAccelInfo = h.localHWAccel() + inventory.HWAccelInfo = h.localHWAccel(w, r) } writeJSON(w, http.StatusOK, inventory) } @@ -183,14 +183,39 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { // A zero-value handler answers with the ffmpeg on PATH and auto-detection // rather than panicking: tests build one directly, and the accessor is wiring // this method should not depend on having received. -func (h *SystemHandler) localHWAccel() playback.HWAccelInfo { +func (h *SystemHandler) localHWAccel(w http.ResponseWriter, r *http.Request) playback.HWAccelInfo { var ffmpegPath, hwAccel, hwDevice string if h.playback != nil { ffmpegPath, hwAccel, hwDevice = h.playback() } + extendHWAccelWriteDeadline(w, r, hwDevice) return playback.DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice) } +// hwAccelWriteSlack covers reading the node list and writing the JSON around the +// walk this route runs. +const hwAccelWriteSlack = 15 * time.Second + +// extendHWAccelWriteDeadline lifts this connection's write deadline to cover a +// synchronous hardware walk. +// +// The walk is bounded by its own budget, which scales with the configured device +// set: eight Intel render devices draw five ffmpeg commands each at three +// seconds apiece, which is already past the API listener's 120-second write +// timeout. Without this the settings page loses its response while every probe +// is still inside its bound, and the operator sees a failed request for a probe +// that is about to succeed. The re-probe route extends its deadline for exactly +// the same reason. +func extendHWAccelWriteDeadline(w http.ResponseWriter, r *http.Request, hwDevice string) { + budget := playback.HWAccelWalkTimeout(hwDevice) + hwAccelWriteSlack + if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(budget)); err != nil { + // A ResponseWriter that cannot carry a deadline (a test recorder, a + // wrapper that does not unwrap) is not a reason to refuse the probe. + slog.WarnContext(r.Context(), "hw-accel write deadline not extended", "component", "api", + "budget", budget, "error", err) + } +} + // HandleBuildInfo handles GET /admin/system/build. func (h *SystemHandler) HandleBuildInfo(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, h.buildInfo) diff --git a/internal/api/handlers/system_test.go b/internal/api/handlers/system_test.go index 5fe398cbe..57aee6e95 100644 --- a/internal/api/handlers/system_test.go +++ b/internal/api/handlers/system_test.go @@ -6,8 +6,10 @@ import ( "log/slog" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + "time" "github.com/go-chi/chi/v5" @@ -247,3 +249,43 @@ func TestHandleHWAccelProbeLogRedactsNodeURLSecrets(t *testing.T) { t.Fatalf("capability probe log lost sanitized node origin: %q", diagnostics) } } + +// The local inventory runs a full hardware walk on the request goroutine, and +// that walk grows with the configured device set — eight Intel render devices +// draw five ffmpeg commands each, already past the API listener's 120-second +// write timeout. Without lifting the deadline the settings page loses its +// response while every probe is still inside its own bound. +func TestHWAccelExtendsTheWriteDeadlineForItsWalk(t *testing.T) { + devices := make([]string, 0, 8) + for i := range 8 { + devices = append(devices, "/dev/dri/renderD"+strconv.Itoa(128+i)) + } + hwDevice := strings.Join(devices, ",") + handler := &SystemHandler{ + playback: func() (string, string, string) { return "/nonexistent/ffmpeg", "auto", hwDevice }, + } + + recorder := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + before := time.Now() + handler.HandleHWAccel(recorder, httptest.NewRequest(http.MethodGet, "/admin/system/hw-accel", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + // Sized from the walk's own budget, so it grows with the device set exactly + // as the walk does. How large that is depends on how the devices classify, + // which depends on the sysfs of the host running this — an Intel host draws + // five commands per device and passes 120 seconds at eight of them, while a + // machine with no such devices prices the same list far lower. So this + // asserts the derivation rather than a number no host agrees on. + want := playback.HWAccelWalkTimeout(hwDevice) + hwAccelWriteSlack + if reserved := recorder.deadline.Sub(before); reserved < want { + t.Fatalf("reserved %s, want at least the walk's own budget plus slack (%s)", reserved, want) + } + // And it is derived from the configured set, not a constant: a single device + // has to price lower than eight. + if single := playback.HWAccelWalkTimeout(devices[0]); single >= playback.HWAccelWalkTimeout(hwDevice) { + t.Fatalf("one device budgets %s and eight budget %s; the walk budget does not follow the device set", + single, playback.HWAccelWalkTimeout(hwDevice)) + } +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 8deed4317..ad4dd5751 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -409,6 +409,20 @@ func CapabilityEndpointTimeout(hwAccel, hwDevice string) time.Duration { tonemap.ProbeEndpointTimeout(hwAccel, hwDevice) } +// HWAccelWalkTimeout is how long a hardware detection walk of this host takes at +// worst, for the configured device set. +// +// Exported for a caller that runs one synchronously while holding an HTTP +// connection open and must size its write deadline to cover it: the walk grows +// with the device count and passes the API listener's own write timeout at eight +// Intel render devices, so a response can be lost while every probe is still +// inside its bound. Unlike the ceiling used to clamp what a *remote* node +// advertises, this classifies devices on the host that is about to walk them, +// which is the host asking. +func HWAccelWalkTimeout(hwDevice string) time.Duration { + return hwAccelWalkTimeout(collectHWCandidates(hwDevice)) +} + // CapabilityRequestTimeout is CapabilityEndpointTimeout plus the transport // margin a remote caller needs. It is what a node advertises and what a caller // of that node's capability endpoint must allow. diff --git a/web/src/pages/AdminNodes.tsx b/web/src/pages/AdminNodes.tsx index 242833029..d92e28948 100644 --- a/web/src/pages/AdminNodes.tsx +++ b/web/src/pages/AdminNodes.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react"; +import { useState } from "react"; import { useAdminServerSettings } from "@/hooks/queries/admin/settings"; import type { FormEvent, ReactNode } from "react"; import type { StreamNode, CreateNodeRequest, UpdateNodeRequest } from "@/api/types"; @@ -54,6 +54,7 @@ import { describeSharedGPU, nodeHWDevicePaths, nodeHasHWDeviceInventory, + hwDeviceSyntaxChanges, nodeUsesCUDADevices, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -550,19 +551,20 @@ function NodeForm({ const clusterHWAccel = serverSettings?.["playback.hw_accel"]; const usesCUDADevices = nodeUsesCUDADevices(node, hwAccelOverride, clusterHWAccel); // A device override written for one syntax means nothing in the other: a - // render path is not a CUDA identity and neither is settable as the other. So - // when the selection moves between them the stored value is dropped rather - // than carried into a policy it cannot express — but only on a change made - // here, never on the first render, which would wipe a valid override just for - // opening the form. - const previousUsesCUDA = useRef(usesCUDADevices); - useEffect(() => { - if (previousUsesCUDA.current === usesCUDADevices) { - return; + // render path is not a CUDA identity and neither is settable as the other, so + // moving the selection between them drops the stored value rather than + // carrying it into a policy it cannot express. + // + // Driven by the operator's own edit rather than by watching usesCUDADevices, + // because that value also moves on its own: the cluster setting arrives after + // the first render, and an effect would read that as a change and erase a + // valid override on a dialog nobody had touched yet. + function selectHWAccelOverride(next: string) { + if (hwDeviceSyntaxChanges(node, hwAccelOverride, next, clusterHWAccel)) { + setHwDeviceOverride(""); } - previousUsesCUDA.current = usesCUDADevices; - setHwDeviceOverride(""); - }, [usesCUDADevices]); + setHwAccelOverride(next); + } const hasDeviceInventory = nodeHasHWDeviceInventory(node) && !usesCUDADevices; const deviceRows = buildNodeHWDeviceRows(node, hwDeviceOverride); const devicePaths = nodeHWDevicePaths(node); @@ -700,7 +702,7 @@ function NodeForm({ <>
- diff --git a/web/src/pages/adminNodesPresentation.test.ts b/web/src/pages/adminNodesPresentation.test.ts index ebef72f9e..c3a1e1d50 100644 --- a/web/src/pages/adminNodesPresentation.test.ts +++ b/web/src/pages/adminNodesPresentation.test.ts @@ -17,6 +17,7 @@ import { formatBitsPerSecond, nodeHWDevicePaths, nodeHasHWDeviceInventory, + hwDeviceSyntaxChanges, nodeUsesCUDADevices, parseHWDeviceOverride, } from "./adminNodesPresentation"; @@ -1180,6 +1181,17 @@ describe("nodeUsesCUDADevices", () => { expect(nodeUsesCUDADevices(node, HW_ACCEL_INHERIT, "vaapi")).toBe(false); }); + // Auto is not inheritance: it tells this node to detect against its own + // hardware, so what the cluster names says nothing about what it will run. + // Reading the cluster there would switch a node whose Auto resolves QSV to + // CUDA syntax and throw away a valid render-path override. + it("keeps an explicit auto on the node's own resolution", () => { + const qsv = makeNode({ capabilities: { resolved: "qsv" } }); + expect(nodeUsesCUDADevices(qsv, "auto", "nvenc")).toBe(false); + const nvenc = makeNode({ capabilities: { resolved: "nvenc" } }); + expect(nodeUsesCUDADevices(nvenc, "auto", "qsv")).toBe(true); + }); + // The node's own selection still wins: it is what that node will run. it("keeps an explicit override ahead of the cluster backend", () => { const node = makeNode({ capabilities: { resolved: "qsv" } }); @@ -1201,6 +1213,36 @@ describe("nodeUsesCUDADevices", () => { }); }); +describe("hwDeviceSyntaxChanges", () => { + // Under an NVENC cluster, giving up a QSV override means the device value has + // to be a CUDA identity — the render path it holds cannot become one. + it("reports a crossing when inheritance flips the syntax", () => { + const node = makeNode({ + capabilities: { resolved: "qsv" }, + hw_accel_override: "qsv", + hw_device_override: "/dev/dri/renderD128", + }); + expect(hwDeviceSyntaxChanges(node, "qsv", HW_ACCEL_INHERIT, "nvenc")).toBe(true); + expect(hwDeviceSyntaxChanges(node, HW_ACCEL_INHERIT, "qsv", "nvenc")).toBe(true); + }); + + // Both name render paths, so the value the operator typed still means what it + // meant. Dropping it here would be losing work for nothing. + it("reports no crossing between two render-device backends", () => { + const node = makeNode({ capabilities: { resolved: "qsv" } }); + expect(hwDeviceSyntaxChanges(node, "qsv", "vaapi", "qsv")).toBe(false); + }); + + // The whole reason this takes both selections: the cluster setting is absent + // on the first render and arrives with the query. Nothing the operator did + // changed, so nothing may be erased. + it("reports no crossing when only the cluster setting arrives", () => { + const node = makeNode({ capabilities: { resolved: "qsv" } }); + expect(hwDeviceSyntaxChanges(node, HW_ACCEL_INHERIT, HW_ACCEL_INHERIT, undefined)).toBe(false); + expect(hwDeviceSyntaxChanges(node, HW_ACCEL_INHERIT, HW_ACCEL_INHERIT, "nvenc")).toBe(false); + }); +}); + describe("capability report staleness from an unconfirmed hash", () => { const base = { id: 1, diff --git a/web/src/pages/adminNodesPresentation.ts b/web/src/pages/adminNodesPresentation.ts index ee8a83fd5..88b0dd604 100644 --- a/web/src/pages/adminNodesPresentation.ts +++ b/web/src/pages/adminNodesPresentation.ts @@ -671,17 +671,19 @@ export const HW_ACCEL_OVERRIDE_OPTIONS: readonly { value: string; label: string * NVENC leaves it holding a render path with no way to type the CUDA identity * the backend actually needs. * - * `override` is the backend selected in the editor, which may be the inherit - * sentinel. Inheriting means running whatever the cluster names, so that is - * what decides the device syntax — not what this node resolves today, which - * still reflects the override being given up. A node overriding QSV under an - * NVENC cluster would otherwise keep the render-path picker while inheritance - * is selected, leaving no way to type the CUDA identity the backend needs and - * letting `/dev/dri/…` be saved as an NVENC policy that cannot work. + * `override` is the backend selected in the editor. Only the inherit sentinel + * defers to the cluster: inheriting means running whatever the cluster names, + * so that is what decides the device syntax — not what this node resolves + * today, which still reflects the override being given up. A node overriding + * QSV under an NVENC cluster would otherwise keep the render-path picker while + * inheritance is selected, leaving no way to type the CUDA identity the backend + * needs and letting `/dev/dri/…` be saved as an NVENC policy that cannot work. * - * Only when the cluster itself names no backend — unset or `auto` — does the - * node's own resolution decide, because then its detection result is what it - * will inherit. + * An explicit `auto` is not inheritance. It is this node's own policy — detect + * against your own hardware — so the cluster's backend says nothing about what + * it will run, and its own resolution decides. That is also the fallback when + * inheritance is selected and the cluster names no backend, since an `auto` + * cluster is inherited as exactly that instruction. */ export function nodeUsesCUDADevices( node: StreamNode | null | undefined, @@ -695,13 +697,41 @@ export function nodeUsesCUDADevices( if (selected !== "" && selected !== HW_ACCEL_INHERIT && selected !== "auto") { return false; } - const cluster = clusterHWAccel?.trim().toLowerCase() ?? ""; - if (cluster !== "" && cluster !== "auto") { - return cluster === "nvenc"; + if (selected === "" || selected === HW_ACCEL_INHERIT) { + const cluster = clusterHWAccel?.trim().toLowerCase() ?? ""; + if (cluster !== "" && cluster !== "auto") { + return cluster === "nvenc"; + } } return node?.capabilities?.resolved?.trim().toLowerCase() === "nvenc"; } +/** + * Whether moving the backend selection from `current` to `next` leaves the + * device override meaningless. + * + * The two syntaxes are not interchangeable: a render path is not a CUDA index + * or UUID and neither is settable as the other, so a value written for one is + * not a value for the other — it is a policy that cannot work. Crossing between + * them drops it. + * + * Takes both selections rather than watching the effective syntax, because that + * also moves on its own: the cluster setting arrives after the first render, + * and treating that as a change would erase a valid override on a dialog nobody + * had touched. + */ +export function hwDeviceSyntaxChanges( + node: StreamNode | null | undefined, + current: string | null | undefined, + next: string | null | undefined, + clusterHWAccel?: string | null, +): boolean { + return ( + nodeUsesCUDADevices(node, current, clusterHWAccel) !== + nodeUsesCUDADevices(node, next, clusterHWAccel) + ); +} + /** A node's own acceleration policy, as rendered beside its GPU inventory. */ export interface NodeAccelerationOverride { /** Compact row text, e.g. "override: qsv · /dev/dri/renderD129". */ From fe14be7646e981b56a210d3946950cd98dfa3dc8 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 04:07:18 -0400 Subject: [PATCH 070/163] fix(nodepool): address nodes through one URL joiner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stored node URL may carry a trailing slash — pasting a base URL is the usual way an operator enters one, and the pools and repository already treat the two forms as the same worker. Concatenating a route onto it produced "//admin/…", which no node's router has, so the request 404'd against a node that was running and reachable. The two routes this PR added were the report, but the same concatenation was everywhere a node is addressed, including the health check — where it would have left such a node permanently unhealthy — and the transcode-start and proxy stream URLs on both the v1 and jellycompat paths, where it breaks playback rather than an admin action. nodepool.NodeEndpoint now does the join for all of them, using the same normalization that already decides whether two URLs are the same worker: the rule that makes the forms equal for comparison has to make them equal for addressing. transcodenode's capability client already trimmed by hand and is left alone, being the one place that had it right. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/nodes.go | 8 ++--- internal/api/handlers/nodes_reprobe_test.go | 26 ++++++++++++++ internal/api/handlers/playback.go | 2 +- internal/api/handlers/playback_transport.go | 3 +- internal/chapterthumbs/remote.go | 2 +- internal/jellycompat/handlers_playback.go | 7 ++-- internal/nodepool/health.go | 2 +- internal/nodepool/health_test.go | 39 +++++++++++++++++++++ internal/nodepool/transcode_pool.go | 12 +++++++ 9 files changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index e8af5ccc5..ca49f46c9 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -392,7 +392,7 @@ func (h *NodeHandler) reloadNodeConfig(ctx context.Context, node *nodepool.Node) } ctx, cancel := context.WithTimeout(ctx, nodeConfigReloadTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/reload-config", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/reload-config"), nil) if err != nil { return false } @@ -529,7 +529,7 @@ func (h *NodeHandler) HandleForceReloadNodes(w http.ResponseWriter, r *http.Requ defer wg.Done() result := ForceReloadResult{NodeID: node.ID, NodeName: node.Name} client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/force-reload", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/force-reload"), nil) if err != nil { result.Status = "error" result.Error = err.Error() @@ -578,7 +578,7 @@ func (h *NodeHandler) HandleForceReloadNode(w http.ResponseWriter, r *http.Reque } client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, node.URL+"/admin/force-reload", nil) + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/force-reload"), nil) if err != nil { writeError(w, http.StatusInternalServerError, "internal_error", err.Error()) return @@ -804,7 +804,7 @@ func (h *NodeHandler) reprobeNode(ctx context.Context, node *nodepool.Node) (nod ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/reprobe-capabilities", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/reprobe-capabilities"), nil) if err != nil { return nodeReprobeResponse{}, err } diff --git a/internal/api/handlers/nodes_reprobe_test.go b/internal/api/handlers/nodes_reprobe_test.go index 284bdd449..30d4f8aba 100644 --- a/internal/api/handlers/nodes_reprobe_test.go +++ b/internal/api/handlers/nodes_reprobe_test.go @@ -354,3 +354,29 @@ func (d *deadlineRecorder) SetWriteDeadline(at time.Time) error { d.deadline = at return nil } + +// The route an operator triggers has to reach a node stored with a trailing +// slash, which the pools and the repository already treat as the same worker. +func TestHandleReprobeNodeReachesATrailingSlashBaseURL(t *testing.T) { + var path string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + writeJSON(w, http.StatusOK, map[string]any{"resolved": "qsv", "capability_hash": "sha256:x"}) + })) + defer node.Close() + + repo := &stubNodeRepository{node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL + "/", Enabled: true, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + if path != "/admin/reprobe-capabilities" { + t.Fatalf("node was asked for %q, want /admin/reprobe-capabilities", path) + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } +} diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 5c5b7b2ad..8a536d446 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -1811,7 +1811,7 @@ func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyN if proxyNode == nil || token == "" { return appendStreamToken(localURL, token) } - return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" + return nodepool.NodeEndpoint(proxyNode.URL, "/stream/transcode/"+token+"/master.m3u8") } // proxyToTranscodeNode forwards a request to the remote transcode node. diff --git a/internal/api/handlers/playback_transport.go b/internal/api/handlers/playback_transport.go index 18c2b796b..3e41d7c34 100644 --- a/internal/api/handlers/playback_transport.go +++ b/internal/api/handlers/playback_transport.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/logredact" + "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodenode" @@ -36,7 +37,7 @@ func (h *PlaybackHandler) startRemotePlaybackTransport(ctx context.Context, node } requestCtx, cancel := context.WithTimeout(ctx, h.remotePlaybackTransportTimeout(nodeURL, request)) defer cancel() - httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body)) + httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(nodeURL, "/transcode/start"), bytes.NewReader(body)) if err != nil { return transcodenode.TranscodeStartResponse{}, 0, logredact.SanitizeURLError(err) } diff --git a/internal/chapterthumbs/remote.go b/internal/chapterthumbs/remote.go index ba6e2c4b3..a3b581463 100644 --- a/internal/chapterthumbs/remote.go +++ b/internal/chapterthumbs/remote.go @@ -70,7 +70,7 @@ func (e *httpRemoteFrameExtractor) ExtractFrame( requestCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, node.URL+"/chapter-thumbnails/extract", bytes.NewReader(body)) + httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/chapter-thumbnails/extract"), bytes.NewReader(body)) if err != nil { return nil, chapterThumbnailNodeUnavailableReason, fmt.Errorf("chapter thumbnail remote extract: build request: %w", err) } diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 510953107..8639b7d8c 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -1045,7 +1045,7 @@ func (h *PlaybackHandler) buildProxyRedirectURL( switch method { case string(playback.PlayDirect): - return proxyNode.URL + "/stream/direct/" + token, nil + return nodepool.NodeEndpoint(proxyNode.URL, "/stream/direct/"+token), nil case string(playback.PlayRemux): remuxPath := "/stream/remux/" if claims.PlayMethod == streamtoken.PlayMethodAudioDownmixRemux { @@ -1057,7 +1057,8 @@ func (h *PlaybackHandler) buildProxyRedirectURL( } return redirectURL, nil case string(playback.PlayTranscode): - return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8?" + playback.SourceTimelineQueryParam + "=1", nil + return nodepool.NodeEndpoint(proxyNode.URL, + "/stream/transcode/"+token+"/master.m3u8?"+playback.SourceTimelineQueryParam+"=1"), nil default: return "", fmt.Errorf("unsupported proxy method %q", method) } @@ -1291,7 +1292,7 @@ func (h *PlaybackHandler) startRemoteTranscodeWithToneMapMode( } requestCtx, cancel := context.WithTimeout(ctx, h.remoteTranscodeStartTimeout(request, nodeProbeTimeoutMillis)) defer cancel() - httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, transcodeNodeURL+"/transcode/start", strings.NewReader(string(body))) + httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(transcodeNodeURL, "/transcode/start"), strings.NewReader(string(body))) if err != nil { return transcodenode.TranscodeStartResponse{}, 0, false, fmt.Errorf("build transcode request: %w", logredact.SanitizeURLError(err)) } diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 79ff3261b..cbf0488af 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -58,7 +58,7 @@ func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKb ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, n.URL+"/api/v1/health", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, NodeEndpoint(n.URL, "/api/v1/health"), nil) if err != nil { return false, 0, 0, "", nil } diff --git a/internal/nodepool/health_test.go b/internal/nodepool/health_test.go index f8ac903df..42aae6b98 100644 --- a/internal/nodepool/health_test.go +++ b/internal/nodepool/health_test.go @@ -539,3 +539,42 @@ func TestCapabilityFetchBackstopSitsAboveTheFetcherBudget(t *testing.T) { t.Fatalf("backstop %v does not clear the %v the fetcher allowed itself", got, big) } } + +// A stored node URL may carry a trailing slash — pasting a base URL is the usual +// way an operator enters one, and everything here already treats the two forms +// as the same worker. Concatenating a route onto it produces "//admin/…", which +// no node's router has: the request 404s against a node that is running and +// reachable, and the operator's action fails for a reason nothing reports. +func TestNodeEndpointJoinsATrailingSlashBaseURL(t *testing.T) { + const want = "http://gpu-1:8082/admin/reprobe-capabilities" + for _, base := range []string{ + "http://gpu-1:8082", + "http://gpu-1:8082/", + "http://gpu-1:8082///", + } { + if got := NodeEndpoint(base, "/admin/reprobe-capabilities"); got != want { + t.Errorf("NodeEndpoint(%q) = %q, want %q", base, got, want) + } + } +} + +// The health check is the first thing that runs against a node, so a base URL +// this could not address would leave the node permanently unhealthy. +func TestCheckNodeReachesATrailingSlashBaseURL(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"active_jobs": 1, "capabilities_hash": "sha256:x"}) + })) + defer node.Close() + + healthy, activeJobs, _, hash, _ := CheckNode(context.Background(), &Node{ID: 1, URL: node.URL + "/"}) + if !healthy { + t.Fatal("a node stored with a trailing slash was reported unhealthy") + } + if activeJobs != 1 || hash != "sha256:x" { + t.Fatalf("active jobs = %d, hash = %q; want the node's own answer", activeJobs, hash) + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 6c4f0b3cf..420891980 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -100,6 +100,18 @@ func (p *TranscodePool) ApplyCapabilities(id int, fetchedFrom string, capabiliti applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) } +// NodeEndpoint joins a stored node URL with one of the node's own routes. +// +// A stored URL may carry a trailing slash — an operator pasting a base URL is +// the usual way, and everything here already treats the two forms as the same +// worker. Concatenating a route onto it produces "//admin/…", which the node's +// router does not have: the request 404s, and the operator's action fails +// against a node that is running and reachable. The normalization that makes +// the two forms equal for comparison has to make them equal for addressing too. +func NodeEndpoint(nodeURL, path string) string { + return normalizeNodeURL(nodeURL) + path +} + // sameNodeURL compares two node addresses the way the pools store them, so a // trailing slash on one side is not a different worker. func sameNodeURL(a, b string) bool { From 4f7b31d7d4dcb0d0145bbdfeef258e2e8f6e11b9 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 04:24:24 -0400 Subject: [PATCH 071/163] fix(nodemetrics): a cap the size of the machine is not a cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cpuset fix stopped at cpusets, but a CFS quota sized to the whole host has exactly the same problem and is just as ordinary: a deployment told to use the box writes a quota to match. Such a quota still selected the cgroup usage domain, so an idle Silo container with a 64-core quota on a saturated 64-core host divided its own leaf usage by 64 and reported near-zero — the misreport this correction exists to prevent, arrived at from the other direction. Both now go through cgroupCapBinds: a cap strictly smaller than the machine binds, one as large as the machine does not, and an unknown machine (no /proc/stat) is no reason to discard a cap that may well be real. Saying it once is the point — the rule decides which cgroup's usage is measured, not just what it is divided by, and having it in two places is what let the quota half go unstated. That makes the budget clamp against host capacity unreachable: a quota that gets past cgroupCapBinds is already smaller than the machine, and one that isn't is now reported from /proc/stat instead. It is removed rather than left as a guard whose comment describes a case that cannot occur. TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost asserted the behavior being removed, and its fixture paired a cgroup burning every core with a host reading 1% busy, which no real host produces. It is rewritten as ...ReportsTheHost with the two numbers consistent and distinguishable, joined by a case at the boundary that matters — quota exactly equal to the host. Co-Authored-By: Claude Opus 5 (1M context) --- docs/wiki/admin/monitoring-nodes.md | 12 ++- internal/nodemetrics/cgroupcpu.go | 36 ++++++--- internal/nodemetrics/sampler.go | 25 ++++--- internal/nodemetrics/sampler_test.go | 106 ++++++++++++++++++++++++--- 4 files changed, 143 insertions(+), 36 deletions(-) diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md index 52ec8b6e5..9bd94b89d 100644 --- a/docs/wiki/admin/monitoring-nodes.md +++ b/docs/wiki/admin/monitoring-nodes.md @@ -113,8 +113,16 @@ good numbers rather than blocking the health response; a path the node cannot se at all is reported as unmeasurable rather than as an empty disk. In a container these numbers are the *container's*, corrected against its cgroup -— except on an LXC host running Docker nested inside it, which needs three -bind-mounts to read its own limits instead of the physical machine's. See the LXC +— but only where the cgroup actually caps something. A CPU quota or cpuset the +size of the whole machine restricts nothing, and both are ordinary: an +unconstrained container inherits a cpuset holding every online CPU, and a +deployment sized to the box writes a matching quota. Silo reads those as +uncapped and keeps reporting the host, because on a shared machine the CPU a +neighbor burns is CPU this node cannot have. A real cap — two cores of a +sixty-four core host — switches both the busy figure and the core count to the +cgroup's. This is also true on an LXC host running Docker nested inside it, which +needs three bind-mounts to read its own limits instead of the physical +machine's. See the LXC notes in [Deploy Silo with Docker](../deployment/docker.md#node-metrics), including the caveat that lxcfs only virtualizes `/proc/loadavg` when it runs with loadavg accounting enabled (`lxcfs -l`; off by default on Proxmox), so diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go index 8f24b9a58..aaf8b9bab 100644 --- a/internal/nodemetrics/cgroupcpu.go +++ b/internal/nodemetrics/cgroupcpu.go @@ -93,15 +93,8 @@ var cgroupCPUSetPaths = []string{ // needs no walk of its own; where only the pre-intersection file exists, the // per-level candidates cover the same ground. // -// hostCores is how many CPUs the machine has, and a cpuset that spans all of -// them is not a restriction — it is what every unconstrained container and -// service publishes, because the effective set is inherited from a root that -// holds every online CPU. Counting it as a cap would be worse than ignoring it: -// cgroupCPU treats any cpuset as binding when nothing else caps CPU, so the -// reading would move to this cgroup's own usage, and a nearly idle Silo on a -// saturated shared host would report a few percent instead of the host's load. -// A host size of 0 means /proc/stat could not be counted, and an unknown host -// is no reason to discard a cpuset that may well be real. +// A cpuset that spans the whole host is no cpuset at all — see cgroupCapBinds, +// which is the rule, and which applies to the CFS quota the same way. func cgroupCPUSetCores(paths []string, hostCores int) int { for _, path := range paths { raw, err := os.ReadFile(path) @@ -112,7 +105,7 @@ func cgroupCPUSetCores(paths []string, hostCores int) int { if count <= 0 { continue } - if hostCores > 0 && count >= hostCores { + if !cgroupCapBinds(float64(count), hostCores) { // The effective set is the intersection with every ancestor, so no // file later in the list can narrow what this one just said was the // whole machine. @@ -123,6 +116,29 @@ func cgroupCPUSetCores(paths []string, hostCores int) int { return 0 } +// cgroupCapBinds reports whether a CPU cap of the given size in cores actually +// restricts a process on a host with hostCores CPUs. +// +// A cap as large as the machine restricts nothing, and it is not a rare +// misconfiguration: every unconstrained container and service publishes an +// effective cpuset holding every online CPU, because it inherits one from a root +// that does, and a deployment sized to "the whole box" writes a quota to match. +// +// Reading such a cap as binding is worse than ignoring it. The cap decides which +// cgroup's usage is measured, not just what it is divided by, so an idle Silo +// beside a saturated neighbor on a shared host would report its own few percent +// as the machine's load — the exact misreport this whole correction exists to +// prevent, arrived at from the other direction. +// +// A host size of 0 means /proc/stat could not be counted, and an unknown host is +// no reason to discard a cap that may well be real. +func cgroupCapBinds(cores float64, hostCores int) bool { + if cores <= 0 { + return false + } + return hostCores <= 0 || cores < float64(hostCores) +} + // countCPUSetEntries counts the CPUs in a Linux cpu list ("0-3,8,12-13"). // An unparseable or empty list counts nothing rather than guessing. func countCPUSetEntries(list string) int { diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go index fbaad95c7..559cead5f 100644 --- a/internal/nodemetrics/sampler.go +++ b/internal/nodemetrics/sampler.go @@ -302,7 +302,7 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { // cgroupCPU decides that, because the answer also settles which cgroup's // usage the reading comes from. sample, quota := s.cgroupCPU(now, cgroupCPUSetCores(s.cgroupCPUSetPaths, hostCores)) - if quota <= 0 { + if !cgroupCapBinds(quota, hostCores) { // Nothing caps this process, so its own cgroup is the wrong domain to // measure: the leaf accounts for Silo alone, and Silo idling beside a // saturated neighbor is not an idle machine. Uncapped, the CPU a @@ -310,6 +310,11 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { // /proc/stat reports — and under lxcfs that file is already narrowed to // the container, so this stays right there too. The next pass to find a // cap starts a fresh pair rather than differencing across the switch. + // + // A quota the size of the machine counts as nothing capping it, for the + // same reason a cpuset spanning the machine does: it cannot restrict this + // process beyond what the machine already does, and treating it as a cap + // moves the measurement to a domain that answers a different question. s.prevCgroupCPU = cgroupCPUSample{} return busyPct, cores } @@ -317,21 +322,17 @@ func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { if !sample.valid { return busyPct, cores } - // The budget is capped at what the host can actually give, the same way the - // reported core count is. A quota above the machine's core count is not a - // limit — a 128-core quota on a 64-core host cannot be spent — so dividing - // by it reports a workload saturating every CPU it has as fifty percent - // busy. cores has already been through that cap; budget has to agree with - // it or the percentage and the denominator describe different machines. - budget := quota - if hostCores > 0 && budget > float64(hostCores) { - budget = float64(hostCores) - } + // No clamp against the host's capacity here: a quota that reached this far + // is one cgroupCapBinds accepted, which means it is smaller than the machine. + // A quota the machine cannot supply took the branch above and is reported + // from /proc/stat instead, which is the same answer without dividing one + // population's CPU time by another's capacity. + // // Once the cgroup can be read it is the only honest source, so its answer // stands even when this pass cannot derive one (the first sample, or a // counter reset). Falling back to the host figure would silently mix two // different machines' busyness across intervals. - cgroupPct, _ := cgroupCPUPercent(s.prevCgroupCPU, sample, budget) + cgroupPct, _ := cgroupCPUPercent(s.prevCgroupCPU, sample, quota) s.prevCgroupCPU = sample return cgroupPct, cores } diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go index 67e0db890..620ad4e57 100644 --- a/internal/nodemetrics/sampler_test.go +++ b/internal/nodemetrics/sampler_test.go @@ -1064,11 +1064,12 @@ func TestMemoryLimitEqualToHostRAMIsNotALimit(t *testing.T) { } } -// A quota above the machine's core count is not a limit — a 128-core quota on a -// 64-core host cannot be spent. cgroupQuotaCores already caps the reported core -// count at the host's; the normalization budget has to agree with it, or a -// workload saturating every CPU it has reports fifty percent busy. -func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { +// A quota the machine cannot supply is not a limit — a 128-core quota on a +// 64-core host cannot be spent — so it must not move the measurement into the +// cgroup either. It is the same rule a cpuset spanning the host gets: an idle +// Silo beside a saturated neighbor would otherwise report its own few percent +// as the machine's load, from a quota that restricts it in no way at all. +func TestCPUQuotaAboveHostCapacityReportsTheHost(t *testing.T) { tree := newProcTree(t) clock := newFakeClock() tree.write("loadavg", "0 0 0 0/0 0\n") @@ -1082,7 +1083,7 @@ func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { } return line } - tree.write("stat", hostStat(100, 9900)) + tree.write("stat", hostStat(100, 900)) dir := t.TempDir() usage := filepath.Join(dir, "cpu.stat") @@ -1104,10 +1105,10 @@ func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { }} s.sample(context.Background()) - // 20 seconds of CPU over 5 seconds of wall time: every one of the four CPUs - // the host has, saturated. - writeUsage(20_000_000) - tree.write("stat", hostStat(200, 19800)) + // Silo spends 5 of the interval's 20 core-seconds — a quarter of the host — + // while the host itself is three quarters busy with someone else's work. + writeUsage(5_000_000) + tree.write("stat", hostStat(850, 1150)) clock.advance(5 * time.Second) s.sample(context.Background()) @@ -1115,8 +1116,8 @@ func TestCPUQuotaAboveHostCapacityNormalizesAgainstTheHost(t *testing.T) { if system.Cores != 4 { t.Fatalf("Cores = %d, want the host's 4 — an 8-core quota is not a limit here", system.Cores) } - if system.CPUPct != 100 { - t.Fatalf("CPUPct = %d, want 100 — the node has spent every CPU it can", system.CPUPct) + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 rather than this process's 25", system.CPUPct) } } @@ -1184,3 +1185,84 @@ func TestCPUIgnoresACpusetSpanningTheWholeHost(t *testing.T) { t.Fatalf("Cores = %d, want the host's 8", system.Cores) } } + +// A quota sized to the whole box is the ordinary way a deployment says "use this +// machine", and it restricts nothing. The boundary matters on its own: the +// over-capacity case is a misconfiguration, while quota == host cores is +// deliberate and common, and reading it as a cap moves the measurement into the +// cgroup exactly where the machine is shared. +func TestCPUQuotaEqualToTheHostReportsTheHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 4 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 900)) + + dir := t.TempDir() + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + // Exactly the host's four cores. + if err := os.WriteFile(quota, []byte("400000 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.sample(context.Background()) + + // Silo is nearly idle; the machine it shares is three quarters busy. + writeUsage(200_000) + tree.write("stat", hostStat(850, 1150)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 — a whole-machine quota caps nothing", system.CPUPct) + } + if system.Cores != 4 { + t.Fatalf("Cores = %d, want the host's 4", system.Cores) + } +} + +// The rule is one predicate, and it is the boundary that carries it: strictly +// smaller than the machine binds, as large as the machine does not, and an +// unknown machine is no reason to discard a cap. +func TestCgroupCapBindsOnlyBelowTheHostSize(t *testing.T) { + for _, test := range []struct { + name string + cores float64 + hostCores int + want bool + }{ + {name: "half the host", cores: 2, hostCores: 4, want: true}, + {name: "just under the host", cores: 3.5, hostCores: 4, want: true}, + {name: "the whole host", cores: 4, hostCores: 4, want: false}, + {name: "more than the host", cores: 8, hostCores: 4, want: false}, + {name: "no cap", cores: 0, hostCores: 4, want: false}, + {name: "unknown host keeps a cap", cores: 2, hostCores: 0, want: true}, + {name: "unknown host still has no cap to keep", cores: 0, hostCores: 0, want: false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := cgroupCapBinds(test.cores, test.hostCores); got != test.want { + t.Fatalf("cgroupCapBinds(%v, %d) = %v, want %v", test.cores, test.hostCores, got, test.want) + } + }) + } +} From be193002ea8f676da6359072769425a8f155af75 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 04:40:13 -0400 Subject: [PATCH 072/163] fix(nodes): invalidate every cached view of a node, under one key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways an invalidation missed its target. The v3 caches are keyed by node URL verbatim, and the admin route invalidated with the URL exactly as the row stores it — which may carry a trailing slash the pools have already dropped. The entry deleted was then not the entry planning reads, and the refresh populated a second key, so a node kept serving the backend it had just been moved off until the original entry expired. The two ways into those maps now canonicalize through nodepool.NormalizeNodeURL, which is the same rule that decides whether two URLs are the same worker; the prepared-download preparer's hand-rolled TrimRight converges on it too. And prepared downloads cache a node's inventory separately from protocol-v3 planning, with their own TTL, but only the v3 cache was being invalidated. A QSV-to-NVENC edit left downloads selecting that node for a tone-map executor it no longer had, so the reconfigured worker rejected the recipe or the download fell back locally for no reason. A policy edit invalidates the node, not one reader of it, so both the policy path and the health sweep's capability-change callback now fan out to every cache that holds an answer. The preparer keeps its learned probe budget across the drop, as it does across a failure: how long the node takes to answer has not changed, and the read the invalidation triggers is the cold one that most needs the real number. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 4 ++ internal/api/handlers/playback_v3.go | 11 +++++ internal/api/handlers/playback_v3_test.go | 28 ++++++++++++ internal/api/router.go | 52 +++++++++++++++------- internal/downloads/remote_preparer.go | 37 ++++++++++++--- internal/downloads/remote_preparer_test.go | 29 ++++++++++++ internal/nodepool/transcode_pool.go | 10 +++++ 7 files changed, 151 insertions(+), 20 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index cc3b6fc51..a108f5982 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -2424,6 +2424,10 @@ func main() { if deps.NodeRepo != nil { preparer.SetOriginLookup(deps.NodeRepo) } + // Prepared downloads cache a node's inventory separately from + // protocol-v3 planning, so the router's invalidation has to reach + // both. Set before NewRouter, which composes them. + deps.NodeCapabilityInvalidator = preparer.InvalidateNodeCapabilities artifactMgr := downloads.NewArtifactManager( downloads.NewArtifactRepository(deps.DB), downloads.NewRepository(deps.DB), diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 65f330c59..05cdf9069 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -320,6 +320,7 @@ func (h *PlaybackHandler) localToneMapProbeTimeoutV3() time.Duration { // minutes, short of the ~136 seconds a two-device node legitimately asks for, // so protocol-v3 planning lost its inventory precisely after an invalidation. func (h *PlaybackHandler) remoteToneMapProbeTimeoutV3(nodeURL string) time.Duration { + nodeURL = nodepool.NormalizeNodeURL(nodeURL) h.v3NodeCapabilitiesMu.Lock() budget := h.v3NodeProbeBudgets[nodeURL] h.v3NodeCapabilitiesMu.Unlock() @@ -416,6 +417,10 @@ func (h *PlaybackHandler) lookupRemoteTransformationsV3(ctx context.Context, nod // lookupRemoteCapabilitiesV3 fetches and jointly caches a node's transformation // and tone-map inventory, optionally reusing short-lived fetch failures. func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeURL string, honorCachedFailure bool) (v3NodeCapabilityCache, error) { + // Canonical here and in RefreshNodeCapabilitiesV3, which are the two ways + // into these maps; everything below is reached from one of them and so is + // already keyed the same way. + nodeURL = nodepool.NormalizeNodeURL(nodeURL) now := time.Now() h.v3NodeCapabilitiesMu.Lock() entry, ok := h.v3NodeCapabilities[nodeURL] @@ -532,6 +537,12 @@ func (h *PlaybackHandler) RefreshNodeCapabilitiesV3(nodeURL string) { if h == nil || nodeURL == "" { return } + // Every map here is keyed by the node's canonical address. This entry point + // is reached from the admin route with the URL exactly as the row stores it, + // which may carry a trailing slash the pools have already dropped — and then + // the entry this deletes is not the entry planning reads, so a node keeps + // serving the backend it was just moved off until the old key expires. + nodeURL = nodepool.NormalizeNodeURL(nodeURL) h.v3NodeCapabilitiesMu.Lock() delete(h.v3NodeCapabilities, nodeURL) if h.v3NodeCapabilityInvalidations == nil { diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index fcf25568b..0fc708b55 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -5677,3 +5677,31 @@ func TestPlaybackV3KeepsALearnedBudgetLargerThanThePolicyPrice(t *testing.T) { t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) } } + +// The admin route invalidates with the URL exactly as the row stores it, which +// may carry a trailing slash the pools have already dropped. Keyed verbatim, the +// entry this deletes is not the entry planning reads, so the node keeps serving +// the backend it was just moved off until the old key expires. +func TestRefreshNodeCapabilitiesV3NormalizesTheCacheKey(t *testing.T) { + const nodeURL = "https://gpu-5.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.v3NodeCapabilitiesMu.Lock() + if handler.v3NodeCapabilities == nil { + handler.v3NodeCapabilities = map[string]v3NodeCapabilityCache{} + } + handler.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{expiresAt: time.Now().Add(time.Hour)} + handler.v3NodeCapabilitiesMu.Unlock() + + handler.RefreshNodeCapabilitiesV3(nodeURL + "/") + + handler.v3NodeCapabilitiesMu.Lock() + _, present := handler.v3NodeCapabilities[nodeURL] + invalidations := handler.v3NodeCapabilityInvalidations[nodeURL] + handler.v3NodeCapabilitiesMu.Unlock() + if present { + t.Fatal("the canonical entry survived an invalidation made with a trailing slash") + } + if invalidations == 0 { + t.Fatal("the invalidation was counted under a different key than planning reads") + } +} diff --git a/internal/api/router.go b/internal/api/router.go index f14624d4d..66192e568 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -114,19 +114,23 @@ type Dependencies struct { StreamTelemetry *streamtelemetry.Registry // local observation-only stream telemetry (may be nil) // StreamTelemetryViewCache serves the merged global view with bounded // staleness so the admin parity endpoint never rebuilds it per request. - StreamTelemetryViewCache *streamtelemetry.ViewCache - SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) - StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) - MovieMatchQueueRepo *metadata.MovieMatchQueueRepository - SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository - Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) - NodeRepo *nodepool.Repository // stream node repository (may be nil) - ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) - TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) - NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) - NodeHealthChecker *nodepool.HealthChecker // periodic node health/capability sweep (may be nil) - ResourceSampler *nodemetrics.Sampler // this host's own resource sampler (may be nil) - SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger + StreamTelemetryViewCache *streamtelemetry.ViewCache + SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) + StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) + MovieMatchQueueRepo *metadata.MovieMatchQueueRepository + SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository + Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) + NodeRepo *nodepool.Repository // stream node repository (may be nil) + ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) + TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) + NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) + NodeHealthChecker *nodepool.HealthChecker // periodic node health/capability sweep (may be nil) + // NodeCapabilityInvalidator drops one node's cached capability inventory + // outside the playback handler — the prepared-download preparer holds its + // own. nil where downloads are not wired; set before NewRouter runs. + NodeCapabilityInvalidator func(nodeURL string) + ResourceSampler *nodemetrics.Sampler // this host's own resource sampler (may be nil) + SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger EventBus cache.EventBus AdminStatsProvider handlers.AdminStatsSource Recommender recommendations.Recommender // nil when disabled @@ -224,6 +228,24 @@ func (d *Dependencies) CurrentConfig() *config.Config { // NewRouter creates a chi.Router with all middleware and routes mounted // under /api/v1/. ABS-compat routes (/abs/*, /login, /socket.io/*) are // mounted at the root level when deps.ABSHandler is non-nil. +// invalidateNodeCapabilities drops every cached view of one node's hardware. +// +// There is more than one: protocol-v3 planning holds an inventory, and prepared +// downloads hold their own with its own TTL. A policy edit or a capability hash +// change invalidates the node itself, not one reader of it, so anything that +// caches the answer has to be told — otherwise a QSV-to-NVENC edit keeps +// selecting the node for a tone-map executor it no longer has, and the +// reconfigured worker rejects the recipe or the download falls back locally for +// no reason. +func (deps Dependencies) invalidateNodeCapabilities(playbackHandler *handlers.PlaybackHandler) func(nodeURL string) { + return func(nodeURL string) { + playbackHandler.RefreshNodeCapabilitiesV3(nodeURL) + if deps.NodeCapabilityInvalidator != nil { + deps.NodeCapabilityInvalidator(nodeURL) + } + } +} + func NewRouter(deps Dependencies) chi.Router { declareNativeMediaRoutes() r := chi.NewRouter() @@ -1038,7 +1060,7 @@ func NewRouter(deps Dependencies) chi.Router { // The health sweep sees a node's capability hash change long before this // cache would expire, so let it invalidate directly. Wired here rather // than at checker construction because the handler does not exist yet. - deps.NodeHealthChecker.SetCapabilitiesChangedCallback(playbackHandler.RefreshNodeCapabilitiesV3) + deps.NodeHealthChecker.SetCapabilitiesChangedCallback(deps.invalidateNodeCapabilities(playbackHandler)) realtimeHub := deps.PlaybackRealtimeHub if realtimeHub == nil { @@ -3172,7 +3194,7 @@ func NewRouter(deps Dependencies) chi.Router { // server's cached view of the node wrong the // moment it lands; the same invalidation the // health sweep uses drops it. - nodeHandler.SetCapabilityInvalidator(playbackHandler.RefreshNodeCapabilitiesV3) + nodeHandler.SetCapabilityInvalidator(deps.invalidateNodeCapabilities(playbackHandler)) // A node with no override of its own runs the // cluster's acceleration policy, and how many // devices that names is what decides how long its diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index cfded198b..5374a7630 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -276,7 +276,7 @@ func (p *NodeAwarePreparer) ToneMapModeAvailable(ctx context.Context, mode tonem capable := make(map[string]struct{}) for nodeURL, capabilities := range byNode { if capabilities.Supports(mode, kind) { - capable[strings.TrimRight(nodeURL, "/")] = struct{}{} + capable[nodepool.NormalizeNodeURL(nodeURL)] = struct{}{} } } available := selector.TranscodeWorkAvailableWith(func(candidate *nodepool.Node) bool { @@ -334,7 +334,7 @@ func (p *NodeAwarePreparer) audioBoostCapableNodeURLs(ctx context.Context) map[s } func (p *NodeAwarePreparer) audioBoostCapabilityForNode(ctx context.Context, nodeURL string) (bool, error) { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) if entry, ok := p.cachedRemoteCapabilitiesForNode(nodeURL, time.Now()); ok { return supportsAudioBoostTransformation(entry.transformations), entry.err } @@ -397,7 +397,7 @@ func (p *NodeAwarePreparer) toneMapCapabilitiesByNode(ctx context.Context) (map[ // toneMapCapabilitiesForNode returns a defensive copy of a fresh cached // inventory or retrieves the node's authenticated hardware capabilities. func (p *NodeAwarePreparer) toneMapCapabilitiesForNode(ctx context.Context, nodeURL string) (tonemap.Capabilities, error) { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) if capabilities, err, ok := p.cachedToneMapCapabilitiesForNode(nodeURL, time.Now()); ok { return capabilities, err } @@ -486,7 +486,7 @@ func (p *NodeAwarePreparer) ToneMapCapabilityTimeout() time.Duration { } func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Duration { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() timeout := p.capabilities[nodeURL].probeRequestTimeout p.capabilityMu.Unlock() @@ -537,10 +537,37 @@ type transcodeNodeLookup interface { TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) } +// InvalidateNodeCapabilities drops one node's cached inventory so the next +// prepared download reads it again. +// +// It exists for the same reason the playback-v3 cache has one: an operator +// changing a node's acceleration policy, or the health sweep noticing the node's +// capability hash move, makes this cache wrong the moment it lands — and a +// download planned from it selects the node for a tone-map executor it no longer +// has, so the reconfigured worker rejects the recipe or the download falls back +// locally for no reason. A minute of TTL is a minute of that. +// +// The learned probe budget survives, exactly as it does across a failure: how +// long this node takes to answer has not changed, and the read the invalidation +// triggers is the cold one that most needs the real number. +func (p *NodeAwarePreparer) InvalidateNodeCapabilities(nodeURL string) { + if p == nil || nodeURL == "" { + return + } + nodeURL = nodepool.NormalizeNodeURL(nodeURL) + p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + entry, ok := p.capabilities[nodeURL] + if !ok { + return + } + p.capabilities[nodeURL] = remoteToneMapCapabilities{probeRequestTimeout: entry.probeRequestTimeout} +} + // cacheToneMapCapabilityFailure negatively caches an unreachable or invalid // node briefly so repeated artifact planning does not amplify the failure. func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, err error) { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() if p.capabilities == nil { p.capabilities = make(map[string]remoteToneMapCapabilities) diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index 830c84f87..a2dea113e 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -1138,3 +1138,32 @@ func TestNodeAwarePreparerKeepsALearnedBudgetLargerThanThePolicyPrice(t *testing t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) } } + +// A policy edit or a capability-hash change makes this cache wrong the moment it +// lands. Left for its TTL, a download planned from it selects the node for a +// tone-map executor it no longer has, and the reconfigured worker rejects the +// recipe or the download falls back locally for no reason. +func TestNodeAwarePreparerInvalidateNodeCapabilitiesDropsTheInventory(t *testing.T) { + const nodeURL = "https://node.example" + preparer := NewNodeAwarePreparer(nil, nil, nil) + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + capabilities: tonemap.Capabilities{{Mode: tonemap.ModeHardware, Backend: tonemap.BackendQSV}}, + probeRequestTimeout: 161 * time.Second, + expiresAt: time.Now().Add(time.Minute), + } + + // The stored URL carries a trailing slash the pools have already dropped; it + // still has to reach the entry planning reads. + preparer.InvalidateNodeCapabilities(nodeURL + "/") + + entry := preparer.capabilities[nodeURL] + if len(entry.capabilities) != 0 || !entry.expiresAt.IsZero() { + t.Fatalf("inventory survived the invalidation: %+v", entry) + } + // The budget describes how long the node takes to answer, which a policy + // change does not alter — and the read this invalidation triggers is the + // cold one that most needs the real number. + if entry.probeRequestTimeout != 161*time.Second { + t.Fatalf("probe budget = %s, want the learned 161s preserved", entry.probeRequestTimeout) + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 420891980..6fde483b8 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -100,6 +100,16 @@ func (p *TranscodePool) ApplyCapabilities(id int, fetchedFrom string, capabiliti applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) } +// NormalizeNodeURL is how a node's address is written wherever it is used as an +// identity: a map key, a comparison, or the base of a request. +// +// A stored URL may carry a trailing slash and mean the same worker, so anything +// that keys by the raw value ends up with two entries for one node — and the +// one an invalidation deletes is not the one a lookup finds. +func NormalizeNodeURL(nodeURL string) string { + return normalizeNodeURL(nodeURL) +} + // NodeEndpoint joins a stored node URL with one of the node's own routes. // // A stored URL may carry a trailing slash — an operator pasting a base URL is From 7e496610bfa5b2db037dc8bad11f1df62f550026 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 04:49:44 -0400 Subject: [PATCH 073/163] test(scanner): write fake tools under the fork lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go check failed on TestProbeFileFallsBackToVideoPacketsForInvalidMetadata with "text file busy" from a stub the test had just written. Nothing in this branch touches the scanner; the test is simply racing the rest of the package. execve refuses a file anyone still holds open for writing. Go opens with O_CLOEXEC, so the writing fd is not meant to survive an exec — but O_CLOEXEC only takes effect at the child's execve, so a fork landing between this open and its close leaves that child holding a copy of the fd for as long as it takes to exec something else. A test that runs the stub it just wrote loses, and it loses for whichever test happened to be beside a fork: never the same one twice, and never on a laptop running one package at a time. writeFakeTool holds syscall.ForkLock — the lock os/exec takes around fork for this exact class of problem — across the write, so no fork can observe the fd and the window does not exist rather than being waited out. That matters more than a retry would: several of these stubs record their invocations or block on a gate, so a probe exec to "wait out" the window would corrupt the very thing the test asserts on. Every write-then-exec site in the package goes through it. Not this PR's concern, but it is this PR's red check. Co-Authored-By: Claude Opus 5 (1M context) --- internal/scanner/audiobook_test.go | 8 ++--- internal/scanner/fake_tool_test.go | 35 +++++++++++++++++++ internal/scanner/probe_duration_test.go | 12 ++----- internal/scanner/probe_primary_video_test.go | 9 ++--- .../probe_repair_copy_safety_persist_test.go | 8 ++--- 5 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 internal/scanner/fake_tool_test.go diff --git a/internal/scanner/audiobook_test.go b/internal/scanner/audiobook_test.go index f4805bba9..f6c9ba719 100644 --- a/internal/scanner/audiobook_test.go +++ b/internal/scanner/audiobook_test.go @@ -77,9 +77,7 @@ func (f *fakeScannerCoverCacher) CacheAudiobookCover(_ context.Context, data []b func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nprintf cover-bytes\n"), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, "#!/bin/sh\nprintf cover-bytes\n") exec := &fakeAudiobookPosterExec{} cacher := &fakeScannerCoverCacher{} @@ -122,9 +120,7 @@ func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { func TestApplyAudiobookEmbeddedCoverPreservesExistingPoster(t *testing.T) { dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nprintf cover-bytes\n"), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, "#!/bin/sh\nprintf cover-bytes\n") reader := &fakeAudiobookPosterReader{posterPath: "local/audiobooks/content-1/poster/original.webp"} exec := &fakeAudiobookPosterExec{} diff --git a/internal/scanner/fake_tool_test.go b/internal/scanner/fake_tool_test.go new file mode 100644 index 000000000..141f14014 --- /dev/null +++ b/internal/scanner/fake_tool_test.go @@ -0,0 +1,35 @@ +package scanner + +import ( + "os" + "syscall" + "testing" +) + +// writeFakeTool writes an executable stub for ffmpeg or ffprobe and fails the +// test if it cannot. +// +// The fork lock is what makes this different from a plain os.WriteFile, and it +// is the whole point of the helper. execve refuses a file that anyone still has +// open for writing, with ETXTBSY — "text file busy". Go opens the file +// O_CLOEXEC, so the fd is not meant to outlive an exec, but O_CLOEXEC only +// takes effect at the child's execve: a fork landing between this open and its +// close leaves that child holding a copy of the writing fd for the whole time +// it takes to exec something else. A test that then runs the stub it just +// wrote fails, and it fails for whichever test happened to be next to a fork — +// never the same one twice, and never on a laptop running one package at a +// time. +// +// syscall.ForkLock is the lock os/exec takes around fork for exactly this class +// of problem. Holding it for the write means no fork can observe the fd at all, +// so the window does not exist rather than being waited out: no retry, no +// sleep, no exec of a stub whose side effects a test is about to assert on. +func writeFakeTool(t *testing.T, path, script string) { + t.Helper() + syscall.ForkLock.Lock() + err := os.WriteFile(path, []byte(script), 0o755) + syscall.ForkLock.Unlock() + if err != nil { + t.Fatalf("writing fake tool %s: %v", path, err) + } +} diff --git a/internal/scanner/probe_duration_test.go b/internal/scanner/probe_duration_test.go index 121b4206c..fc1506cbc 100644 --- a/internal/scanner/probe_duration_test.go +++ b/internal/scanner/probe_duration_test.go @@ -315,9 +315,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "long.mp4") if err != nil { @@ -439,9 +437,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") if err != nil { @@ -469,9 +465,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") if err != nil { diff --git a/internal/scanner/probe_primary_video_test.go b/internal/scanner/probe_primary_video_test.go index ee9f0371d..177861923 100644 --- a/internal/scanner/probe_primary_video_test.go +++ b/internal/scanner/probe_primary_video_test.go @@ -3,7 +3,6 @@ package scanner import ( "context" "errors" - "os" "path/filepath" "testing" "time" @@ -86,9 +85,7 @@ func TestProbePrimaryVideoTrackRejectsMalformedJSON(t *testing.T) { func TestProbePrimaryVideoTrackHonorsCallerTimeout(t *testing.T) { ffprobe := filepath.Join(t.TempDir(), "ffprobe") - if err := os.WriteFile(ffprobe, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { - t.Fatal(err) - } + writeFakeTool(t, ffprobe, "#!/bin/sh\nexec sleep 30\n") ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() _, err := ProbePrimaryVideoTrack(ctx, ffprobe, "movie.mkv") @@ -101,8 +98,6 @@ func writePrimaryVideoFFprobe(t *testing.T, output string) string { t.Helper() path := filepath.Join(t.TempDir(), "ffprobe") script := "#!/bin/sh\nprintf '%s' '" + output + "'\n" - if err := os.WriteFile(path, []byte(script), 0o755); err != nil { - t.Fatal(err) - } + writeFakeTool(t, path, script) return path } diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index 20f2d5e6e..554e13665 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -32,9 +32,7 @@ func fakeFFmpeg(t *testing.T, stdoutPayload string, delay time.Duration) (string sleep = fmt.Sprintf("sleep %.2f\n", delay.Seconds()) } script := fmt.Sprintf("#!/bin/sh\necho run >> %q\n%sprintf '%s'\n", logPath, sleep, stdoutPayload) - if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, script) return ffmpegPath, func() int { return countFFmpegRuns(t, logPath) } } @@ -51,9 +49,7 @@ func fakeFFmpegGated(t *testing.T, stdoutPayload string) (ffmpegPath string, run // The invocation is logged before the gate so the log is the signal that // this process has started, not that it has finished. script := fmt.Sprintf("#!/bin/sh\necho run >> %q\nwhile [ ! -f %q ]; do sleep 0.01; done\nprintf '%s'\n", logPath, releasePath, stdoutPayload) - if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { - t.Fatalf("write gated fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, script) runs = func() int { return countFFmpegRuns(t, logPath) } release = func() { if err := os.WriteFile(releasePath, nil, 0o644); err != nil { From 0d3b54d11131bf1e6d4be904309185abee486b36 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 04:55:02 -0400 Subject: [PATCH 074/163] fix(downloads): fence the capability cache against an overtaken fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping the entry was not enough. A probe already in flight when the invalidation arrives writes the report it was sent to collect, so the pre-edit inventory comes back for a full TTL and prepared downloads keep selecting a tone-map executor the reconfigured node no longer has. The cold-cache case was worse: there was no entry to drop, so the invalidation recorded nothing at all and the fetch installed its answer unopposed — and a fetch in flight is exactly what a policy edit interrupts, since planning is what prompted the operator to look. The playback-v3 cache already had the answer: count invalidations per node, snapshot the count before asking the node anything, and install only if it has not moved. The count is now kept whether or not anything is cached, which is what closes the cold case. The overtaken answer still goes back to the caller waiting on it. Its request is already in flight, most policy edits do not remove the executor it is about to pick, and refusing would fail a download over a change that probably does not affect it. Only the durable part is withheld, so the next caller reads the node again rather than reading this answer for a minute. Co-Authored-By: Claude Opus 5 (1M context) --- internal/downloads/remote_preparer.go | 40 ++++++++++++++-- internal/downloads/remote_preparer_test.go | 55 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index 5374a7630..6b62bd6c5 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -35,6 +35,12 @@ type NodeAwarePreparer struct { capabilityMu sync.Mutex capabilities map[string]remoteToneMapCapabilities capabilityFlight singleflight.Group + // capabilityInvalidations counts how many times each node's inventory has + // been dropped. A fetch snapshots it before asking the node and refuses to + // install its answer if it has moved since — otherwise a probe already in + // flight when an operator changed the node's policy writes the report it + // was sent to collect, restoring the pre-edit inventory for a full TTL. + capabilityInvalidations map[string]uint64 } // remoteToneMapCapabilities caches one node's validated inventory; an empty @@ -446,6 +452,9 @@ func (p *NodeAwarePreparer) cachedRemoteCapabilitiesForNode(nodeURL string, now } func (p *NodeAwarePreparer) fetchToneMapCapabilitiesForNode(ctx context.Context, nodeURL string) (tonemap.Capabilities, error) { + // Snapshotted before the node is asked anything, so an invalidation landing + // during the request is visible at install time. + generation := p.capabilityInvalidationsFor(nodeURL) cfg := p.config() if cfg == nil || strings.TrimSpace(cfg.Auth.JWTSecret) == "" { err := errors.New("transcode node credentials unavailable") @@ -471,14 +480,30 @@ func (p *NodeAwarePreparer) fetchToneMapCapabilitiesForNode(ctx context.Context, probeRequestTimeout: normalizeRemoteToneMapProbeTimeout(info.ProbeRequestTimeoutMillis), } p.capabilityMu.Lock() - if p.capabilities == nil { - p.capabilities = make(map[string]remoteToneMapCapabilities) + if p.capabilityInvalidations[nodeURL] == generation { + if p.capabilities == nil { + p.capabilities = make(map[string]remoteToneMapCapabilities) + } + p.capabilities[nodeURL] = entry } - p.capabilities[nodeURL] = entry p.capabilityMu.Unlock() + // The answer still goes back to the caller that is waiting on it, overtaken + // or not. Its request is already in flight, most policy edits do not remove + // the executor it is about to pick, and refusing would fail a download over + // a change that probably does not affect it. What must not happen is the + // durable part: nothing is written, so the next caller asks the node again + // rather than reading this answer for a minute. return append(tonemap.Capabilities(nil), entry.capabilities...), nil } +// capabilityInvalidationsFor reports how many times a node's inventory has been +// dropped. +func (p *NodeAwarePreparer) capabilityInvalidationsFor(nodeURL string) uint64 { + p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + return p.capabilityInvalidations[nodeURL] +} + // ToneMapCapabilityTimeout returns the complete cold-node capability budget // used when pooled nodes are the only eligible tone-map executors. func (p *NodeAwarePreparer) ToneMapCapabilityTimeout() time.Duration { @@ -557,6 +582,15 @@ func (p *NodeAwarePreparer) InvalidateNodeCapabilities(nodeURL string) { nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() defer p.capabilityMu.Unlock() + // Counted whether or not anything is cached. A cold cache is the case where + // dropping an entry does nothing and a fetch is most likely to be in flight: + // the invalidation that follows a policy edit arrives while planning is + // already asking the node, and without a mark that fetch's answer would be + // installed after the edit as though it described the node afterwards. + if p.capabilityInvalidations == nil { + p.capabilityInvalidations = make(map[string]uint64) + } + p.capabilityInvalidations[nodeURL]++ entry, ok := p.capabilities[nodeURL] if !ok { return diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index a2dea113e..b54353424 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -1167,3 +1167,58 @@ func TestNodeAwarePreparerInvalidateNodeCapabilitiesDropsTheInventory(t *testing t.Fatalf("probe budget = %s, want the learned 161s preserved", entry.probeRequestTimeout) } } + +// A policy edit invalidates while planning is already asking the node. The +// answer in flight describes the node before the edit, so installing it would +// restore the pre-edit inventory for a full TTL — and downloads would keep +// selecting a tone-map executor the reconfigured worker no longer has. +func TestNodeAwarePreparerDoesNotCacheAnOvertakenCapabilityFetch(t *testing.T) { + var hits atomic.Int32 + invalidated := make(chan struct{}) + released := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + // The edit lands while this request is still open. + close(invalidated) + <-released + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + ToneMapCapabilities: tonemap.Capabilities{{ + Mode: tonemap.ModeSoftware, Backend: tonemap.BackendSoftware, + Filter: tonemap.SoftwareFilterBT2390, SourceKinds: []tonemap.SourceKind{tonemap.SourcePQ}, + }}, + }) + })) + defer remote.Close() + cfg := &config.Config{} + cfg.Auth.JWTSecret = "secret" + preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) + + fetched := make(chan error, 1) + go func() { + _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + fetched <- err + }() + <-invalidated + preparer.InvalidateNodeCapabilities(remote.URL) + close(released) + if err := <-fetched; err != nil { + t.Fatalf("the caller waiting on the fetch got an error: %v", err) + } + + // Nothing durable was written, so the next planning pass asks the node again + // rather than reading the overtaken answer. + key := nodepool.NormalizeNodeURL(remote.URL) + preparer.capabilityMu.Lock() + entry, cached := preparer.capabilities[key] + preparer.capabilityMu.Unlock() + if cached && len(entry.capabilities) > 0 { + t.Fatalf("an overtaken fetch repopulated the cache: %+v", entry) + } + if _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL); err != nil { + t.Fatalf("second lookup: %v", err) + } + if hits.Load() != 2 { + t.Fatalf("node was asked %d times, want a second read after the invalidation", hits.Load()) + } +} From e40ae9374fdb4190ab7606a9da28e7ea204905b1 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 05:11:02 -0400 Subject: [PATCH 075/163] fix(downloads): fence failed capability fetches too, and correct the drift docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The invalidation fence covered the successful install and left the failure path open, which is the sharper case: a negative entry does not merely go stale, it takes the node out of planning entirely for its TTL. A fetch that fails because the node is mid-reload — exactly what a policy edit causes — kept downloads off the node that edit had just reconfigured, falling back locally or failing where fallback is off, after the change that would have fixed it had landed. Two documentation corrections beside it. capability_drift_baseline was written as `{"devices": [[alias, ...]]}` when the field serializes objects carrying a uuid alongside the aliases; a client written from the documented schema would have failed to decode every baseline with a lost device. And the skipped-backend bullet claimed a backend that stops being reported is a loss, contradicting both the rule stated three bullets above it and computeCapabilityDrift, which ignores an absent backend precisely because a policy change removes one. An operator reading it would have waited for a warning nothing writes. The rule that was already correct now also names nvidia_gpu_uuids, since that is where a card without a DRM node is tracked. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 12 ++--- internal/downloads/remote_preparer.go | 21 +++++++-- internal/downloads/remote_preparer_test.go | 55 +++++++++++++++++++++- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/docs/admin-api.md b/docs/admin-api.md index d60063333..5f1e19165 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -34,7 +34,7 @@ Always `200 OK` with a JSON array. | `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | | `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | | `capability_drift` | string | Human-readable note describing how the node's hardware got worse at the last capability refetch. Omitted when the last refetch found no regression (see below). | -| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": [...], "devices": [[alias, ...], ...]}`. Never present without `capability_drift`; absent with it only for a note written before this field existed (see below). Each device is every stable name it answered to, so it is recognized if it returns renumbered. | +| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": ["nvenc"], "devices": [{"uuid": "GPU-8a7b…", "aliases": ["GPU-8a7b…", "0000:03:00.0", "/dev/dri/renderD128"]}]}`. Never present without `capability_drift`; absent with it only for a note written before this field existed (see below). Each device carries every stable name it answered to, so it is recognized if it returns renumbered; `uuid` is held apart because it is the only name that can prove a *different* card, a replacement in the same slot inheriting both the slot and the render path. Either key is omitted when empty. | ### Acceleration overrides @@ -270,14 +270,14 @@ Semantics worth knowing: absent from the report was not asked about — detection probes the backends the configured `hw_device` gives it candidates for — so repointing a node from a QSV render path to an NVENC index is not a regression. Hardware actually - disappearing shows up in `render_devices`, which is the host's own inventory - and owes nothing to the configuration. + disappearing shows up in the device inventory, which is the host's own and + owes nothing to the configuration: `render_devices` for cards with a DRM node, + and `nvidia_gpu_uuids` for those without one, which is the ordinary shape of + an NVENC container. A card that vanishes from either is a loss. - A backend reported as `skipped` neither sets the note nor holds it open. Skipping means no probe ran because the node cannot open the backend's configured devices, which is a statement about access rather than about - hardware — the GPU column reports that state separately. A backend that was - probed and *failed* is a loss, as is one that stopped being reported at all, - which is what a card disappearing looks like. + hardware — the GPU column reports that state separately. - Only a loss is reported. Added hardware is not drift. - A node's first stored report carries none — there is nothing to compare it against. diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index 6b62bd6c5..2c7a5744d 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -458,19 +458,19 @@ func (p *NodeAwarePreparer) fetchToneMapCapabilitiesForNode(ctx context.Context, cfg := p.config() if cfg == nil || strings.TrimSpace(cfg.Auth.JWTSecret) == "" { err := errors.New("transcode node credentials unavailable") - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } requestCtx, cancel := context.WithTimeout(ctx, p.remoteToneMapProbeTimeout(nodeURL)) defer cancel() info, status, err := transcodenode.FetchHWCapabilities(requestCtx, p.probeClient, nodeURL, cfg.Auth.JWTSecret) if err != nil { - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } if status != http.StatusOK { err := fmt.Errorf("transcode node returned %d", status) - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } entry := remoteToneMapCapabilities{ @@ -600,9 +600,21 @@ func (p *NodeAwarePreparer) InvalidateNodeCapabilities(nodeURL string) { // cacheToneMapCapabilityFailure negatively caches an unreachable or invalid // node briefly so repeated artifact planning does not amplify the failure. -func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, err error) { +// +// Fenced on the same invalidation count a successful result is, and for a +// sharper reason: a negative entry does not merely go stale, it takes the node +// out of planning entirely for its TTL. A fetch that failed because the node +// was mid-reload — which is exactly what a policy edit causes — would otherwise +// keep downloads off the node it was just reconfigured for, falling back +// locally or failing outright where local fallback is off, after the change +// that would have fixed it had already landed. +func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, generation uint64, err error) { nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + if p.capabilityInvalidations[nodeURL] != generation { + return + } if p.capabilities == nil { p.capabilities = make(map[string]remoteToneMapCapabilities) } @@ -613,7 +625,6 @@ func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, err er expiresAt: time.Now().Add(remoteToneMapCapabilityErrorTTL), probeRequestTimeout: probeRequestTimeout, } - p.capabilityMu.Unlock() } func remotePreparedArtifact(node *nodepool.Node, result downloadprepare.Result) PreparedArtifact { diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index b54353424..e0be3aaf6 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -833,7 +833,7 @@ func TestNodeAwarePreparerCapabilityFailurePreservesNodeProbeBudget(t *testing.T expiresAt: time.Now().Add(-time.Second), } - preparer.cacheToneMapCapabilityFailure(nodeURL, context.DeadlineExceeded) + preparer.cacheToneMapCapabilityFailure(nodeURL, preparer.capabilityInvalidationsFor(nodeURL), context.DeadlineExceeded) if got, want := preparer.remoteToneMapProbeTimeout(nodeURL), 161*time.Second; got != want { t.Fatalf("probe timeout after transient failure = %s, want preserved node budget %s", got, want) @@ -1222,3 +1222,56 @@ func TestNodeAwarePreparerDoesNotCacheAnOvertakenCapabilityFetch(t *testing.T) { t.Fatalf("node was asked %d times, want a second read after the invalidation", hits.Load()) } } + +// A negative entry does not merely go stale, it takes the node out of planning +// for its TTL. A fetch failing because the node was mid-reload — exactly what a +// policy edit causes — would otherwise keep downloads off the node that edit had +// just reconfigured, after the change that would have fixed it already landed. +func TestNodeAwarePreparerDoesNotCacheAnOvertakenCapabilityFailure(t *testing.T) { + var hits atomic.Int32 + invalidated := make(chan struct{}) + released := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + close(invalidated) + <-released + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + ToneMapCapabilities: tonemap.Capabilities{{ + Mode: tonemap.ModeSoftware, Backend: tonemap.BackendSoftware, + Filter: tonemap.SoftwareFilterBT2390, SourceKinds: []tonemap.SourceKind{tonemap.SourcePQ}, + }}, + }) + })) + defer remote.Close() + cfg := &config.Config{} + cfg.Auth.JWTSecret = "secret" + preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) + + failed := make(chan error, 1) + go func() { + _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + failed <- err + }() + <-invalidated + preparer.InvalidateNodeCapabilities(remote.URL) + close(released) + if err := <-failed; err == nil { + t.Fatal("the node answered 503 and the caller saw no error") + } + + // The reconfigured node is asked again rather than sitting behind a negative + // entry the invalidation should have outranked. + capabilities, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + if err != nil { + t.Fatalf("second lookup: %v", err) + } + if len(capabilities) != 1 { + t.Fatalf("capabilities = %#v, want the reconfigured node's answer", capabilities) + } + if hits.Load() != 2 { + t.Fatalf("node was asked %d times, want a second read after the invalidation", hits.Load()) + } +} From 58cba08f4c8f7385a46ad7b68442c7245ac9c0a6 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:36:50 -0400 Subject: [PATCH 076/163] ci(macos): build native arm64 server artifact --- .github/workflows/macos.yml | 133 ++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/macos.yml diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..0d5414103 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,133 @@ +name: Native macOS Server + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: macos-${{ github.ref }} + cancel-in-progress: true + +env: + GOPROXY: https://proxy.golang.org,direct + GOPRIVATE: github.com/Silo-Server/* + GONOSUMDB: github.com/Silo-Server/* + +permissions: + contents: read + +jobs: + build: + name: Build Darwin arm64 + runs-on: macos-15 + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install native dependencies + run: brew install vips ffmpeg + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + package_json_file: web/package.json + + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: web/pnpm-lock.yaml + + - name: Install frontend dependencies + working-directory: web + run: pnpm install --frozen-lockfile + + - name: Build frontend assets + working-directory: web + run: pnpm run build + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Reject committed local SDK replaces + run: | + if grep -Eq '^replace github.com/Silo-Server/silo-plugin-sdk => /' go.mod; then + echo "go.mod contains a machine-local silo-plugin-sdk replace." + exit 1 + fi + + - name: Validate SDK resolves from the module graph + env: + GOWORK: off + run: | + sdk_module_json="$(mktemp)" + go list -m -json github.com/Silo-Server/silo-plugin-sdk > "$sdk_module_json" + if grep -q '"Main": true' "$sdk_module_json"; then + echo "silo-plugin-sdk resolved from the local workspace instead of go.mod." + cat "$sdk_module_json" + exit 1 + fi + + - name: Test Go on macOS + run: go test ./... + + - name: Build native server archive + id: build + env: + BUILDINFO_PKG: github.com/Silo-Server/silo-server/internal/buildinfo + run: | + built_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + archive="silo-darwin-arm64-${GITHUB_SHA::8}.tar.gz" + package_dir="dist/silo-darwin-arm64" + mkdir -p "$package_dir" + + go build -trimpath \ + -ldflags "-X ${BUILDINFO_PKG}.revisionOverride=${GITHUB_SHA} -X ${BUILDINFO_PKG}.dirtyOverride=false -X ${BUILDINFO_PKG}.builtAtOverride=${built_at}" \ + -o "$package_dir/silo" ./cmd/silo/ + + cp LICENSE "$package_dir/LICENSE" + go version -m "$package_dir/silo" > "$package_dir/build-info.txt" + otool -L "$package_dir/silo" > "$package_dir/linked-libraries.txt" + tar -czf "dist/$archive" -C dist silo-darwin-arm64 + (cd dist && shasum -a 256 "$archive" > "$archive.sha256") + echo "archive=$archive" >> "$GITHUB_OUTPUT" + + - name: Verify native runtime and VideoToolbox + env: + ARCHIVE: ${{ steps.build.outputs.archive }} + run: | + file dist/silo-darwin-arm64/silo | grep -q 'Mach-O 64-bit executable arm64' + dist/silo-darwin-arm64/silo -h >/dev/null + grep -aFq "$GITHUB_SHA" dist/silo-darwin-arm64/silo + + ffmpeg -hide_banner -hwaccels > "$RUNNER_TEMP/ffmpeg-hwaccels.txt" 2>/dev/null + grep -Fxq 'videotoolbox' "$RUNNER_TEMP/ffmpeg-hwaccels.txt" + ffmpeg -hide_banner -encoders > "$RUNNER_TEMP/ffmpeg-encoders.txt" 2>/dev/null + grep -q 'h264_videotoolbox' "$RUNNER_TEMP/ffmpeg-encoders.txt" + ffmpeg -hide_banner -loglevel error \ + -f lavfi -i color=c=black:s=128x72:r=24 \ + -frames:v 12 -c:v h264_videotoolbox -f null - + + (cd dist && shasum -a 256 -c "$ARCHIVE.sha256") + + - name: Upload native server + uses: actions/upload-artifact@v4 + with: + name: silo-darwin-arm64-${{ github.sha }} + path: | + dist/${{ steps.build.outputs.archive }} + dist/${{ steps.build.outputs.archive }}.sha256 + if-no-files-found: error + retention-days: 14 From 26a0abb19b1e1bb3ec67d9c1211e3a327425fca1 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:52:57 -0400 Subject: [PATCH 077/163] feat(admin): reorganize admin settings into 9 tabs with Essential/Advanced tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admins deploying Silo were getting lost in 20 settings tabs that showed every server key at once. This collapses the settings area into nine intent-oriented tabs (General, Appearance, Security & Access, Library & Metadata, Playback, Integrations, Notifications, Compatibility, Infrastructure) and gives every setting a tier: Essential (always shown), Advanced (one collapsible per section), or Hidden (no UI; key still saved and readable via the API). Shared primitives replace the bespoke per-tab patterns: AdvancedSection, SecretField (one configured/replace/keep control), LimitField ("Unlimited" instead of 0-means-unlimited hints), ProviderCard (uniform third-party credential cards), and a RestartBadge sourced from config.RestartRequired via a new GET /admin/settings/restart-keys endpoint. Log Retention and Theming move onto the shared useSettingsForm + SaveBar model, and useSettingsForm now guards beforeunload when dirty. Navigation: Settings becomes its own sidebar group with the nine tabs inline, Autoscan lives under Libraries, the command palette (⌘K) mounts globally with a visible search button, and every old ?tab= id redirects to the tab that absorbed it. Missing admin defaults are registered (matching the loader fallbacks so runtime behaviour is unchanged). Deliberately deferred: renaming the un-namespaced transcode keys to playback.*, a canonical server.public_url, and deleting legacy s3.operational_* rows. Co-Authored-By: Claude Fable 5 --- docs/architecture/admin-settings-ux.md | 76 + internal/api/handlers/admin.go | 34 +- .../api/handlers/admin_restart_keys_test.go | 61 + .../handlers/admin_settings_checks_test.go | 77 + internal/api/router.go | 1 + internal/config/admin_settings.go | 73 +- internal/config/admin_settings_test.go | 68 + internal/config/restart_keys.go | 30 +- internal/config/restart_keys_test.go | 33 + web/src/App.tsx | 7 +- web/src/components/AdminLayout.tsx | 69 +- .../AdminSectionCommandDialog.test.tsx | 14 +- .../components/AdminSectionCommandDialog.tsx | 24 +- web/src/components/AdminSidebar.tsx | 13 +- .../settings/AdvancedSection.test.tsx | 143 ++ .../components/settings/AdvancedSection.tsx | 113 ++ .../components/settings/LimitField.test.tsx | 97 ++ web/src/components/settings/LimitField.tsx | 99 ++ web/src/components/settings/ProviderCard.tsx | 248 +++ web/src/components/settings/RestartBadge.tsx | 27 + .../components/settings/SecretField.test.tsx | 107 ++ web/src/components/settings/SecretField.tsx | 114 ++ web/src/hooks/queries/admin/settings.ts | 21 + web/src/hooks/queries/keys.ts | 1 + web/src/hooks/useRestartKeys.test.ts | 43 + web/src/hooks/useRestartKeys.ts | 43 + web/src/hooks/useSettingsForm.test.ts | 51 + web/src/hooks/useSettingsForm.ts | 20 + web/src/lib/adminNavigation.ts | 71 +- web/src/lib/adminSettingsSearch.ts | 690 ++++---- web/src/pages/AdminAutoscan.tsx | 28 +- web/src/pages/AdminDashboard.tsx | 15 - web/src/pages/AdminDevices.tsx | 27 +- web/src/pages/AdminLibraries.tsx | 727 ++++---- .../AIServicesSettings.test.tsx | 234 --- .../admin-settings/AIServicesSettings.tsx | 609 ------- .../AdminSettingsLayout.test.tsx | 109 +- .../admin-settings/AdminSettingsLayout.tsx | 63 +- .../AppearanceSettings.test.tsx | 153 ++ .../admin-settings/AppearanceSettings.tsx | 536 ++++++ .../pages/admin-settings/BrandingSettings.tsx | 316 ---- .../CompatibilityProxiesSettings.test.tsx | 82 +- .../CompatibilityProxiesSettings.tsx | 440 +++-- .../admin-settings/DatabaseSettings.test.tsx | 81 - .../pages/admin-settings/DatabaseSettings.tsx | 165 -- .../pages/admin-settings/DownloadSettings.tsx | 119 -- .../pages/admin-settings/EmailSettings.tsx | 182 -- .../admin-settings/GeneralSettings.test.tsx | 101 ++ .../pages/admin-settings/GeneralSettings.tsx | 131 +- .../InfrastructureSettings.test.tsx | 197 +++ .../admin-settings/InfrastructureSettings.tsx | 858 ++++++++++ .../IntegrationsSettings.test.tsx | 309 ++++ .../admin-settings/IntegrationsSettings.tsx | 1502 +++++++++++++++-- .../pages/admin-settings/InviteCodesTab.tsx | 47 +- .../LibraryMetadataSettings.test.tsx | 133 ++ .../LibraryMetadataSettings.tsx | 311 ++++ .../admin-settings/LogRetentionSettings.tsx | 414 ----- ...ettings.tsx => MarkerProviderSettings.tsx} | 107 +- .../NotificationsAdminSettings.test.tsx | 155 +- .../NotificationsAdminSettings.tsx | 700 ++++---- .../pages/admin-settings/OverlaySettings.tsx | 214 --- .../admin-settings/PlaybackSettings.test.tsx | 139 +- .../pages/admin-settings/PlaybackSettings.tsx | 494 ++++-- .../admin-settings/RateLimitSettings.tsx | 438 ----- .../pages/admin-settings/ScannerSettings.tsx | 87 - .../pages/admin-settings/SearchSettings.tsx | 378 ----- .../admin-settings/SearchStatusPanel.tsx | 160 ++ .../SecurityAccessSettings.test.tsx | 133 ++ .../admin-settings/SecurityAccessSettings.tsx | 522 ++++++ web/src/pages/admin-settings/SettingField.tsx | 43 +- .../admin-settings/StorageSettings.test.tsx | 230 --- .../pages/admin-settings/StorageSettings.tsx | 546 ------ .../admin-settings/SubtitlesSettings.tsx | 340 ---- .../admin-settings/ThemeSettings.test.tsx | 103 -- .../pages/admin-settings/ThemeSettings.tsx | 198 --- .../admin-settings/WatchProvidersSettings.tsx | 166 -- .../admin-settings/logRetentionPolicy.test.ts | 58 + .../admin-settings/logRetentionPolicy.ts | 86 + 78 files changed, 8804 insertions(+), 6850 deletions(-) create mode 100644 docs/architecture/admin-settings-ux.md create mode 100644 internal/api/handlers/admin_restart_keys_test.go create mode 100644 web/src/components/settings/AdvancedSection.test.tsx create mode 100644 web/src/components/settings/AdvancedSection.tsx create mode 100644 web/src/components/settings/LimitField.test.tsx create mode 100644 web/src/components/settings/LimitField.tsx create mode 100644 web/src/components/settings/ProviderCard.tsx create mode 100644 web/src/components/settings/RestartBadge.tsx create mode 100644 web/src/components/settings/SecretField.test.tsx create mode 100644 web/src/components/settings/SecretField.tsx create mode 100644 web/src/hooks/useRestartKeys.test.ts create mode 100644 web/src/hooks/useRestartKeys.ts delete mode 100644 web/src/pages/admin-settings/AIServicesSettings.test.tsx delete mode 100644 web/src/pages/admin-settings/AIServicesSettings.tsx create mode 100644 web/src/pages/admin-settings/AppearanceSettings.test.tsx create mode 100644 web/src/pages/admin-settings/AppearanceSettings.tsx delete mode 100644 web/src/pages/admin-settings/BrandingSettings.tsx delete mode 100644 web/src/pages/admin-settings/DatabaseSettings.test.tsx delete mode 100644 web/src/pages/admin-settings/DatabaseSettings.tsx delete mode 100644 web/src/pages/admin-settings/DownloadSettings.tsx delete mode 100644 web/src/pages/admin-settings/EmailSettings.tsx create mode 100644 web/src/pages/admin-settings/GeneralSettings.test.tsx create mode 100644 web/src/pages/admin-settings/InfrastructureSettings.test.tsx create mode 100644 web/src/pages/admin-settings/InfrastructureSettings.tsx create mode 100644 web/src/pages/admin-settings/IntegrationsSettings.test.tsx create mode 100644 web/src/pages/admin-settings/LibraryMetadataSettings.test.tsx create mode 100644 web/src/pages/admin-settings/LibraryMetadataSettings.tsx delete mode 100644 web/src/pages/admin-settings/LogRetentionSettings.tsx rename web/src/pages/admin-settings/{IntroSettings.tsx => MarkerProviderSettings.tsx} (81%) delete mode 100644 web/src/pages/admin-settings/OverlaySettings.tsx delete mode 100644 web/src/pages/admin-settings/RateLimitSettings.tsx delete mode 100644 web/src/pages/admin-settings/ScannerSettings.tsx delete mode 100644 web/src/pages/admin-settings/SearchSettings.tsx create mode 100644 web/src/pages/admin-settings/SearchStatusPanel.tsx create mode 100644 web/src/pages/admin-settings/SecurityAccessSettings.test.tsx create mode 100644 web/src/pages/admin-settings/SecurityAccessSettings.tsx delete mode 100644 web/src/pages/admin-settings/StorageSettings.test.tsx delete mode 100644 web/src/pages/admin-settings/StorageSettings.tsx delete mode 100644 web/src/pages/admin-settings/SubtitlesSettings.tsx delete mode 100644 web/src/pages/admin-settings/ThemeSettings.test.tsx delete mode 100644 web/src/pages/admin-settings/ThemeSettings.tsx delete mode 100644 web/src/pages/admin-settings/WatchProvidersSettings.tsx diff --git a/docs/architecture/admin-settings-ux.md b/docs/architecture/admin-settings-ux.md new file mode 100644 index 000000000..cf377a0aa --- /dev/null +++ b/docs/architecture/admin-settings-ux.md @@ -0,0 +1,76 @@ +# Admin settings UX + +Admin settings are organized by admin intent ("I want subtitles to download +automatically"), not by subsystem, and collapse into 9 tabs: General, +Appearance, Security & Access, Library & Metadata, Playback, Integrations, +Notifications, Compatibility, and Infrastructure. Settings promoted to its own +sidebar group, with the 9 tabs listed inline; old `?tab=` ids from the +previous 20-tab layout redirect to their new tab rather than 404ing. `⌘K` +(`AdminSectionCommandDialog`) is mounted in `AdminLayout` so search works from +every admin page, not just the Dashboard. + +## Three tiers, and how to pick one for a new setting + +Every admin setting is one of: + +- **Essential** — shown by default, no disclosure needed. Target at most ~8 + essential controls per tab above the fold. A setting is Essential only if a + household admin on a single-node install would plausibly need it without + being told to look for it (on/off toggles for a whole feature, the handful + of values that make the feature usable at all). +- **Advanced** — correct but not essential; collapsed by default behind one + `AdvancedSection` disclosure per tab (or per `FieldGroup` on a dense tab). + Open state persists in `localStorage` and auto-expands when a search match + or a dirty/invalid field lives inside it. Tuning knobs, alternate backends, + and anything whose default is good enough that most admins never touch it + belong here. +- **Hidden** — no UI at all, on any tab. The setting is still a normal + `server_settings` row: readable and writable through the admin settings API + and environment configuration exactly as before this reorganization. Use + Hidden for legacy key families kept for compatibility, settings that only + make sense with expert knowledge of the codebase, or values better derived + automatically (e.g. from node pool capacity) than hand-set. + +The tier is a UI-only decision. It must never change a key's validation, +default resolution, or API visibility — moving a setting to Hidden is +reversible by adding UI back, not by a data migration. + +## Shared primitives + +Reuse these instead of adding a bespoke variant per tab: + +- `SettingField` / `FieldGroup` / `SaveBar` (`web/src/pages/admin-settings/`) + and `useSettingsForm` (`web/src/hooks/`) — the one save model. Every tab + batches edits and commits them through one `SaveBar` with Discard; provider + credential cards are the only exception, and only because they need + Test-before-commit, which is itself one shared card component rather than a + bespoke one per provider. +- `AdvancedSection` — the one collapsible-disclosure primitive for the + Advanced tier. Do not add another `
`, another bespoke collapsible + component, or a per-page expand/collapse toggle. +- `SecretField` — the one "configured · Replace / Keep" credential control. +- `LimitField` — the one "Unlimited" checkbox pattern, replacing "0 = unlimited" + hint text conventions. +- A restart badge on `SettingField` itself, sourced from + `config.RestartRequired` (`internal/config/restart_keys.go`), not hand-copied + into hint text or inferred by a page-local heuristic. `RestartRequired` is + the single source of truth for which keys need a process restart to take + effect; a new field's badge must read that function (directly, or via a + manifest/meta endpoint built on top of it) rather than duplicating its + judgment. + +## Deferred out of this reorganization + +These were identified during the review but deliberately left for later work, +not folded into this pass: + +- Key renames (e.g. un-namespaced `allow_4k_transcode`, + `enable_transcode_throttle`, `transcode_throttle_seconds` moving under + `playback.*`). +- Introducing `server.public_url` as one canonical public URL that + `jellyfin_compat.public_url` and friends would derive from. +- Deleting the legacy `s3.operational_*` rows that a past migration copied and + never removed. + +Each still applies via its existing key and behavior; only the UI +reorganization and tiering in this document shipped now. diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 1a650092a..6e83dc172 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -1628,6 +1628,22 @@ func (h *AdminHandler) HandleGetEffectiveSettings(w http.ResponseWriter, r *http writeJSON(w, http.StatusOK, effective) } +type restartKeysResponse struct { + Keys []string `json:"keys"` + Prefixes []string `json:"prefixes"` +} + +// HandleGetRestartKeys handles GET /admin/settings/restart-keys. The registry +// is compiled into the binary (internal/config), so the response only changes +// across deploys; the admin UI caches it and uses it to badge the fields whose +// saved value waits on a restart. +func (h *AdminHandler) HandleGetRestartKeys(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, restartKeysResponse{ + Keys: config.RestartRequiredKeys(), + Prefixes: config.RestartRequiredPrefixes(), + }) +} + type sensitiveStatusResponse struct { Configured []string `json:"configured"` ManagedByEnv []string `json:"managed_by_env,omitempty"` @@ -1684,8 +1700,10 @@ func (h *AdminHandler) HandleGetSensitiveStatus(w http.ResponseWriter, r *http.R type adminSettingResponse struct { Key string `json:"key"` Value string `json:"value"` - // RestartRequired reports whether the saved value only takes effect - // after a server restart (set on update responses only). + // RestartRequired reports whether the value only takes effect after a + // server restart. It is populated from the compiled restart-key registry + // on read responses as well as on updates, so the admin UI never has to + // hand-copy the list into hint text. RestartRequired bool `json:"restart_required,omitempty"` } @@ -2205,7 +2223,11 @@ func (h *AdminHandler) HandleGetSetting(w http.ResponseWriter, r *http.Request) } if value, ok := h.BootstrapSensitiveValues[key]; ok && value != "" { - writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: value}) + writeJSON(w, http.StatusOK, adminSettingResponse{ + Key: key, + Value: value, + RestartRequired: config.RestartRequired(key), + }) return } @@ -2219,7 +2241,11 @@ func (h *AdminHandler) HandleGetSetting(w http.ResponseWriter, r *http.Request) return } - writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: value}) + writeJSON(w, http.StatusOK, adminSettingResponse{ + Key: key, + Value: value, + RestartRequired: config.RestartRequired(key), + }) } type updateSettingRequest struct { diff --git a/internal/api/handlers/admin_restart_keys_test.go b/internal/api/handlers/admin_restart_keys_test.go new file mode 100644 index 000000000..472fbdd30 --- /dev/null +++ b/internal/api/handlers/admin_restart_keys_test.go @@ -0,0 +1,61 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "sort" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" +) + +// The admin UI badges restart-required fields from this endpoint, so it has to +// report the compiled registry verbatim — both exact keys and whole namespaces. +func TestHandleGetRestartKeys(t *testing.T) { + handler := &AdminHandler{} + rec := httptest.NewRecorder() + + handler.HandleGetRestartKeys(rec, httptest.NewRequest(http.MethodGet, "/admin/settings/restart-keys", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + var body struct { + Keys []string `json:"keys"` + Prefixes []string `json:"prefixes"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + + if !slices.Equal(body.Keys, config.RestartRequiredKeys()) { + t.Errorf("keys = %v, want %v", body.Keys, config.RestartRequiredKeys()) + } + if !slices.Equal(body.Prefixes, config.RestartRequiredPrefixes()) { + t.Errorf("prefixes = %v, want %v", body.Prefixes, config.RestartRequiredPrefixes()) + } + if !sort.StringsAreSorted(body.Keys) { + t.Errorf("keys are not sorted: %v", body.Keys) + } + + // Spot-check both matching modes so a registry refactor that drops one of + // them fails here rather than silently un-badging the UI. + if !slices.Contains(body.Keys, "auth.jwt_secret") { + t.Errorf("keys missing auth.jwt_secret: %v", body.Keys) + } + if !slices.Contains(body.Prefixes, "database.") { + t.Errorf("prefixes missing database.: %v", body.Prefixes) + } + for _, key := range body.Keys { + if !config.RestartRequired(key) { + t.Errorf("reported key %q is not restart-required", key) + } + } + for _, prefix := range body.Prefixes { + if !config.RestartRequired(prefix + "example") { + t.Errorf("reported prefix %q does not mark its namespace restart-required", prefix) + } + } +} diff --git a/internal/api/handlers/admin_settings_checks_test.go b/internal/api/handlers/admin_settings_checks_test.go index 9bf471472..2772e80c4 100644 --- a/internal/api/handlers/admin_settings_checks_test.go +++ b/internal/api/handlers/admin_settings_checks_test.go @@ -1741,3 +1741,80 @@ func withChiParam(r *http.Request, key, value string) *http.Request { routeCtx.URLParams.Add(key, value) return r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, routeCtx)) } + +func TestAdminGetSettingReportsRestartRequired(t *testing.T) { + const restartKey = "scanner.max_concurrent_libraries" + const liveKey = "server.log_level" + + handler := &AdminHandler{ + SettingsRepo: &fakeServerSettingsStore{values: map[string]string{ + restartKey: "4", + liveKey: "debug", + }}, + BootstrapSensitiveValues: map[string]string{ + "playback.ffmpeg_path": "/opt/ffmpeg", + }, + } + + for _, tc := range []struct { + name string + key string + wantValue string + wantRestart bool + wantRestartSeen bool + }{ + { + name: "stored restart-required key", + key: restartKey, + wantValue: "4", + wantRestart: true, + wantRestartSeen: true, + }, + { + name: "stored hot-reloading key omits the flag", + key: liveKey, + wantValue: "debug", + }, + { + name: "bootstrap value", + key: "playback.ffmpeg_path", + wantValue: "/opt/ffmpeg", + wantRestart: true, + wantRestartSeen: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if config.RestartRequired(tc.key) != tc.wantRestart { + t.Fatalf("test fixture drifted: config.RestartRequired(%q) = %v", tc.key, !tc.wantRestart) + } + + req := httptest.NewRequest(http.MethodGet, "/admin/settings/"+tc.key, nil) + req = withChiParam(req, "key", tc.key) + rec := httptest.NewRecorder() + + handler.HandleGetSetting(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var payload struct { + Key string `json:"key"` + Value string `json:"value"` + RestartRequired bool `json:"restart_required"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Key != tc.key || payload.Value != tc.wantValue { + t.Fatalf("payload = %+v, want key %q value %q", payload, tc.key, tc.wantValue) + } + if payload.RestartRequired != tc.wantRestart { + t.Fatalf("restart_required = %v, want %v", payload.RestartRequired, tc.wantRestart) + } + // omitempty must keep the flag off the wire for live keys. + if seen := strings.Contains(rec.Body.String(), "restart_required"); seen != tc.wantRestartSeen { + t.Fatalf("restart_required present = %v, want %v; body=%s", seen, tc.wantRestartSeen, rec.Body.String()) + } + }) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 243ef26e5..47aefee08 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2939,6 +2939,7 @@ func NewRouter(deps Dependencies) chi.Router { r.Post("/jellyfin-compat/web/update", adminHandler.HandleUpdateJellyfinCompatWeb) r.Post("/jellyfin-compat/web/remove", adminHandler.HandleRemoveJellyfinCompatWeb) r.Get("/settings/sensitive-status", adminHandler.HandleGetSensitiveStatus) + r.Get("/settings/restart-keys", adminHandler.HandleGetRestartKeys) r.Post("/settings/check/{kind}", adminHandler.HandleCheckSettingsConnection) if sectionSettingsHandler != nil { r.Get("/settings/sections", sectionSettingsHandler.HandleGet) diff --git a/internal/config/admin_settings.go b/internal/config/admin_settings.go index 5798ac0c3..6092211f2 100644 --- a/internal/config/admin_settings.go +++ b/internal/config/admin_settings.go @@ -53,23 +53,30 @@ var adminSettingDefaults = map[string]string{ "clientip.trusted_proxies": "10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 127.0.0.0/8, ::1/128", "theme.catalog_url": DefaultThemeCatalogURL, - "database.max_connections": "20", - "s3.public_path_style": "true", - "s3.public_url_auth": "presigned", - "s3.public_token_param": "verify", - "s3.public_token_ttl": "10800", - "s3.private_path_style": "true", - "s3.user_db_path_style": "true", - "userdb.backend": "postgres", - "userdb.pool_max_open": "500", - "userdb.idle_timeout": "12h", - - "scanner.workers": "8", - "matcher.workers": "8", - "matcher.batch_size": "500", - "metadata.cache_images": "false", - "markers.mode": "local", - "markers.lazy_playback": "false", + "database.max_connections": "20", + "s3.public_path_style": "true", + "s3.public_url_auth": "presigned", + "s3.public_token_param": "verify", + "s3.public_token_ttl": "10800", + "s3.private_path_style": "true", + "s3.metadata_presign_expiry": "4h", + "s3.user_db_path_style": "true", + "userdb.backend": "postgres", + "userdb.pool_max_open": "500", + "userdb.idle_timeout": "12h", + + "scanner.workers": "8", + "scanner.max_concurrent_libraries": "1", + "scanner.max_concurrent_scoped": "2", + "scanner.file_removal_grace": "24h", + "scanner.empty_trash_after_scan": "true", + "matcher.workers": "8", + "matcher.batch_size": "500", + "matcher.enable_tv_series_root_queue": "true", + "matcher.enable_tv_series_group_queue": "false", + "metadata.cache_images": "false", + "markers.mode": "local", + "markers.lazy_playback": "false", "playback.ffmpeg_path": "", playbackTranscodeDirSettingKey: DefaultTranscodeDir, @@ -101,6 +108,8 @@ var adminSettingDefaults = map[string]string{ "jellyfin_compat.playback_session_ttl": "6h", "recommendations.enabled": "false", + "recommendations.embedding_provider": "ollama", + "recommendations.embeddings_job_timeout": "24h", "recommendations.embedding_base_url": "http://ollama:11434", "recommendations.embedding_model": "all-minilm", "recommendations.embeddings_cron": "0 3 * * *", @@ -119,6 +128,7 @@ var adminSettingDefaults = map[string]string{ "subtitle_ai.batch_size": "40", "subtitle_ai.context_neighbors": "2", "subtitle_ai.asr_chunk_seconds": "600", + "subtitle_ai.live_asr_chunk_seconds": "30", "subtitle_ai.transcribe_quota_jobs": "0", "subtitle_ai.transcribe_quota_period": "day", "metadata_ai.enabled": "false", @@ -134,6 +144,8 @@ var adminSettingDefaults = map[string]string{ "download.max_concurrent_prepares": "2", "download.artifact_max_bytes": "0", + "policy.editor_enabled": "false", + "policy.eval_timeout_ms": "25", "policy.decision_log_verbosity": "digest", "policy.decision_log_scope_sample_rate": "50", "policy.decision_log_retention_days": "14", @@ -173,6 +185,7 @@ var adminSettingDefaults = map[string]string{ "taskmanager.history_retention_days": "30", "taskmanager.history_keep_per_task": "1000", + "opslog.capture_level": "info", "opslog.retention_days": "7", "opslog.cleanup_interval_minutes": "15", "opslog.max_rows": "1000000", @@ -302,6 +315,8 @@ func NormalizeAdminSetting(key, raw string) (string, error) { "jellyfin_compat.enabled", "jellyfin_compat.web_enabled", "recommendations.enabled", "subtitle_ai.enabled", "subtitle_ai.transcribe_enabled", "metadata_ai.enabled", "download.enabled", "download.transcode_enabled", "email.enabled", "signup.enabled", + "scanner.empty_trash_after_scan", "matcher.enable_tv_series_root_queue", + "matcher.enable_tv_series_group_queue", "policy.editor_enabled", "overlays.enabled", "notifications.release_events_enabled", "notifications.fanout_enabled", "notifications.ui_enabled", "notifications.webhooks_enabled", "notifications.webhooks.allow_private_destinations", "notifications.email_enabled", @@ -317,7 +332,8 @@ func NormalizeAdminSetting(key, raw string) (string, error) { return normalizeAdminInt(key, value, 1, 10000) case "userdb.pool_max_open": return normalizeAdminInt(key, value, 1, 100000) - case "scanner.workers", "matcher.workers": + case "scanner.workers", "matcher.workers", + "scanner.max_concurrent_libraries", "scanner.max_concurrent_scoped": return normalizeAdminInt(key, value, 1, 1024) case "matcher.batch_size": return normalizeAdminInt(key, value, 1, 100000) @@ -337,6 +353,10 @@ func NormalizeAdminSetting(key, raw string) (string, error) { return normalizeAdminInt(key, value, 0, 100) case "subtitle_ai.asr_chunk_seconds": return normalizeAdminInt(key, value, 60, 600) + case "subtitle_ai.live_asr_chunk_seconds": + // 15s is the transcriber's hard floor (clampASRChunkSeconds); accepting + // less would store a value the runtime silently raises. + return normalizeAdminInt(key, value, 15, 600) case "subtitle_ai.transcribe_quota_jobs": return normalizeAdminInt(key, value, 0, math.MaxInt32) case "download.server_bandwidth_mbps", "download.user_bandwidth_mbps": @@ -346,6 +366,8 @@ func NormalizeAdminSetting(key, raw string) (string, error) { return normalizeAdminInt64(key, value, 0, math.MaxInt64) case "policy.decision_log_scope_sample_rate", "policy.decision_log_retention_days": return normalizeAdminInt(key, value, 1, math.MaxInt32) + case "policy.eval_timeout_ms": + return normalizeAdminInt(key, value, 1, 60000) case "email.smtp_port": return normalizeAdminInt(key, value, 1, 65535) case "notifications.fanout.settle_seconds": @@ -391,11 +413,24 @@ func NormalizeAdminSetting(key, raw string) (string, error) { case "auth.access_token_expiry", "auth.refresh_token_expiry", "userdb.idle_timeout", "download.period_duration", "jellyfin_compat.session_ttl", - "jellyfin_compat.playback_session_ttl": + "jellyfin_compat.playback_session_ttl", "s3.metadata_presign_expiry", + "recommendations.embeddings_job_timeout": return normalizeAdminDuration(key, value) + case "scanner.file_removal_grace": + // The scanner deliberately tolerates a zero or negative grace as + // "remove missing files immediately" (LoadFromDB warns and clamps to + // zero), so only require a parseable duration here. + if _, err := parseDuration(value); err != nil { + return "", fmt.Errorf("%s must be a duration", key) + } + return value, nil case "server.log_level": return normalizeAdminEnum(key, value, "debug", "info", "warn", "error") + case "opslog.capture_level": + // "warning" is accepted because the startup reader in cmd/silo treats + // it as an alias for "warn". + return normalizeAdminEnum(key, value, "debug", "info", "warn", "warning", "error") case "userdb.backend": return normalizeAdminEnum(key, value, "postgres", "sqlite") case "playback.hw_accel": diff --git a/internal/config/admin_settings_test.go b/internal/config/admin_settings_test.go index 9cb65ffb2..404294cfe 100644 --- a/internal/config/admin_settings_test.go +++ b/internal/config/admin_settings_test.go @@ -241,6 +241,18 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) { {key: "theme.catalog_url", value: "http://raw.githubusercontent.com/Silo-Server/silo-themes/main/catalog.json"}, {key: "theme.catalog_url", value: "https://example.com/catalog.json"}, {key: "redis.url", value: "not-a-url"}, + {key: "scanner.max_concurrent_libraries", value: "0"}, + {key: "scanner.max_concurrent_scoped", value: "-1"}, + {key: "scanner.empty_trash_after_scan", value: "sometimes"}, + {key: "scanner.file_removal_grace", value: "a while"}, + {key: "matcher.enable_tv_series_root_queue", value: "yes please"}, + {key: "matcher.enable_tv_series_group_queue", value: "yes please"}, + {key: "policy.editor_enabled", value: "maybe"}, + {key: "policy.eval_timeout_ms", value: "0"}, + {key: "subtitle_ai.live_asr_chunk_seconds", value: "0"}, + {key: "opslog.capture_level", value: "chatty"}, + {key: "s3.metadata_presign_expiry", value: "0s"}, + {key: "recommendations.embeddings_job_timeout", value: "soon"}, } for _, tc := range tests { t.Run(tc.key, func(t *testing.T) { @@ -324,3 +336,59 @@ func TestNormalizeAdminSettingCanonicalizesRedisURL(t *testing.T) { t.Fatalf("normalized Redis URL = %q", got) } } + +// TestNormalizeAdminSettingKeepsPermissiveScannerGrace locks the loader's +// documented behavior: a zero or negative grace means "remove missing files +// immediately", so the admin API must not reject it. +func TestNormalizeAdminSettingKeepsPermissiveScannerGrace(t *testing.T) { + for _, value := range []string{"0s", "-1h", "72h"} { + got, err := NormalizeAdminSetting("scanner.file_removal_grace", " "+value+" ") + if err != nil { + t.Fatalf("NormalizeAdminSetting(scanner.file_removal_grace, %q): %v", value, err) + } + if got != value { + t.Fatalf("normalized grace = %q, want %q", got, value) + } + } +} + +// TestOpslogCaptureLevelAcceptsWarningAlias mirrors the startup reader in +// cmd/silo, which treats "warning" as "warn". +func TestOpslogCaptureLevelAcceptsWarningAlias(t *testing.T) { + got, err := NormalizeAdminSetting("opslog.capture_level", "WARNING") + if err != nil { + t.Fatal(err) + } + if got != "warning" { + t.Fatalf("normalized capture level = %q, want warning", got) + } +} + +// TestHiddenTierDefaultsAreExposed guards the keys that have no admin UI: the +// API must still report the value the server is actually running. +func TestHiddenTierDefaultsAreExposed(t *testing.T) { + effective := EffectiveAdminSettings(nil) + want := map[string]string{ + "recommendations.embedding_provider": "ollama", + "recommendations.embeddings_job_timeout": "24h", + "policy.editor_enabled": "false", + "policy.eval_timeout_ms": "25", + "subtitle_ai.live_asr_chunk_seconds": "30", + "scanner.max_concurrent_libraries": "1", + "scanner.max_concurrent_scoped": "2", + "scanner.file_removal_grace": "24h", + "scanner.empty_trash_after_scan": "true", + "matcher.enable_tv_series_root_queue": "true", + "matcher.enable_tv_series_group_queue": "false", + "opslog.capture_level": "info", + "s3.metadata_presign_expiry": "4h", + } + for key, value := range want { + if got := effective[key]; got != value { + t.Errorf("effective[%q] = %q, want %q", key, got, value) + } + if _, err := NormalizeAdminSetting(key, value); err != nil { + t.Errorf("default for %q is rejected by NormalizeAdminSetting: %v", key, err) + } + } +} diff --git a/internal/config/restart_keys.go b/internal/config/restart_keys.go index 27d6267ae..70dd77793 100644 --- a/internal/config/restart_keys.go +++ b/internal/config/restart_keys.go @@ -1,6 +1,9 @@ package config -import "strings" +import ( + "sort" + "strings" +) // restartRequiredKeys lists server_settings keys whose values are captured at // process startup (listeners, connection pools, HTTP clients, worker pools) @@ -118,3 +121,28 @@ func RestartRequired(key string) bool { } return false } + +// RestartRequiredKeys returns the sorted list of exact server_settings keys +// that require a restart. The admin UI reads this over +// GET /admin/settings/restart-keys to render its restart badges, so it must +// stay a copy: callers must not be able to mutate the registry. +func RestartRequiredKeys() []string { + keys := make([]string, 0, len(restartRequiredKeys)) + for key, required := range restartRequiredKeys { + if required { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +// RestartRequiredPrefixes returns the sorted list of key prefixes whose whole +// namespace requires a restart. Pair it with RestartRequiredKeys: a key needs a +// restart when it is listed exactly or carries one of these prefixes. +func RestartRequiredPrefixes() []string { + prefixes := make([]string, len(restartRequiredPrefixes)) + copy(prefixes, restartRequiredPrefixes) + sort.Strings(prefixes) + return prefixes +} diff --git a/internal/config/restart_keys_test.go b/internal/config/restart_keys_test.go index a81b773ad..4924bca4f 100644 --- a/internal/config/restart_keys_test.go +++ b/internal/config/restart_keys_test.go @@ -58,3 +58,36 @@ func TestRestartRequired(t *testing.T) { } } } + +// The exported accessors feed the admin API, so they must agree with +// RestartRequired, stay sorted, and hand out copies rather than the registry. +func TestRestartRequiredAccessors(t *testing.T) { + keys := RestartRequiredKeys() + if len(keys) != len(restartRequiredKeys) { + t.Fatalf("RestartRequiredKeys() returned %d keys, want %d", len(keys), len(restartRequiredKeys)) + } + for i, key := range keys { + if i > 0 && keys[i-1] >= key { + t.Fatalf("RestartRequiredKeys() is not sorted at %d: %v", i, keys) + } + if !RestartRequired(key) { + t.Errorf("RestartRequiredKeys() reported %q, which RestartRequired rejects", key) + } + } + + prefixes := RestartRequiredPrefixes() + if len(prefixes) != len(restartRequiredPrefixes) { + t.Fatalf("RestartRequiredPrefixes() returned %d prefixes, want %d", len(prefixes), len(restartRequiredPrefixes)) + } + for _, prefix := range prefixes { + if !RestartRequired(prefix + "anything") { + t.Errorf("RestartRequiredPrefixes() reported %q, which RestartRequired rejects", prefix) + } + } + + // Mutating a returned slice must not corrupt the registry. + prefixes[0] = "mutated." + if RestartRequiredPrefixes()[0] == "mutated." { + t.Error("RestartRequiredPrefixes() exposes the package-level slice") + } +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 79ea7923c..08eb207d8 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -77,7 +77,6 @@ const AdminDiagnostics = lazy(() => import("@/pages/AdminDiagnostics")); const AdminAccessGroups = lazy(() => import("@/pages/AdminAccessGroups")); const AdminUsers = lazy(() => import("@/pages/AdminUsers")); const AdminRequests = lazy(() => import("@/pages/AdminRequests")); -const AdminAutoscan = lazy(() => import("@/pages/AdminAutoscan")); const AdminDevices = lazy(() => import("@/pages/AdminDevices")); const AdminLibraries = lazy(() => import("@/pages/AdminLibraries")); const AdminSettingsLayout = lazy(() => import("@/pages/admin-settings/AdminSettingsLayout")); @@ -457,7 +456,11 @@ function AppRoutes() { } /> } /> } /> - } /> + {/* Autoscan is a tab on Libraries now; keep old links working. */} + } + /> } /> } /> } /> diff --git a/web/src/components/AdminLayout.tsx b/web/src/components/AdminLayout.tsx index 9c749b815..25a5d2c09 100644 --- a/web/src/components/AdminLayout.tsx +++ b/web/src/components/AdminLayout.tsx @@ -1,6 +1,7 @@ -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Outlet, useLocation } from "react-router"; import AdminSidebar from "@/components/AdminSidebar"; +import { AdminSectionCommandDialog } from "@/components/AdminSectionCommandDialog"; import ServerActivity from "@/components/ServerActivity"; import { Sheet, @@ -11,8 +12,12 @@ import { SheetTrigger, } from "@/components/ui/sheet"; import { useDocumentTitle } from "@/hooks/useDocumentTitle"; +import { useAdminPluginInstallations } from "@/hooks/queries/admin/plugins"; +import { usePolicyCapability } from "@/hooks/queries/admin/policy"; +import { buildAdminCommandNavSections } from "@/lib/adminNavigation"; import { resolveAdminDocumentTitle } from "@/lib/documentTitle"; -import { Menu, X } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { Menu, Search, X } from "lucide-react"; import { useWatchPlaybackController } from "@/playback/watchPlaybackContext"; import { useAudiobookPlaybackController } from "@/pages/audiobooks/player/audiobookPlaybackContext"; @@ -20,7 +25,19 @@ const ADMIN_DESKTOP_MEDIA_QUERY = "(min-width: 64rem)"; export default function AdminLayout() { const [mobileOpen, setMobileOpen] = useState(false); + const [commandOpen, setCommandOpen] = useState(false); const location = useLocation(); + const { data: adminInstallations } = useAdminPluginInstallations(); + const policyCapability = usePolicyCapability(); + // Mounted here rather than on the dashboard so Cmd+K reaches every admin + // page, which is what the pages that advertise the shortcut assume. + const adminSearchSections = useMemo( + () => + buildAdminCommandNavSections(adminInstallations, { + policyEditorAvailable: policyCapability.data?.editor_available === true, + }), + [adminInstallations, policyCapability.data?.editor_available], + ); const { isBackgroundBarVisible } = useWatchPlaybackController(); const audiobookPlayback = useAudiobookPlaybackController(); const hasBackgroundBar = isBackgroundBarVisible || audiobookPlayback?.isBackgroundBarVisible; @@ -44,6 +61,11 @@ export default function AdminLayout() { return (
+
- +
+ setCommandOpen(true)} className="h-11 w-11" /> + +
{/* Mobile sidebar drawer */} @@ -113,8 +138,9 @@ export default function AdminLayout() { - {/* Desktop activity indicator */} -
+ {/* Desktop header controls */} +
+ setCommandOpen(true)} showShortcut />
@@ -132,3 +158,36 @@ export default function AdminLayout() {
); } + +function AdminSearchButton({ + onClick, + className, + showShortcut = false, +}: { + onClick: () => void; + className?: string; + showShortcut?: boolean; +}) { + return ( + + ); +} diff --git a/web/src/components/AdminSectionCommandDialog.test.tsx b/web/src/components/AdminSectionCommandDialog.test.tsx index e18cc6735..33f86ea90 100644 --- a/web/src/components/AdminSectionCommandDialog.test.tsx +++ b/web/src/components/AdminSectionCommandDialog.test.tsx @@ -66,18 +66,20 @@ describe("AdminSectionCommandDialog", () => { expect(screen.queryByRole("option", { name: /Settings/ })).not.toBeInTheDocument(); }); - it("searches individual admin setting labels from the dashboard dialog", async () => { + it("searches individual admin setting labels from the admin dialog", async () => { renderDialog(); const searchBox = await openDialog(); - await userEvent.type(searchBox, "pool max open"); + await userEvent.type(searchBox, "maximum postgres connections"); - expect(screen.getByRole("option", { name: /Database/ })).toBeInTheDocument(); - expect(screen.getByText("Pool Max Open")).toBeInTheDocument(); + expect(screen.getByRole("option", { name: /Infrastructure/ })).toBeInTheDocument(); + expect(screen.getByText("Maximum Postgres connections")).toBeInTheDocument(); - await userEvent.click(screen.getByRole("option", { name: /Database/ })); + await userEvent.click(screen.getByRole("option", { name: /Infrastructure/ })); - expect(screen.getByLabelText("Current path")).toHaveTextContent("/admin/settings?tab=database"); + expect(screen.getByLabelText("Current path")).toHaveTextContent( + "/admin/settings?tab=infrastructure", + ); }); it("includes admin plugin app destinations", async () => { diff --git a/web/src/components/AdminSectionCommandDialog.tsx b/web/src/components/AdminSectionCommandDialog.tsx index a1cd5d8ad..95b8787f9 100644 --- a/web/src/components/AdminSectionCommandDialog.tsx +++ b/web/src/components/AdminSectionCommandDialog.tsx @@ -15,10 +15,26 @@ import { cn } from "@/lib/utils"; interface AdminSectionCommandDialogProps { sections: readonly AdminNavGroup[]; + /** Controlled open state, so a visible search button can open the palette. */ + open?: boolean; + onOpenChange?: (open: boolean) => void; } -export function AdminSectionCommandDialog({ sections }: AdminSectionCommandDialogProps) { - const [open, setOpen] = useState(false); +export function AdminSectionCommandDialog({ + sections, + open: openProp, + onOpenChange, +}: AdminSectionCommandDialogProps) { + const [uncontrolledOpen, setUncontrolledOpen] = useState(false); + const open = openProp ?? uncontrolledOpen; + const isControlled = openProp !== undefined; + const setOpen = useCallback( + (next: boolean) => { + if (!isControlled) setUncontrolledOpen(next); + onOpenChange?.(next); + }, + [isControlled, onOpenChange], + ); const [query, setQuery] = useState(""); const [selectedIndex, setSelectedIndex] = useState(0); const inputRef = useRef(null); @@ -56,12 +72,12 @@ export function AdminSectionCommandDialog({ sections }: AdminSectionCommandDialo setOpen(false); setQuery(""); setSelectedIndex(0); - }, []); + }, [setOpen]); const openDialog = useCallback(() => { setOpen(true); focusSearch(); - }, [focusSearch]); + }, [focusSearch, setOpen]); useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { diff --git a/web/src/components/AdminSidebar.tsx b/web/src/components/AdminSidebar.tsx index ff9450a89..4f6ad6127 100644 --- a/web/src/components/AdminSidebar.tsx +++ b/web/src/components/AdminSidebar.tsx @@ -77,8 +77,17 @@ export default function AdminSidebar({ onNavigate, embedded = false }: AdminSide } function isActive(item: SidebarItem) { - if (item.exact) return location.pathname === item.href; - return location.pathname === item.href || location.pathname.startsWith(`${item.href}/`); + // Settings tabs are one path with a `?tab=` discriminator, so a plain + // pathname compare would light up all nine (or none) at once. + const [itemPath, itemQuery] = item.href.split("?"); + if (itemQuery) { + if (location.pathname !== itemPath) return false; + const wanted = new URLSearchParams(itemQuery); + const current = new URLSearchParams(location.search); + return [...wanted.entries()].every(([key, value]) => current.get(key) === value); + } + if (item.exact) return location.pathname === itemPath; + return location.pathname === itemPath || location.pathname.startsWith(`${itemPath}/`); } return ( diff --git a/web/src/components/settings/AdvancedSection.test.tsx b/web/src/components/settings/AdvancedSection.test.tsx new file mode 100644 index 000000000..6e56e3fe1 --- /dev/null +++ b/web/src/components/settings/AdvancedSection.test.tsx @@ -0,0 +1,143 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { AdvancedSection } from "@/components/settings/AdvancedSection"; + +function renderSection(props: Partial[0]> = {}) { + return render( + +
ffmpeg path
+ , + ); +} + +describe("AdvancedSection", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("starts collapsed and labels the disclosure with the setting count", () => { + renderSection(); + + const toggle = screen.getByRole("button", { name: /Advanced · 3 settings/ }); + expect(toggle).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByText("ffmpeg path")).not.toBeInTheDocument(); + }); + + it("uses the singular form for a single setting", () => { + renderSection({ count: 1 }); + + expect(screen.getByRole("button", { name: /Advanced · 1 setting$/ })).toBeInTheDocument(); + }); + + it("persists the open state under the section id", async () => { + const user = userEvent.setup(); + const { unmount } = renderSection(); + + await user.click(screen.getByRole("button", { name: /Advanced/ })); + expect(screen.getByText("ffmpeg path")).toBeInTheDocument(); + expect(localStorage.getItem("silo.admin.advanced.playback.transcoding")).toBe("true"); + + unmount(); + renderSection(); + expect(screen.getByText("ffmpeg path")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /Advanced/ })); + expect(localStorage.getItem("silo.admin.advanced.playback.transcoding")).toBe("false"); + expect(screen.queryByText("ffmpeg path")).not.toBeInTheDocument(); + }); + + it("does not inherit another section's persisted state", () => { + localStorage.setItem("silo.admin.advanced.playback.transcoding", "true"); + renderSection({ id: "downloads" }); + + expect(screen.queryByText("ffmpeg path")).not.toBeInTheDocument(); + }); + + it("honours defaultOpen only until a choice is persisted", async () => { + const user = userEvent.setup(); + const { unmount } = renderSection({ defaultOpen: true }); + expect(screen.getByText("ffmpeg path")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /Advanced/ })); + unmount(); + + renderSection({ defaultOpen: true }); + expect(screen.queryByText("ffmpeg path")).not.toBeInTheDocument(); + }); + + it("opens automatically while forceOpen is set", () => { + const { rerender } = render( + +
bandwidth
+ , + ); + expect(screen.queryByText("bandwidth")).not.toBeInTheDocument(); + + rerender( + +
bandwidth
+ , + ); + expect(screen.getByText("bandwidth")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Advanced/ })).toHaveAttribute( + "aria-expanded", + "true", + ); + }); + + it("opens on the first render when forceOpen is already set", () => { + render( + +
bandwidth
+ , + ); + + expect(screen.getByText("bandwidth")).toBeInTheDocument(); + }); + + it("re-expands when a new reason to force it open arrives after a manual collapse", async () => { + const user = userEvent.setup(); + const { rerender } = render( + +
bandwidth
+ , + ); + + await user.click(screen.getByRole("button", { name: /Advanced/ })); + expect(screen.queryByText("bandwidth")).not.toBeInTheDocument(); + + // The reason clears (the field was saved), then a different field inside + // goes dirty. The stale manual collapse must not keep it hidden. + rerender( + +
bandwidth
+ , + ); + expect(screen.queryByText("bandwidth")).not.toBeInTheDocument(); + + rerender( + +
bandwidth
+ , + ); + expect(screen.getByText("bandwidth")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Advanced/ })).toHaveAttribute( + "aria-expanded", + "true", + ); + }); + + it("lets an auto-expanded section be collapsed again", async () => { + const user = userEvent.setup(); + render( + +
bandwidth
+ , + ); + + await user.click(screen.getByRole("button", { name: /Advanced/ })); + expect(screen.queryByText("bandwidth")).not.toBeInTheDocument(); + }); +}); diff --git a/web/src/components/settings/AdvancedSection.tsx b/web/src/components/settings/AdvancedSection.tsx new file mode 100644 index 000000000..96d77b628 --- /dev/null +++ b/web/src/components/settings/AdvancedSection.tsx @@ -0,0 +1,113 @@ +import { useState, type ReactNode } from "react"; +import { ChevronDown } from "lucide-react"; + +import { cn } from "@/lib/utils"; + +const STORAGE_PREFIX = "silo.admin.advanced."; + +function storageKey(id: string) { + return `${STORAGE_PREFIX}${id}`; +} + +function readPersisted(id: string): boolean | null { + try { + const raw = localStorage.getItem(storageKey(id)); + if (raw === "true") return true; + if (raw === "false") return false; + return null; + } catch { + return null; + } +} + +function writePersisted(id: string, open: boolean): void { + try { + localStorage.setItem(storageKey(id), open ? "true" : "false"); + } catch { + // Storage full or unavailable: the disclosure still works this session. + } +} + +export interface AdvancedSectionProps { + /** Stable id for the persisted open state, e.g. `playback.transcoding`. */ + id: string; + /** Number of settings inside, rendered as "Advanced · N settings". */ + count?: number; + title?: string; + /** Open state used when nothing is persisted yet. */ + defaultOpen?: boolean; + /** + * Forces the section open regardless of the persisted state — pass the + * section's dirty/invalid/search-match state so a hidden field can never be + * the reason a save bar refuses to save. + */ + forceOpen?: boolean; + children: ReactNode; +} + +/** + * The single disclosure primitive for advanced admin settings. Collapsed by + * default, remembers the admin's choice per section in localStorage, and + * auto-expands while `forceOpen` is set. + */ +export function AdvancedSection({ + id, + count, + title = "Advanced", + defaultOpen = false, + forceOpen = false, + children, +}: AdvancedSectionProps) { + // Persisted choice, read once: a section's id is fixed for the life of the + // instance (give the component a `key` if a caller ever swaps ids). + const [persistedOpen, setPersistedOpen] = useState(() => readPersisted(id) ?? defaultOpen); + // Explicit toggle this session, which also wins over `forceOpen` so an + // auto-expanded section can still be collapsed. + const [override, setOverride] = useState(null); + const [wasForcedOpen, setWasForcedOpen] = useState(forceOpen); + + // A manual collapse only outranks the *current* reason to force the section + // open. When a new one arrives (a field inside just went dirty or invalid, or + // a search started matching), drop the override so the save bar can never + // block on a field the admin cannot see. Adjusting state during render is + // cheaper than an effect: React re-renders before committing. + if (forceOpen !== wasForcedOpen) { + setWasForcedOpen(forceOpen); + if (forceOpen) setOverride(null); + } + + const open = override ?? (persistedOpen || forceOpen); + + function toggle() { + const next = !open; + setOverride(next); + setPersistedOpen(next); + writePersisted(id, next); + } + + const label = + typeof count === "number" ? `${title} · ${count} setting${count === 1 ? "" : "s"}` : title; + + return ( +
+ + {open ?
{children}
: null} +
+ ); +} diff --git a/web/src/components/settings/LimitField.test.tsx b/web/src/components/settings/LimitField.test.tsx new file mode 100644 index 000000000..41e5f6174 --- /dev/null +++ b/web/src/components/settings/LimitField.test.tsx @@ -0,0 +1,97 @@ +import { useState } from "react"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { LimitField } from "@/components/settings/LimitField"; + +function Harness({ + initial, + onChange, + unlimitedValue, +}: { + initial: string; + onChange?: (value: string) => void; + unlimitedValue?: string; +}) { + const [value, setValue] = useState(initial); + return ( + { + setValue(next); + onChange?.(next); + }} + /> + ); +} + +describe("LimitField", () => { + it("reads the sentinel as unlimited and hides it from the input", () => { + render(); + + expect(screen.getByRole("checkbox", { name: "Unlimited" })).toBeChecked(); + const input = screen.getByLabelText("Per-user bandwidth"); + expect(input).toBeDisabled(); + expect(input).toHaveValue(null); + }); + + it("writes the sentinel when Unlimited is checked", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + const checkbox = screen.getByRole("checkbox", { name: "Unlimited" }); + expect(checkbox).not.toBeChecked(); + + await user.click(checkbox); + expect(onChange).toHaveBeenLastCalledWith("0"); + expect(screen.getByLabelText("Per-user bandwidth")).toBeDisabled(); + }); + + it("restores the previous limit when Unlimited is unchecked", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + const checkbox = screen.getByRole("checkbox", { name: "Unlimited" }); + await user.click(checkbox); + await user.click(checkbox); + + expect(onChange).toHaveBeenLastCalledWith("50"); + expect(screen.getByLabelText("Per-user bandwidth")).toHaveValue(50); + }); + + it("falls back to an empty limit when unlimited was the saved value", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.click(screen.getByRole("checkbox", { name: "Unlimited" })); + expect(onChange).toHaveBeenLastCalledWith(""); + expect(screen.getByLabelText("Per-user bandwidth")).toBeEnabled(); + }); + + it("supports a non-zero unlimited sentinel", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + expect(screen.getByRole("checkbox", { name: "Unlimited" })).not.toBeChecked(); + await user.click(screen.getByRole("checkbox", { name: "Unlimited" })); + expect(onChange).toHaveBeenLastCalledWith("-1"); + expect(screen.getByRole("checkbox", { name: "Unlimited" })).toBeChecked(); + }); + + it("passes typed limits straight through", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.type(screen.getByLabelText("Per-user bandwidth"), "25"); + expect(onChange).toHaveBeenLastCalledWith("25"); + }); +}); diff --git a/web/src/components/settings/LimitField.tsx b/web/src/components/settings/LimitField.tsx new file mode 100644 index 000000000..77617af46 --- /dev/null +++ b/web/src/components/settings/LimitField.tsx @@ -0,0 +1,99 @@ +import { useId, useState } from "react"; + +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { RestartBadge } from "@/components/settings/RestartBadge"; + +export interface LimitFieldProps { + label: string; + /** Stored value; equal to `unlimitedValue` when the limit is off. */ + value: string; + onChange: (value: string) => void; + /** Sentinel the backend reads as "no limit". */ + unlimitedValue?: string; + /** Fallback used when a limit is re-enabled and nothing was typed before. */ + fallbackValue?: string; + unlimitedLabel?: string; + /** Rendered after the input, e.g. "Mbps". */ + unit?: string; + hint?: string; + min?: number; + disabled?: boolean; + restartRequired?: boolean; +} + +/** + * Number input paired with an "Unlimited" checkbox, replacing the + * "0 = unlimited" hint convention. The sentinel never reaches the admin's + * eyes, but the saved value is unchanged. + */ +export function LimitField({ + label, + value, + onChange, + unlimitedValue = "0", + fallbackValue = "", + unlimitedLabel = "Unlimited", + unit, + hint, + min = 0, + disabled = false, + restartRequired = false, +}: LimitFieldProps) { + const controlId = useId(); + const checkboxId = useId(); + const hintId = useId(); + const unlimited = value.trim() === unlimitedValue; + // Remembers the limit that Unlimited replaced so unchecking restores it + // instead of dumping the admin back onto an empty box. + const [lastLimit, setLastLimit] = useState(fallbackValue); + + function toggleUnlimited(checked: boolean) { + if (checked) { + setLastLimit(unlimited ? fallbackValue : value); + onChange(unlimitedValue); + return; + } + onChange(lastLimit.trim() === unlimitedValue ? fallbackValue : lastLimit); + } + + return ( +
+
+ + {restartRequired && } +
+
+ onChange(e.target.value)} + disabled={disabled || unlimited} + className="w-full sm:w-40" + aria-describedby={hint ? hintId : undefined} + /> + {unit && {unit}} + +
+ {hint && ( +

+ {hint} +

+ )} +
+ ); +} diff --git a/web/src/components/settings/ProviderCard.tsx b/web/src/components/settings/ProviderCard.tsx new file mode 100644 index 000000000..06e794fc0 --- /dev/null +++ b/web/src/components/settings/ProviderCard.tsx @@ -0,0 +1,248 @@ +import { useId, useState, type ReactNode } from "react"; +import type { LucideIcon } from "lucide-react"; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { Switch } from "@/components/ui/switch"; +import { RestartBadge } from "@/components/settings/RestartBadge"; +import { cn } from "@/lib/utils"; + +/** Outcome of a provider's Test action, whatever endpoint produced it. */ +export interface ProviderTestResult { + success: boolean; + message?: string; +} + +export type ProviderStatus = "connected" | "unconfigured" | "failing"; + +const STATUS_LABELS: Record = { + connected: "Connected", + unconfigured: "Not set up", + failing: "Failing", +}; + +const STATUS_CLASSES: Record = { + connected: "border-green-500/30 text-green-600 dark:text-green-400", + unconfigured: "border-border text-muted-foreground", + failing: "border-amber-500/30 text-amber-600 dark:text-amber-400", +}; + +function StatusChip({ status, label }: { status: ProviderStatus; label?: string }) { + return ( + + + ); +} + +export interface ProviderCardProps { + title: string; + description?: ReactNode; + icon?: LucideIcon; + status: ProviderStatus; + /** Overrides the default chip text ("Connected" / "Not set up" / "Failing"). */ + statusLabel?: string; + /** Pass together with `onEnabledChange` to show the on/off switch. */ + enabled?: boolean; + onEnabledChange?: (enabled: boolean) => void; + /** Marks the whole card with a restart badge; drive it from `useRestartKeys`. */ + restartRequired?: boolean; + /** Disables every control while a request is in flight. */ + busy?: boolean; + /** Credential fields — use `SecretField` for anything the server stores. */ + children?: ReactNode; + onSave?: () => void; + saveLabel?: string; + isSaving?: boolean; + saveDisabled?: boolean; + onTest?: () => void; + testLabel?: string; + testPendingLabel?: string; + isTesting?: boolean; + testDisabled?: boolean; + testResult?: ProviderTestResult | null; + /** Shows a Clear button guarded by a confirmation dialog. */ + onClear?: () => void; + clearLabel?: string; + clearTitle?: string; + clearDescription?: string; + clearActionLabel?: string; + /** Rendered under the actions — notes, restart prompts, doc links. */ + footer?: ReactNode; +} + +/** + * The single card for a third-party integration: status, credentials, and the + * Save / Test / Clear trio. Credentials save per card because a provider has to + * be testable before its values are committed, so this card owns its own + * actions rather than feeding the page's save bar. + */ +export function ProviderCard({ + title, + description, + icon: Icon, + status, + statusLabel, + enabled, + onEnabledChange, + restartRequired = false, + busy = false, + children, + onSave, + saveLabel = "Save", + isSaving = false, + saveDisabled = false, + onTest, + testLabel = "Test", + testPendingLabel = "Testing...", + isTesting = false, + testDisabled = false, + testResult, + onClear, + clearLabel = "Clear credentials", + clearTitle, + clearDescription, + clearActionLabel = "Clear", + footer, +}: ProviderCardProps) { + const headingId = useId(); + const switchId = useId(); + const [confirmClear, setConfirmClear] = useState(false); + + return ( +
+
+
+ {Icon && ( +
+
+ )} +
+
+

+ {title} +

+ {restartRequired && } +
+ {description && ( +

{description}

+ )} +
+
+
+ + {onEnabledChange && ( + + )} +
+
+ + {children &&
{children}
} + + {(onSave || onTest || onClear) && ( +
+ {onTest && ( + + )} + {onSave && ( + + )} + {onClear && ( + + )} +
+ )} + + {testResult && ( +

+ {testResult.message ?? + (testResult.success ? "Connection successful." : "Connection failed.")} +

+ )} + + {footer} + + {onClear && ( + + + + {clearTitle ?? `Clear ${title} credentials?`} + + {clearDescription ?? + `Silo stops using ${title} until new credentials are saved here.`} + + + + Cancel + { + onClear(); + setConfirmClear(false); + }} + > + {clearActionLabel} + + + + + )} +
+ ); +} diff --git a/web/src/components/settings/RestartBadge.tsx b/web/src/components/settings/RestartBadge.tsx new file mode 100644 index 000000000..f7dfded01 --- /dev/null +++ b/web/src/components/settings/RestartBadge.tsx @@ -0,0 +1,27 @@ +import { RotateCw } from "lucide-react"; + +import { cn } from "@/lib/utils"; + +const RESTART_TITLE = "Takes effect after a server restart"; + +/** + * Small amber chip marking a setting whose value is only read at startup. + * Driven by the compiled restart-required key list (see `useRestartKeys`) so + * the fact never has to be hand-copied into a field hint. + */ +export function RestartBadge({ className }: { className?: string }) { + return ( + + + ); +} diff --git a/web/src/components/settings/SecretField.test.tsx b/web/src/components/settings/SecretField.test.tsx new file mode 100644 index 000000000..a12868396 --- /dev/null +++ b/web/src/components/settings/SecretField.test.tsx @@ -0,0 +1,107 @@ +import { useState } from "react"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { SecretField } from "@/components/settings/SecretField"; + +function Harness({ + configured, + onChange, + onKeep, +}: { + configured: boolean; + onChange?: (value: string) => void; + onKeep?: () => void; +}) { + const [value, setValue] = useState(""); + return ( + { + setValue(next); + onChange?.(next); + }} + onKeep={onKeep} + /> + ); +} + +describe("SecretField", () => { + it("shows a password input when nothing is saved", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + expect(screen.queryByText("Configured")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Keep saved Secret key" })).not.toBeInTheDocument(); + + const input = screen.getByLabelText("Secret key"); + expect(input).toHaveAttribute("type", "password"); + await user.type(input, "abc"); + expect(onChange).toHaveBeenLastCalledWith("abc"); + }); + + it("summarises a saved secret behind a Replace button", () => { + render(); + + expect(screen.getByText("Configured")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Replace Secret key" })).toBeInTheDocument(); + expect(screen.queryByLabelText("Secret key")).not.toBeInTheDocument(); + }); + + it("reveals an input with Keep saved value after Replace", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Replace Secret key" })); + + const input = screen.getByLabelText("Secret key"); + expect(input).toHaveAttribute("type", "password"); + expect(screen.getByText("Enter a replacement value.")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Keep saved Secret key" })); + expect(screen.getByText("Configured")).toBeInTheDocument(); + expect(screen.queryByLabelText("Secret key")).not.toBeInTheDocument(); + }); + + it("delegates the revert to onKeep so the parent's draft stays authoritative", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const onKeep = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: "Replace Secret key" })); + await user.type(screen.getByLabelText("Secret key"), "x"); + onChange.mockClear(); + + await user.click(screen.getByRole("button", { name: "Keep saved Secret key" })); + expect(onKeep).toHaveBeenCalledTimes(1); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("clears its own draft when no onKeep is supplied", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: "Replace Secret key" })); + await user.type(screen.getByLabelText("Secret key"), "x"); + onChange.mockClear(); + + await user.click(screen.getByRole("button", { name: "Keep saved Secret key" })); + expect(onChange).toHaveBeenCalledWith(""); + }); + + it("follows a controlled editing prop", () => { + const { rerender } = render( + , + ); + expect(screen.getByText("Configured")).toBeInTheDocument(); + + rerender(); + expect(screen.getByLabelText("Secret key")).toBeInTheDocument(); + }); +}); diff --git a/web/src/components/settings/SecretField.tsx b/web/src/components/settings/SecretField.tsx new file mode 100644 index 000000000..1c2f29e10 --- /dev/null +++ b/web/src/components/settings/SecretField.tsx @@ -0,0 +1,114 @@ +import { useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { RestartBadge } from "@/components/settings/RestartBadge"; +import { SettingField } from "@/pages/admin-settings/SettingField"; + +export interface SecretFieldProps { + label: string; + /** Staged plaintext value; always empty while the saved secret is kept. */ + value: string; + /** Whether the server already stores a value for this key. */ + configured: boolean; + onChange: (value: string) => void; + /** + * Controlled replacement state. Leave undefined to let the field track it + * itself; pass it when the parent has to reset several secrets at once. + */ + editing?: boolean; + /** Called when the admin starts replacing a saved secret. */ + onReplace?: () => void; + /** Called when the admin abandons the replacement (revert the staged value). */ + onKeep?: () => void; + hint?: string; + disabled?: boolean; + restartRequired?: boolean; +} + +/** + * The single credential control for admin settings. Three states: + * saved (summary + Replace), replacing (password input + Keep saved value), + * and unset (plain password input). + */ +export function SecretField({ + label, + value, + configured, + onChange, + editing, + onReplace, + onKeep, + hint, + disabled = false, + restartRequired = false, +}: SecretFieldProps) { + const [internalEditing, setInternalEditing] = useState(false); + const isEditing = editing ?? internalEditing; + + function beginReplace() { + if (disabled) return; + setInternalEditing(true); + onReplace?.(); + } + + function keepSaved() { + if (disabled) return; + setInternalEditing(false); + // A parent that stages values (useSettingsForm) reverts the draft itself; + // clearing through onChange there would leave the key marked dirty. + if (onKeep) onKeep(); + else onChange(""); + } + + if (configured && !isEditing) { + return ( +
+
+ + {restartRequired && } +
+
+ Configured + +
+ {hint &&

{hint}

} +
+ ); + } + + return ( +
+ + {configured && ( + + )} +
+ ); +} diff --git a/web/src/hooks/queries/admin/settings.ts b/web/src/hooks/queries/admin/settings.ts index 201389acf..408aa9190 100644 --- a/web/src/hooks/queries/admin/settings.ts +++ b/web/src/hooks/queries/admin/settings.ts @@ -100,6 +100,27 @@ export function useAdminServerSettings() { }); } +/** Shape of `GET /admin/settings/restart-keys`. */ +export interface RestartKeysResponse { + keys: string[]; + prefixes: string[]; +} + +/** + * The compiled restart-required registry (`internal/config/restart_keys.go`). + * It only changes across deploys, so it is cached aggressively and never + * retried: an older server without the endpoint degrades to "nothing needs a + * restart" rather than to a broken settings page. + */ +export function useAdminRestartKeys() { + return useQuery({ + queryKey: adminKeys.restartKeys(), + queryFn: () => api("/admin/settings/restart-keys"), + staleTime: 5 * 60_000, + retry: false, + }); +} + export function useAdminServerStatus() { return useQuery({ queryKey: adminKeys.serverStatus(), diff --git a/web/src/hooks/queries/keys.ts b/web/src/hooks/queries/keys.ts index 8dd6b00b3..a9ab6ba7c 100644 --- a/web/src/hooks/queries/keys.ts +++ b/web/src/hooks/queries/keys.ts @@ -376,6 +376,7 @@ export const adminKeys = { sessions: () => ["admin", "sessions"] as const, serverSettings: () => ["admin", "serverSettings"] as const, serverStatus: () => ["admin", "serverStatus"] as const, + restartKeys: () => ["admin", "restartKeys"] as const, catalogSearchStatus: () => ["admin", "catalogSearchStatus"] as const, jellyfinCompatStatus: () => ["admin", "jellyfinCompatStatus"] as const, requestsRoot: () => ["admin", "requests"] as const, diff --git a/web/src/hooks/useRestartKeys.test.ts b/web/src/hooks/useRestartKeys.test.ts new file mode 100644 index 000000000..77aa13a85 --- /dev/null +++ b/web/src/hooks/useRestartKeys.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; + +import { createRestartKeyMatcher } from "./useRestartKeys"; + +describe("createRestartKeyMatcher", () => { + it("matches keys listed exactly", () => { + const matcher = createRestartKeyMatcher({ keys: ["auth.jwt_secret"], prefixes: [] }); + + expect(matcher.has("auth.jwt_secret")).toBe(true); + expect(matcher.has("auth.jwt_expiry")).toBe(false); + }); + + it("matches every key under a listed prefix", () => { + const matcher = createRestartKeyMatcher({ keys: [], prefixes: ["database.", "redis."] }); + + expect(matcher.has("database.max_connections")).toBe(true); + expect(matcher.has("redis.url")).toBe(true); + // The prefix includes its trailing dot, so a sibling namespace that merely + // starts with the same word must not be badged. + expect(matcher.has("databases_extra.url")).toBe(false); + expect(matcher.has("branding.server_name")).toBe(false); + }); + + it("treats a missing or malformed payload as 'nothing needs a restart'", () => { + expect(createRestartKeyMatcher(undefined).has("database.max_connections")).toBe(false); + expect( + createRestartKeyMatcher({ keys: [], prefixes: [] }).has("database.max_connections"), + ).toBe(false); + // An older server can answer with nulls where the arrays should be. + const malformed = { keys: null, prefixes: null } as unknown as { + keys: string[]; + prefixes: string[]; + }; + expect(createRestartKeyMatcher(malformed).has("auth.jwt_secret")).toBe(false); + }); + + it("ignores empty strings so a blank prefix cannot match everything", () => { + const matcher = createRestartKeyMatcher({ keys: ["", "s3.bucket"], prefixes: [""] }); + + expect(matcher.has("s3.bucket")).toBe(true); + expect(matcher.has("branding.server_name")).toBe(false); + }); +}); diff --git a/web/src/hooks/useRestartKeys.ts b/web/src/hooks/useRestartKeys.ts new file mode 100644 index 000000000..c12e37dda --- /dev/null +++ b/web/src/hooks/useRestartKeys.ts @@ -0,0 +1,43 @@ +import { useMemo } from "react"; + +import { useAdminRestartKeys, type RestartKeysResponse } from "@/hooks/queries/admin/settings"; + +/** + * Prefix-aware lookup over the server's restart-required registry. It is + * deliberately `Set`-shaped (`has(key)`) so call sites read the same whether + * the key is listed exactly or covered by a namespace prefix. + */ +export interface RestartKeyMatcher { + has(key: string): boolean; +} + +const EMPTY_MATCHER: RestartKeyMatcher = { has: () => false }; + +/** + * Builds a matcher from the endpoint payload. Anything that is not the + * expected `{ keys, prefixes }` shape — a loading query, or a server too old + * to serve the endpoint — degrades to "no key needs a restart" rather than to + * a broken page. + */ +export function createRestartKeyMatcher(data: RestartKeysResponse | undefined): RestartKeyMatcher { + const exact = new Set(Array.isArray(data?.keys) ? data.keys.filter(isNonEmptyString) : []); + const prefixes = Array.isArray(data?.prefixes) ? data.prefixes.filter(isNonEmptyString) : []; + if (exact.size === 0 && prefixes.length === 0) return EMPTY_MATCHER; + return { + has: (key: string) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix)), + }; +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value !== ""; +} + +/** + * Keys whose saved value only takes effect after a server restart. Feed it to + * `SettingField`'s `restartRequired` prop instead of writing "requires a + * restart" into hint text. + */ +export function useRestartKeys(): RestartKeyMatcher { + const { data } = useAdminRestartKeys(); + return useMemo(() => createRestartKeyMatcher(data), [data]); +} diff --git a/web/src/hooks/useSettingsForm.test.ts b/web/src/hooks/useSettingsForm.test.ts index 52e7fc9dc..aa1362ab8 100644 --- a/web/src/hooks/useSettingsForm.test.ts +++ b/web/src/hooks/useSettingsForm.test.ts @@ -140,3 +140,54 @@ describe("useSettingsForm save()", () => { expect(result.current.restartRequired).toBe(true); }); }); + +describe("useSettingsForm unsaved-changes guard", () => { + function fireBeforeUnload(): Event { + // jsdom has no BeforeUnloadEvent, and its legacy `returnValue` is a + // boolean mirror of the canceled flag — `defaultPrevented` is the portable + // signal that the browser would prompt. + const event = new Event("beforeunload", { cancelable: true }); + window.dispatchEvent(event); + return event; + } + + it("does not warn while the form is clean", () => { + renderHook(() => useSettingsForm({ keys: KEYS })); + + expect(fireBeforeUnload().defaultPrevented).toBe(false); + }); + + it("warns before the page unloads with staged edits", () => { + const { result } = renderHook(() => useSettingsForm({ keys: KEYS })); + + act(() => { + result.current.setValue("branding.server_name", "Casa"); + }); + + expect(fireBeforeUnload().defaultPrevented).toBe(true); + }); + + it("stops warning once the edits are discarded", () => { + const { result } = renderHook(() => useSettingsForm({ keys: KEYS })); + + act(() => { + result.current.setValue("branding.server_name", "Casa"); + }); + act(() => { + result.current.discard(); + }); + + expect(fireBeforeUnload().defaultPrevented).toBe(false); + }); + + it("stops warning after the hook unmounts", () => { + const { result, unmount } = renderHook(() => useSettingsForm({ keys: KEYS })); + + act(() => { + result.current.setValue("branding.server_name", "Casa"); + }); + unmount(); + + expect(fireBeforeUnload().defaultPrevented).toBe(false); + }); +}); diff --git a/web/src/hooks/useSettingsForm.ts b/web/src/hooks/useSettingsForm.ts index 4351b31e7..58f09532f 100644 --- a/web/src/hooks/useSettingsForm.ts +++ b/web/src/hooks/useSettingsForm.ts @@ -84,6 +84,26 @@ export function useSettingsForm({ keys }: UseSettingsFormOptions) { const dirtyCount = dirty.size; const dirtyKeys = useMemo(() => Array.from(dirty), [dirty]); + // Every admin settings tab stages edits and only writes them through the + // SaveBar, so closing or reloading the tab would silently drop them. One + // guard here covers all tabs. + // + // Only the browser-level teardown is guarded: react-router's `useBlocker` + // needs a data router, and the app mounts the declarative `` + // (web/src/App.tsx), where calling it throws. Wire it up here if the app + // ever moves to `createBrowserRouter`. + useEffect(() => { + if (dirtyCount === 0) return; + function warnOnUnload(event: BeforeUnloadEvent) { + event.preventDefault(); + // Older browsers only show the prompt for a truthy returnValue; the text + // itself is ignored everywhere. + event.returnValue = ""; + } + window.addEventListener("beforeunload", warnOnUnload); + return () => window.removeEventListener("beforeunload", warnOnUnload); + }, [dirtyCount]); + const isDirty = useCallback((key: string) => dirty.has(key), [dirty]); const buildConnectionCheckRequest = useCallback( diff --git a/web/src/lib/adminNavigation.ts b/web/src/lib/adminNavigation.ts index 973bbfced..732c0bc01 100644 --- a/web/src/lib/adminNavigation.ts +++ b/web/src/lib/adminNavigation.ts @@ -14,13 +14,11 @@ import { PanelsTopLeft, Puzzle, Radio, - RefreshCw, ScrollText, Send, Server, ShieldCheck, SkipForward, - SlidersHorizontal, Users, UsersRound, Wrench, @@ -29,7 +27,7 @@ import type { LucideIcon } from "lucide-react"; import type { PluginInstallation } from "@/api/types"; import type { SettingsSearchGroup, SettingsSearchItem } from "@/components/settings/settingsSearch"; -import { ADMIN_SETTINGS_GROUPS } from "@/lib/adminSettingsSearch"; +import { ADMIN_SETTINGS_NAV } from "@/lib/adminSettingsSearch"; import { pluginRouteHref } from "@/lib/pluginRouteHref"; export interface AdminNavItem extends SettingsSearchItem { @@ -86,8 +84,18 @@ export const ADMIN_NAV_SECTIONS: AdminNavGroup[] = [ items: [ { label: "Libraries", - description: "Media libraries, paths, scanning, and catalog import/export.", - keywords: ["library", "paths", "scan", "catalog", "seed"], + description: "Media libraries, paths, scanning, autoscan sources, and catalog import.", + keywords: [ + "library", + "paths", + "scan", + "catalog", + "seed", + "autoscan", + "scan queue", + "polling", + "webhook source", + ], icon: Library, href: "/admin/libraries", }, @@ -117,13 +125,6 @@ export const ADMIN_NAV_SECTIONS: AdminNavGroup[] = [ { label: "Automation", items: [ - { - label: "Autoscan", - description: "Autoscan sources, queue state, and poller behavior.", - keywords: ["scan queue", "cephfs", "polling", "matcher"], - icon: RefreshCw, - href: "/admin/autoscan", - }, { label: "Scheduled Tasks", description: "Background task schedules, runs, and job history.", @@ -132,7 +133,7 @@ export const ADMIN_NAV_SECTIONS: AdminNavGroup[] = [ href: "/admin/tasks", }, { - label: "Subtitles", + label: "Subtitle Files", description: "Downloaded subtitle records and subtitle admin tools.", keywords: ["captions", "subtitle downloads", "providers"], icon: Captions, @@ -194,16 +195,20 @@ export const ADMIN_NAV_SECTIONS: AdminNavGroup[] = [ }, ], }, + { + label: "Settings", + items: ADMIN_SETTINGS_NAV.map((item) => ({ + label: item.label, + description: item.description, + keywords: ["settings", "configuration", ...(item.keywords ?? [])], + settings: item.settings, + icon: item.icon, + href: `/admin/settings?tab=${encodeURIComponent(item.id)}`, + })), + }, { label: "System", items: [ - { - label: "Settings", - description: "Server-wide settings, integrations, storage, and compatibility proxies.", - keywords: ["configuration", "server settings", "admin settings"], - icon: SlidersHorizontal, - href: "/admin/settings", - }, { label: "Plugins", description: "Plugin catalog, repositories, installs, and plugin configuration.", @@ -294,31 +299,11 @@ export function appendAdminPluginNavSection( ]; } -export function appendAdminSettingsNavSection(sections: readonly AdminNavGroup[]): AdminNavGroup[] { - return [ - ...sections.map((section) => ({ ...section, items: [...section.items] })), - { - label: "Admin Settings", - items: ADMIN_SETTINGS_GROUPS.flatMap((group) => - group.items.map((item) => ({ - label: item.label, - description: item.description, - keywords: ["admin settings", group.label, ...(item.keywords ?? [])], - settings: item.settings, - icon: item.icon, - href: `/admin/settings?tab=${encodeURIComponent(item.id)}`, - })), - ), - }, - ]; -} - export function buildAdminCommandNavSections( installations: readonly PluginInstallation[] | undefined, visibility: AdminNavVisibility = {}, ): AdminNavGroup[] { - return appendAdminPluginNavSection( - appendAdminSettingsNavSection(buildAdminNavSections(visibility)), - installations, - ); + // The settings tabs are part of the base nav now, so the command palette + // needs nothing appended for them. + return appendAdminPluginNavSection(buildAdminNavSections(visibility), installations); } diff --git a/web/src/lib/adminSettingsSearch.ts b/web/src/lib/adminSettingsSearch.ts index b503bfcd4..4e71952b4 100644 --- a/web/src/lib/adminSettingsSearch.ts +++ b/web/src/lib/adminSettingsSearch.ts @@ -1,24 +1,13 @@ import { Bell, - Captions, - Cloud, - Database, - Download, - Gauge, - HardDrive, - Layers, - Mail, - Image, Network, Paintbrush, PlayCircle, Puzzle, - Search, - ScanSearch, - ScrollText, + Server, Settings2, - Sparkles, - Subtitles, + ShieldCheck, + Wand2, } from "lucide-react"; import type { LucideIcon } from "lucide-react"; @@ -31,14 +20,18 @@ export interface AdminSettingsSearchItem extends SettingsSearchItem { keywords?: readonly string[]; settings?: readonly { label: string; description?: string; keywords?: readonly string[] }[]; icon: LucideIcon; + /** Short qualifier rendered next to the label in the settings nav. */ + badge?: string; } export type AdminSettingsSearchGroup = SettingsSearchGroup; const settingIndex = (...labels: string[]) => labels.map((label) => ({ label })); -// Tab ids are stable URL state (?tab=...) — regroup or reorder freely, but -// renaming an id breaks bookmarks and deep links. +// Tab ids are stable URL state (?tab=...). Old ids from the 20-tab layout are +// kept working by LEGACY_ADMIN_SETTINGS_TAB_ALIASES below, not by keeping the +// tabs themselves — regroup or reorder freely, but add an alias entry whenever +// an id disappears. export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [ { label: "Server", @@ -46,28 +39,39 @@ export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [ { id: "general", label: "General", - description: "Authentication, token lifetimes, and server logging.", - keywords: ["access token", "refresh token", "expiry", "log level", "quiet subsystems"], + description: "Server identity, public signups, and logging.", + keywords: [ + "server name", + "login subtitle", + "signup", + "invite", + "log level", + "quiet", + "branding name", + ], settings: settingIndex( - "Access Token Expiry", - "Refresh Token Expiry", + "Identity", + "Server Name", + "Login Page Subtitle", + "Access", + "Public Signups", + "Logging", "Log Level", - "Quiet Subsystems", + "Silenced Log Messages", ), icon: Settings2, }, { - id: "branding", - label: "Branding", - description: "White-label logo, favicon, server name, login background, and accent color.", + id: "appearance", + label: "Appearance", + description: "Logos, accent color, default theme, custom CSS, and poster badges.", keywords: [ "logo", "wordmark", - "icon", "favicon", + "login background", "white label", "brand", - "login background", "accent color", "default theme", "app name", @@ -75,305 +79,238 @@ export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [ "light theme logo", "light wordmark", "light icon", + "theme", + "custom css", + "community themes", + "overlays", + "poster badges", ], settings: settingIndex( - "Server Name", - "Login Page Subtitle", + "Logos & Icons", "Logo (wordmark)", "Logo (wordmark, light themes)", "Logo (icon)", "Logo (icon, light themes)", "Favicon", - "Login Background", - "Brand Accent Color", - "Default Theme", + "Login background", + "Colors & Theme", + "Accent color", + "Custom accent color", + "Default theme", + "Individual colors and fonts", + "Custom CSS", + "Community theme list", + "Card Overlays", + "Show badges on poster art", + "Badge style", ), - icon: Image, - }, - { - id: "theming", - label: "Theming", - description: "Server theme token overrides, custom CSS, and the theme catalog.", - keywords: ["theme", "custom css", "community themes", "appearance", "token overrides"], - settings: settingIndex("Preview", "Token Overrides", "Custom CSS", "Theme Catalog URL"), icon: Paintbrush, }, { - id: "overlays", - label: "Card Overlays", - description: "Server-wide card quick-action and poster badge defaults.", + id: "security", + label: "Security & Access", + description: "Sign-in sessions, trusted proxies, and request rate limits.", keywords: [ - "poster", - "badges", - "quick actions", - "favorites", - "watch indicator", - "defaults.card_overlays", - "overlay preset", + "access token", + "refresh token", + "expiry", + "session", + "proxy", + "x-forwarded-for", + "client ip", + "rate limit", + "throttle", + "429", + "api key tier", ], settings: settingIndex( - "Card Quick Actions Default", - "Card quick actions", - "Card Overlays Default", - "Default Configuration", - "Default style preset", - "Overlay position", - "Overlay enabled", - ), - icon: Layers, - }, - ], - }, - { - label: "Media", - items: [ - { - id: "scanner", - label: "Scanner & Matcher", - description: "Scan workers, matcher workers, batch size, and image caching.", - keywords: ["scanner workers", "matcher workers", "batch size", "metadata cache images"], - settings: settingIndex( - "Scanner Workers", - "Matcher Workers", - "Matcher Batch Size", - "Cache Images to S3", + "Sign-in Sessions", + "Access Token Expiry", + "Refresh Token Expiry", + "Network", + "Trusted Proxies", + "Rate Limiting", + "Enable Rate Limiting", + "Where Counters Are Kept", + "Per Client Address", + "Burst allowance", + "Standard API keys", + "Elevated API keys", + "Sign-in and Webhook Endpoints", ), - icon: ScanSearch, + icon: ShieldCheck, }, { - id: "search", - label: "Search", - description: "Catalog search provider, Meilisearch connection, and index maintenance.", + id: "library", + label: "Library & Metadata", + description: "Artwork caching, scanning, intro and credits markers, and catalog search.", keywords: [ - "catalog search", - "meilisearch", - "postgres fts", - "full text search", - "index", - "typo tolerance", + "scanner workers", + "matcher", "batch size", - "index scope", - "semantic search", - "hybrid search", - "vectors", - "embeddings", - "embedder", - "binary quantization", - "quantized", - ], - settings: settingIndex( - "Preferred Provider", - "URL", - "API Key", - "Index Prefix", - "Timeout (ms)", - "Matching Strategy", - "Sync Batch Size", - "Rebuild Batch Size", - "Rebuild Queue Depth", - "Indexed Types", - "Semantic Search", - "Semantic Ratio", - "Embedder", - "Vectorized Documents", - "Status", - ), - icon: Search, - }, - { - id: "intro", - label: "Intro Markers", - description: "Marker lookup mode, playback fetches, providers, and submissions.", - keywords: [ + "cache images", "intro", "credits", "recap", "markers", - "chapter markers", - "provider contributions", - ], - settings: settingIndex( - "Mode", - "Fetch Markers at Playback if Missing", - "Use for Online Marker Lookup", - "Allow Contributions", - "Auto-submit Local Markers", - "Marker Providers", - ), - icon: Captions, - }, - { - id: "subtitles", - label: "Subtitles", - description: "Downloaded subtitles, provider settings, and subtitle appearance.", - keywords: ["opensubtitles", "providers", "subtitle language", "caption", "downloaded"], - settings: settingIndex( - "Provider settings", - "Downloaded subtitles", - "Subtitle appearance", - "Subtitle language", - "Subtitle behavior", - "Forced subtitles", - ), - icon: Subtitles, - }, - { - id: "ai", - label: "AI Services", - description: "AI provider endpoints, translation, transcription, and quotas.", - keywords: [ - "openai", - "ollama", - "base url", - "api key", - "chat model", - "translation", - "transcription", - "subtitles", - "quota", + "meilisearch", + "postgres search", + "semantic", ], settings: settingIndex( - "Text translation", - "Base URL", - "Chat model", - "API Key", - "Test Text AI", - "Speech-to-text", - "Transcription model", - "Transcription base URL", - "Transcription API key", - "Test Speech-to-Text", - "Features", - "Max concurrent jobs", - "Subtitle translation", - "Subtitle generation from audio", - "Description translation", - "On-view translation", - "Subtitle batch size", - "Subtitle context lines", - "Transcription chunk length (seconds)", - "Transcription limit per account", - "Transcription limit period", - "Advanced", + "Metadata", + "Store artwork on this server", + "Scanning", + "Scanner workers", + "Matcher workers", + "Matcher batch size", + "Intro and credits markers", + "Find intros and credits", + "Look up missing markers when playback starts", + "Marker providers", + "Search", + "Search engine", + "Meilisearch URL", + "Meilisearch API key", + "Index name prefix", + "Query timeout (ms)", + "When a search has several words", + "Items sent to the index per batch", + "Match by meaning as well as words", + "Meaning-based share of results", + "Search status", ), - icon: Sparkles, + icon: Wand2, }, { id: "playback", label: "Playback", - description: "FFmpeg, transcoding, hardware acceleration, segments, and resume behavior.", + description: "Transcoding, hardware acceleration, watch thresholds, and downloads.", keywords: [ "ffmpeg", "transcode", "hardware acceleration", + "gpu", "chapter thumbnails", "watched threshold", - "resume threshold", + "resume", "4k", + "downloads", + "bandwidth", + "offline", ], settings: settingIndex( - "FFmpeg Path", - "Transcode Directory", - "Hardware Acceleration", - "Transcoding Enabled", - "Local Transcode Fallback", - "Allow 4K Transcoding", - "Enable Transcode Throttling", - "Throttle Buffer (seconds)", - "Chapter Thumbnail Workers", - "Chapter Thumbnail Execution", - "Chapter Thumbnail Node Capacity", - "HDR Chapter Thumbnail Policy", - "Enable CPU Tone Mapping", - "Watched Threshold (%)", - "Min Resume Threshold (%)", + "Transcoding", + "Hardware acceleration", + "Allow 4K transcoding", + "FFmpeg path", + "Transcode directory", + "GPU devices", + "Transcode on this server when no node is free", + "Pause transcoding once it runs far ahead", + "Buffer ahead (seconds)", + "Chapter thumbnail workers", + "Generate chapter thumbnails on", + "Chapter thumbnails for HDR video", + "Convert HDR colors on the CPU when the GPU cannot", + "Watch behavior", + "Mark watched at (%)", + "Show resume after (%)", + "Downloads", + "Allow downloads", + "Per-user bandwidth", + "Server bandwidth", + "Downloads at once per user", + "Downloads per period", + "Period length", + "Prepare downloads in a device-friendly format", + "Prepared file directory", + "Prepared file storage budget", ), icon: PlayCircle, }, - { - id: "downloads", - label: "Downloads", - description: "Download enablement, bandwidth, concurrency, and period limits.", - keywords: ["bandwidth", "concurrent downloads", "download limit", "period duration"], - settings: settingIndex( - "Downloads Enabled", - "Server Bandwidth (Mbps)", - "Per-User Bandwidth (Mbps)", - "Max Concurrent Downloads Per User", - "Max Downloads Per Period", - "Period Duration", - ), - icon: Download, - }, ], }, { - label: "Connections", + label: "Connections & Data", items: [ - { - id: "watch-providers", - label: "Watch Providers", - description: "Provider integrations for watch history and scrobbling.", - keywords: ["trakt", "simkl", "import", "export", "scrobble", "watch history", "favorites"], - settings: settingIndex( - "Trakt Client ID", - "Trakt Client Secret", - "Simkl Client ID", - "Simkl Client Secret", - ), - icon: Cloud, - }, { id: "integrations", label: "Integrations", - description: "Third-party integration keys and service connections.", - keywords: ["mdblist", "api key", "metadata lists"], - settings: settingIndex("API Key"), - icon: Puzzle, - }, - { - id: "email", - label: "Email", - description: "SMTP delivery, sender address, digest schedule, and external URL.", - keywords: ["smtp", "mail", "from address", "digest", "external url", "tls"], + description: "Subtitle, watch, metadata, and AI provider accounts plus AI features.", + keywords: [ + "opensubtitles", + "subdl", + "subsource", + "trakt", + "simkl", + "mdblist", + "discord app", + "bot token", + "openai", + "ollama", + "groq", + "whisper", + "api key", + "translation", + "transcription", + ], settings: settingIndex( - "Email Enabled", - "From Address", - "From Name", - "Host", - "Port", - "Security", - "Username", - "Password", - "Verify", + "Subtitle providers", + "Watch providers", + "Trakt", + "Simkl", + "Metadata", + "MDBList", + "Apps", + "Discord app", + "Client ID", + "Client secret", + "Bot token", + "AI services", + "Text model", + "Speech-to-text", + "Base URL", + "Model", + "API key", + "AI features", + "Translate subtitles", + "Create subtitles from audio", + "Translate descriptions", + "Description translation for viewers", + "Jobs running at once", + "Subtitle lines per request", + "Surrounding lines sent for context", + "Audio sent per request (seconds)", + "Transcriptions per account", + "Allowance resets", ), - icon: Mail, + icon: Puzzle, }, { id: "notifications", label: "Notifications", - description: - "Server notification channels, release events, Silo Push Relay, Discord, web push, and webhooks.", + description: "Release events, delivery channels, the mail server, and webhooks.", keywords: [ "release events", "new episode", + "email", + "smtp", + "mail", "silo push relay", "mobile push", - "apple push", - "android push", "apns", - "push relay", - "privacy disclosure", + "fcm", "discord", - "browser push", "web push", "webhooks", "server channels", + "digest", ], settings: settingIndex( - "Record events", + "Notice new content", "Enable release events", - "Fan out", + "Work out who wants it", "Enable fanout", "Delivery Channels", "In-App", @@ -381,153 +318,158 @@ export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [ "Silo Push Relay", "Relay URL", "Deployment ID", - "Register Relay", - "Clear Relay Credential", - "Privacy Disclosure", "Email", - "Allow Per-Episode Email", - "Digest Hour", - "External URL", + "Send email from this server", + "From address", + "From name", + "Mail server address", + "Port", + "Encryption", + "Username", + "Password", + "Test email recipient", + "Let people pick an email per episode", + "Send the daily summary at", + "Link back to this server at", "Discord", - "Client ID", - "Client Secret", - "Bot Token", - "Invite Bot to Server", - "Clear Discord Credentials", - "Allow Per-Episode DMs", - "Embed Posters", + "Let people pick a DM per episode", + "Show artwork in Discord messages", + "Mention the requester on Discord", "Personal Webhooks", - "Max Webhooks Per Profile", - "Deliveries Per Minute Per Profile", - "Allow Private Destinations", + "Webhooks each person may create", + "Webhook calls per minute, per person", + "Allow webhooks to private addresses", "Server Channels", - "Batch Window (seconds)", - "Mention Requesters on Discord", - "Settle Delay (seconds)", - "Max Series Burst", - "Max Event Age (hours)", - "Read Notifications (days)", - "Unread Notifications (days)", - "Processed Events (days)", + "Grouping and flood control", + "Wait before sending (seconds)", + "Most messages per show at once", + "How long notifications are kept", ), icon: Bell, }, { - id: "compatibility-proxies", - label: "Compatibility Proxies", - description: "Jellyfin and Audiobookshelf compatibility proxy settings.", - keywords: ["jellyfin", "audiobookshelf", "abs", "public url", "server id", "session ttl"], + id: "compatibility", + label: "Compatibility", + description: + "Jellyfin and Audiobookshelf client compatibility and the Jellyfin web player.", + keywords: [ + "jellyfin", + "audiobookshelf", + "abs", + "proxy", + "public url", + "server id", + "session ttl", + "web player", + ], settings: settingIndex( - "Public URL", - "Server Name", + "Jellyfin", + "Allow Jellyfin apps to connect", + "Address Jellyfin apps should use", + "Jellyfin Web install progress", + "Web player version to install", + "Web player install folder", + "Name shown to Jellyfin apps", "Server ID", - "Emulated Server Version", - "Session TTL", - "Playback Session TTL", - "Enable Audiobookshelf Proxy", + "Jellyfin version to report", + "Stay signed in for", + "Forget idle playback after", + "Audiobookshelf", + "Allow Audiobookshelf apps to connect", ), icon: Network, }, { - id: "rate-limiting", - label: "Rate Limiting", - description: "Request limits, API tiers, admin limits, and authentication throttles.", - keywords: ["limits", "tiers", "requests", "throttle", "429", "api keys"], - settings: settingIndex( - "Enable Rate Limiting", - "Backend", - "Global Requests Per Second", - "Per-IP Limits", - "Requests / Second", - "Requests / Minute", - "Burst", - "Standard", - "Elevated", - "Login", - "Signup", - "Setup", - "Authentication endpoints", - ), - icon: Gauge, - }, - ], - }, - { - label: "Data", - items: [ - { - id: "database", - label: "Database", - description: "Postgres, Redis, user database pooling, and Litestream settings.", + id: "infrastructure", + label: "Infrastructure", + description: "Redis, S3 storage buckets, the database, and log retention.", + badge: "Advanced", keywords: [ - "postgres", "redis", - "connection url", - "user db", + "s3", + "bucket", + "endpoint", + "region", + "access key", + "secret key", + "postgres", "pool", - "litestream", - "stale grace", + "user db", + "ops log", + "retention", + "decision log", + "opa", ], settings: settingIndex( - "Max Connections", - "Enable Redis", + "Redis", + "Use Redis", "Connection URL", - "User DB Backend", - "Pool Max Open", - "Idle Timeout", - "Litestream Sync Interval", - "Stale Grace Seconds", - ), - icon: Database, - }, - { - id: "storage", - label: "Storage", - description: "Public and private S3 storage buckets, endpoints, and credentials.", - keywords: ["s3", "bucket", "endpoint", "region", "access key", "secret key", "uploads"], - settings: settingIndex( + "Public storage", + "Private storage", "Endpoint", "Region", - "Path Style", "Bucket", "Access Key", "Secret Key", - "URL Auth Method", - "Read Endpoint", + "Put the bucket name in the URL path", + "Folder inside the bucket", + "How asset links are authorized", + "Address clients download from", "Token Secret", - "Token Param", - "Token TTL (seconds)", + "Token query parameter", + "Link lifetime (seconds)", + "Database", + "Maximum Postgres connections", + "Where per-user data is stored", + "Open files per user", + "Close idle user databases after", + "Server logs", + "How much to record", + "Delete log entries older than (days)", + "Maximum log entries", + "Maximum log size (MB)", + "Permission checks", + "Delete permission records older than (days)", + "Record one allowed check in every", ), - icon: HardDrive, - }, - { - id: "log-retention", - label: "Log Retention", - description: - "Operations log cleanup, policy decision log cleanup, access log cleanup, and retention policies.", - keywords: [ - "ops log", - "access log", - "policy decision log", - "decision log", - "opa", - "cleanup", - "retention", - "history", - ], - settings: settingIndex( - "Retention Days", - "Max Rows", - "Max Size (MB)", - "Decision Log Retention Days", - "Decision Log Verbosity", - "Scope Sample Rate", - "Bucket Overrides", - ), - icon: ScrollText, + icon: Server, }, ], }, ]; export const ADMIN_SETTINGS_NAV = ADMIN_SETTINGS_GROUPS.flatMap((group) => group.items); + +const ADMIN_SETTINGS_TAB_IDS = new Set(ADMIN_SETTINGS_NAV.map((item) => item.id)); + +/** + * Deep links from the 20-tab layout. Bookmarks, docs, and older client builds + * still point at these ids, so every one of them resolves to the tab that + * absorbed it rather than falling through to the settings index. + */ +export const LEGACY_ADMIN_SETTINGS_TAB_ALIASES: Readonly> = { + branding: "appearance", + theming: "appearance", + overlays: "appearance", + "rate-limiting": "security", + scanner: "library", + search: "library", + intro: "library", + subtitles: "integrations", + ai: "integrations", + "watch-providers": "integrations", + downloads: "playback", + email: "notifications", + jellyfin: "compatibility", + "compatibility-proxies": "compatibility", + database: "infrastructure", + storage: "infrastructure", + "log-retention": "infrastructure", +}; + +/** Resolves a `?tab=` value to a current tab id, or null when it names none. */ +export function resolveAdminSettingsTabID(value: string | null): string | null { + if (!value) return null; + if (ADMIN_SETTINGS_TAB_IDS.has(value)) return value; + return LEGACY_ADMIN_SETTINGS_TAB_ALIASES[value] ?? null; +} diff --git a/web/src/pages/AdminAutoscan.tsx b/web/src/pages/AdminAutoscan.tsx index b7e1cd9ea..f5d81043b 100644 --- a/web/src/pages/AdminAutoscan.tsx +++ b/web/src/pages/AdminAutoscan.tsx @@ -126,9 +126,19 @@ function SettingsTab() { // Page // --------------------------------------------------------------------------- -export default function AdminAutoscan() { +interface AdminAutoscanProps { + /** + * Rendered inside the Libraries page rather than as its own route. The + * heading drops to an h2 and the Sources/Activity selection moves to `view`, + * because `tab` already names the Libraries tab that hosts this panel. + */ + embedded?: boolean; +} + +export default function AdminAutoscan({ embedded = false }: AdminAutoscanProps = {}) { const [searchParams, setSearchParams] = useSearchParams(); - const requestedTab = searchParams.get("tab"); + const tabParam = embedded ? "view" : "tab"; + const requestedTab = searchParams.get(tabParam); const activeTab = normalizeTab(requestedTab); const trigger = useTriggerAutoscan(); const settings = useAutoscanSettings(); @@ -148,9 +158,9 @@ export default function AdminAutoscan() { function setActiveTab(value: string) { const next = new URLSearchParams(searchParams); if (value === "sources") { - next.delete("tab"); + next.delete(tabParam); } else { - next.set("tab", value); + next.set(tabParam, value); } setSearchParams(next, { replace: true }); } @@ -160,9 +170,13 @@ export default function AdminAutoscan() {
-

- Autoscan -

+ {embedded ? ( +

Autoscan

+ ) : ( +

+ Autoscan +

+ )} {settings.data && (enabled ? ( Enabled diff --git a/web/src/pages/AdminDashboard.tsx b/web/src/pages/AdminDashboard.tsx index 148c6d9a0..3ff191eb7 100644 --- a/web/src/pages/AdminDashboard.tsx +++ b/web/src/pages/AdminDashboard.tsx @@ -2,11 +2,8 @@ import type { ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Link, useNavigate } from "react-router"; import { AdminSessionActions } from "@/components/AdminSessionActions"; -import { AdminSectionCommandDialog } from "@/components/AdminSectionCommandDialog"; import { useEventChannel } from "@/components/realtimeEventsContext"; import { fetchAdminStats, useAdminStats, useAdminSessions } from "@/hooks/queries/admin/stats"; -import { useAdminPluginInstallations } from "@/hooks/queries/admin/plugins"; -import { usePolicyCapability } from "@/hooks/queries/admin/policy"; import { useAdminUsers } from "@/hooks/queries/admin/users"; import { useAdminLibraries, @@ -52,7 +49,6 @@ import { useQueryClient } from "@tanstack/react-query"; import { adminKeys } from "@/hooks/queries/keys"; import { usePageActivity } from "@/hooks/usePageActivity"; import { cn } from "@/lib/utils"; -import { buildAdminCommandNavSections } from "@/lib/adminNavigation"; import { compareActiveScans, formatActiveScanMode, formatActiveScanProgress } from "@/lib/scanRuns"; import { JellyfinSessionPill } from "@/components/JellyfinSessionPill"; import { @@ -87,8 +83,6 @@ export default function AdminDashboard() { const sessionsQuery = useAdminSessions(); const librariesQuery = useAdminLibraries(); const usersQuery = useAdminUsers(); - const { data: adminInstallations } = useAdminPluginInstallations(); - const policyCapability = usePolicyCapability(); const scanAll = useScanAllLibraries(); const pageActivity = usePageActivity(); const manualRefreshStartedAtRef = useRef(null); @@ -119,13 +113,6 @@ export default function AdminDashboard() { const lastUpdatedLabel = lastDashboardUpdatedAt ? formatRelativeUpdatedLabel(relativeUpdatedNow, lastDashboardUpdatedAt) : null; - const adminSearchSections = useMemo( - () => - buildAdminCommandNavSections(adminInstallations, { - policyEditorAvailable: policyCapability.data?.editor_available === true, - }), - [adminInstallations, policyCapability.data?.editor_available], - ); useEffect(() => { if (!lastDashboardUpdatedAt) { @@ -230,8 +217,6 @@ export default function AdminDashboard() { return (
- - {/* Page header */}
diff --git a/web/src/pages/AdminDevices.tsx b/web/src/pages/AdminDevices.tsx index d510bd707..42257f13e 100644 --- a/web/src/pages/AdminDevices.tsx +++ b/web/src/pages/AdminDevices.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { useMemo, useState, type ReactNode } from "react"; import { Link, useParams, useSearchParams } from "react-router"; import { Activity, @@ -216,20 +216,6 @@ export default function AdminDevices() { const [groupBy, setGroupBy] = useState("user"); const [overridesOnly, setOverridesOnly] = useState(false); - // ⌘K focuses the global search input - const searchRef = useRef(null); - useEffect(() => { - const onKey = (e: KeyboardEvent) => { - if ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k") { - e.preventDefault(); - searchRef.current?.focus(); - searchRef.current?.select(); - } - }; - window.addEventListener("keydown", onKey); - return () => window.removeEventListener("keydown", onKey); - }, []); - // anomaly detection const anomalies = useMemo(() => detectAnomalies(devices), [devices]); const scopedDevices = useMemo( @@ -373,15 +359,14 @@ export default function AdminDevices() { ⌘K {" "} - to jump. + to jump to another admin page.

setSearch(event.target.value)} @@ -395,11 +380,7 @@ export default function AdminDevices() { > - ) : ( - - ⌘K - - )} + ) : null}
diff --git a/web/src/pages/AdminLibraries.tsx b/web/src/pages/AdminLibraries.tsx index a9cbf2458..eaa31ea68 100644 --- a/web/src/pages/AdminLibraries.tsx +++ b/web/src/pages/AdminLibraries.tsx @@ -64,7 +64,7 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { Link } from "react-router"; +import { Link, useSearchParams } from "react-router"; import { Plus, Pencil, @@ -88,6 +88,8 @@ import { Search, FolderOpen, } from "lucide-react"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import AdminAutoscan from "@/pages/AdminAutoscan"; import { ConfirmDialog } from "@/components/ConfirmDialog"; import { cn } from "@/lib/utils"; import { toast } from "sonner"; @@ -134,8 +136,31 @@ const EMPTY_ROOT_WARNING_TEXT = const EMPTY_ROOT_WARNING_HINT = "Run another scan after storage returns, or confirm deletion before the next empty-root scan."; +const LIBRARY_TABS = ["libraries", "autoscan"] as const; +type LibraryTab = (typeof LIBRARY_TABS)[number]; + export default function AdminLibraries() { useEventChannel("scans"); + // Autoscan used to be its own sidebar page even though it only ever + // configured how these libraries get scanned; it is a tab here now, and + // /admin/autoscan redirects to it. + const [searchParams, setSearchParams] = useSearchParams(); + const requestedTab = searchParams.get("tab"); + const activeTab: LibraryTab = LIBRARY_TABS.includes(requestedTab as LibraryTab) + ? (requestedTab as LibraryTab) + : "libraries"; + + function setActiveTab(value: string) { + const next = new URLSearchParams(searchParams); + if (value === "libraries") { + next.delete("tab"); + next.delete("view"); + } else { + next.set("tab", value); + } + setSearchParams(next, { replace: true }); + } + const { data: libraries = [], isLoading } = useAdminLibraries(); const { data: activeScans = [] } = useActiveScans(); const { data: libraryRefreshJobs = [] } = useLibraryRefreshJobs(); @@ -292,7 +317,8 @@ export default function AdminLibraries() { }); } - if (isLoading) return
Loading libraries...
; + if (isLoading && activeTab === "libraries") + return
Loading libraries...
; return (
@@ -331,7 +357,7 @@ export default function AdminLibraries() { Manage library roots and scans. Catalog import/export now lives under Maintenance.

-
+
{activeScanGroups.length > 0 && (
- -
- - - - - Name - Paths - Type - Status - Last Scanned - Actions - - - l.id)} - strategy={verticalListSortingStrategy} - > - - {orderedLibraries.map((lib) => { - const isScanning = scanMutation.isPending && scanMutation.variables === lib.id; - const activeRefreshJob = activeRefreshJobsByLibraryId.get(lib.id); - const activeLibraryScans = activeScansByLibraryId.get(lib.id) ?? []; - const runningLibraryScans = activeLibraryScans.filter( - (scan) => scan.status === "running", - ).length; - const queuedLibraryScans = activeLibraryScans.length - runningLibraryScans; - const isRefreshStarting = - refreshMutation.isPending && refreshMutation.variables === lib.id; - const isCheckingMount = - mountCheckMutation.isPending && mountCheckMutation.variables === lib.id; - const mountCheck = lastMountCheckByLibraryId[lib.id]; - const hasActiveWork = - activeRefreshJob !== undefined || activeLibraryScans.length > 0; - const isCancellingLibraryScans = - cancelScansMutation.isPending && cancelScansMutation.variables === lib.id; - const isCancellingRefreshJob = - activeRefreshJob !== undefined && - cancelAdminJobMutation.isPending && - cancelAdminJobMutation.variables === activeRefreshJob.id; - return ( - - - {lib.name} - - {lib.paths.length === 1 ? ( - {lib.paths[0]} - ) : ( - - )} - - - {lib.type} - - -
- - {lib.enabled ? "Enabled" : "Disabled"} - - {runningLibraryScans > 0 ? ( - {runningLibraryScans} running - ) : null} - {queuedLibraryScans > 0 ? ( - {queuedLibraryScans} queued - ) : null} - {lib.scan_warning_code === "empty_root" ? ( - Empty root guarded - ) : null} - {lib.scan_warning_code === "dead_root" ? ( - Root unreachable - ) : null} -
- - -
-
- {lib.last_scanned_at ? formatDateTime(lib.last_scanned_at) : "Never"} -
- {lib.scan_warning_at ? ( -
- Warning: {formatDateTime(lib.scan_warning_at)} -
- ) : null} -
- - -
- -
+ + + + Name + Paths + Type + Status + Last Scanned + Actions + + + l.id)} + strategy={verticalListSortingStrategy} + > + + {orderedLibraries.map((lib) => { + const isScanning = + scanMutation.isPending && scanMutation.variables === lib.id; + const activeRefreshJob = activeRefreshJobsByLibraryId.get(lib.id); + const activeLibraryScans = activeScansByLibraryId.get(lib.id) ?? []; + const runningLibraryScans = activeLibraryScans.filter( + (scan) => scan.status === "running", + ).length; + const queuedLibraryScans = activeLibraryScans.length - runningLibraryScans; + const isRefreshStarting = + refreshMutation.isPending && refreshMutation.variables === lib.id; + const isCheckingMount = + mountCheckMutation.isPending && mountCheckMutation.variables === lib.id; + const mountCheck = lastMountCheckByLibraryId[lib.id]; + const hasActiveWork = + activeRefreshJob !== undefined || activeLibraryScans.length > 0; + const isCancellingLibraryScans = + cancelScansMutation.isPending && cancelScansMutation.variables === lib.id; + const isCancellingRefreshJob = + activeRefreshJob !== undefined && + cancelAdminJobMutation.isPending && + cancelAdminJobMutation.variables === activeRefreshJob.id; + return ( + + + {lib.name} + + {lib.paths.length === 1 ? ( + {lib.paths[0]} ) : ( - + )} - - - - - {lib.scan_warning_code === "empty_root" || - lib.scan_warning_code === "dead_root" ? ( - - ) : null} - - - - {hasActiveWork ? ( - cancelAdminJobMutation.mutate(jobID)} - onCancelScans={(libraryID) => cancelScansMutation.mutate(libraryID)} - /> - ) : null} - - ); - })} - {orderedLibraries - .filter( - (lib) => - lib.scan_warning_code === "empty_root" || - lib.scan_warning_code === "dead_root", - ) - .map((lib) => { - const mountCheck = lastMountCheckByLibraryId[lib.id]; - const isCheckingMount = - mountCheckMutation.isPending && mountCheckMutation.variables === lib.id; - return ( - - -
-
- {lib.scan_warning_code === "dead_root" - ? DEAD_ROOT_WARNING_TEXT - : EMPTY_ROOT_WARNING_TEXT} -
-
- {lib.scan_warning_message ?? - (lib.scan_warning_code === "dead_root" - ? DEAD_ROOT_WARNING_HINT - : EMPTY_ROOT_WARNING_HINT)} -
-
- - {lib.scan_warning_code === "dead_root" ? ( + + + {lib.type} + + +
+ + {lib.enabled ? "Enabled" : "Disabled"} + + {runningLibraryScans > 0 ? ( + {runningLibraryScans} running + ) : null} + {queuedLibraryScans > 0 ? ( + {queuedLibraryScans} queued + ) : null} + {lib.scan_warning_code === "empty_root" ? ( + Empty root guarded + ) : null} + {lib.scan_warning_code === "dead_root" ? ( + Root unreachable + ) : null} +
+ + +
+
+ {lib.last_scanned_at + ? formatDateTime(lib.last_scanned_at) + : "Never"} +
+ {lib.scan_warning_at ? ( +
+ Warning: {formatDateTime(lib.scan_warning_at)} +
+ ) : null} +
+ + +
- ) : null} -
-
- - - ); - })} - - -
-
- - {activeLibrary ? ( - - - - - - - {activeLibrary.name} - - {activeLibrary.paths.length === 1 - ? activeLibrary.paths[0] - : `${activeLibrary.paths.length} folders`} - - - {activeLibrary.type} - - - - - - -
- ) : null} - - - - - - - {skippedRoots.length > 0 ? : null} - {staleIDs.length > 0 && } + + + + + {lib.scan_warning_code === "empty_root" || + lib.scan_warning_code === "dead_root" ? ( + + ) : null} +
+ + + {hasActiveWork ? ( + cancelAdminJobMutation.mutate(jobID)} + onCancelScans={(libraryID) => cancelScansMutation.mutate(libraryID)} + /> + ) : null} + + ); + })} + {orderedLibraries + .filter( + (lib) => + lib.scan_warning_code === "empty_root" || + lib.scan_warning_code === "dead_root", + ) + .map((lib) => { + const mountCheck = lastMountCheckByLibraryId[lib.id]; + const isCheckingMount = + mountCheckMutation.isPending && mountCheckMutation.variables === lib.id; + return ( + + +
+
+ {lib.scan_warning_code === "dead_root" + ? DEAD_ROOT_WARNING_TEXT + : EMPTY_ROOT_WARNING_TEXT} +
+
+ {lib.scan_warning_message ?? + (lib.scan_warning_code === "dead_root" + ? DEAD_ROOT_WARNING_HINT + : EMPTY_ROOT_WARNING_HINT)} +
+
+ + {lib.scan_warning_code === "dead_root" ? ( + + ) : null} +
+
+ + + ); + })} + + + +
+ + {activeLibrary ? ( + + + + + + + {activeLibrary.name} + + {activeLibrary.paths.length === 1 + ? activeLibrary.paths[0] + : `${activeLibrary.paths.length} folders`} + + + {activeLibrary.type} + + + + + + +
+ ) : null} + + + + + + + {skippedRoots.length > 0 ? : null} + {staleIDs.length > 0 && } + + + + + +
); } diff --git a/web/src/pages/admin-settings/AIServicesSettings.test.tsx b/web/src/pages/admin-settings/AIServicesSettings.test.tsx deleted file mode 100644 index 9813e67f9..000000000 --- a/web/src/pages/admin-settings/AIServicesSettings.test.tsx +++ /dev/null @@ -1,234 +0,0 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { renderToStaticMarkup } from "react-dom/server"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import AIServicesSettings from "./AIServicesSettings"; - -const mocks = vi.hoisted(() => ({ - checkConnection: vi.fn(), - discard: vi.fn(), - save: vi.fn(), - setValue: vi.fn(), - toastError: vi.fn(), -})); - -const values: Record = { - "ai.base_url": "https://text.example.test", - "ai.chat_model": "chat-model", - "ai.asr_base_url": "", - "ai.asr_model": "whisper-model", - "ai.max_concurrent_jobs": "2", - "subtitle_ai.base_url": "https://legacy.example.test", - "subtitle_ai.chat_model": "legacy-chat-model", - "subtitle_ai.max_concurrent_jobs": "3", - "subtitle_ai.enabled": "true", - "subtitle_ai.transcribe_enabled": "false", - "subtitle_ai.batch_size": "40", - "subtitle_ai.context_neighbors": "2", - "subtitle_ai.asr_chunk_seconds": "600", - "subtitle_ai.transcribe_quota_jobs": "0", - "subtitle_ai.transcribe_quota_period": "day", - "metadata_ai.enabled": "false", - "metadata_ai.on_view": "button", -}; - -let dirtyCount = 0; - -const useSettingsFormMock = vi.fn((_options?: { keys: string[] }) => ({ - isLoading: false, - getValue: (key: string) => values[key] ?? "", - setValue: mocks.setValue, - dirtyCount, - dirtyKeys: [], - isDirty: vi.fn(() => false), - save: mocks.save, - discard: mocks.discard, - isSaving: false, - restartRequired: false, - sensitiveConfigured: ["subtitle_ai.api_key"], - sensitiveManagedByEnv: [], - buildConnectionCheckRequest: vi.fn(() => ({ values: {}, dirty_keys: [] })), -})); - -vi.mock("@/hooks/useSettingsForm", () => ({ - useSettingsForm: (options: { keys: string[] }) => useSettingsFormMock(options), -})); - -vi.mock("@/hooks/queries/admin/settings", () => ({ - useAdminServerSettings: () => ({ data: values }), - useAdminSensitiveStatus: () => ({ data: { configured: ["ai.api_key"] } }), - useUpdateServerSetting: () => ({ mutateAsync: vi.fn(), isPending: false }), - useCheckAdminSettingsConnection: () => ({ - mutateAsync: mocks.checkConnection, - isPending: false, - }), -})); - -vi.mock("sonner", () => ({ - toast: { - error: mocks.toastError, - }, -})); - -describe("AIServicesSettings", () => { - beforeEach(() => { - dirtyCount = 0; - mocks.checkConnection.mockReset(); - mocks.discard.mockReset(); - mocks.save.mockReset(); - mocks.setValue.mockReset(); - mocks.toastError.mockReset(); - values["ai.base_url"] = "https://text.example.test"; - values["ai.chat_model"] = "chat-model"; - values["ai.asr_base_url"] = ""; - values["ai.asr_model"] = "whisper-model"; - values["ai.max_concurrent_jobs"] = "2"; - values["subtitle_ai.batch_size"] = "40"; - values["subtitle_ai.context_neighbors"] = "2"; - values["subtitle_ai.asr_chunk_seconds"] = "600"; - }); - - it("separates text translation from speech-to-text configuration", () => { - const markup = renderToStaticMarkup(); - - expect(markup).toContain("Text translation"); - expect(markup).toContain("Speech-to-text"); - expect(markup).toContain("Test Text AI"); - expect(markup).toContain("Test Speech-to-Text"); - expect(markup).toContain("Uses the Text translation endpoint"); - }); - - it("shows effective legacy endpoint values until modern keys are saved", () => { - const currentBaseURL = values["ai.base_url"]!; - const currentChatModel = values["ai.chat_model"]!; - values["ai.base_url"] = ""; - values["ai.chat_model"] = ""; - - try { - const markup = renderToStaticMarkup(); - - expect(markup).toContain("https://legacy.example.test"); - expect(markup).toContain("legacy-chat-model"); - } finally { - values["ai.base_url"] = currentBaseURL; - values["ai.chat_model"] = currentChatModel; - } - }); - - it("marks known chat-only fallback endpoints as incompatible with speech-to-text", () => { - const currentBaseURL = values["ai.base_url"]!; - values["ai.base_url"] = "https://openrouter.ai/api"; - - try { - const markup = renderToStaticMarkup(); - - expect(markup).toContain("Incompatible endpoint"); - } finally { - values["ai.base_url"] = currentBaseURL; - } - }); - - it("exposes transcription preset selection to assistive technology", () => { - const markup = renderToStaticMarkup(); - - expect(markup).toContain('aria-pressed="false"'); - }); - - it("explains feature dependencies and keeps advanced tuning secondary", () => { - const markup = renderToStaticMarkup(); - - expect(markup).toContain("Text AI required"); - expect(markup).toContain("Speech-to-text required"); - expect(markup).toContain("Inactive until Description translation is enabled"); - expect(markup).toContain("Advanced"); - }); - - it("points recommendation embeddings to their separate configuration", () => { - const markup = renderToStaticMarkup(); - - expect(markup).toContain("Recommendation embeddings are configured separately"); - expect(markup).toContain('href="/admin/recommendations"'); - expect(markup).not.toContain("Changes take effect after a server restart"); - }); - - it("applies a transcription preset", async () => { - const user = userEvent.setup(); - render(); - - await user.click(screen.getByRole("button", { name: "Groq - fast" })); - - expect(mocks.setValue).toHaveBeenCalledWith("ai.asr_base_url", "https://api.groq.com/openai"); - expect(mocks.setValue).toHaveBeenCalledWith("ai.asr_model", "whisper-large-v3-turbo"); - }); - - it("runs both connection checks and clears their results when drafts are discarded", async () => { - const user = userEvent.setup(); - dirtyCount = 1; - mocks.checkConnection - .mockResolvedValueOnce({ success: true, message: "Text connection verified." }) - .mockResolvedValueOnce({ success: true, message: "Speech connection verified." }); - render(); - - await user.click(screen.getByRole("button", { name: "Test Text AI" })); - await user.click(screen.getByRole("button", { name: "Test Speech-to-Text" })); - expect(await screen.findByText("Text connection verified.")).toBeInTheDocument(); - expect(await screen.findByText("Speech connection verified.")).toBeInTheDocument(); - expect(mocks.checkConnection).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ kind: "ai_chat" }), - ); - expect(mocks.checkConnection).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ kind: "ai_transcription" }), - ); - - await user.click(screen.getByRole("button", { name: "Discard" })); - - expect(mocks.discard).toHaveBeenCalledOnce(); - await waitFor(() => { - expect(screen.queryByText("Text connection verified.")).not.toBeInTheDocument(); - expect(screen.queryByText("Speech connection verified.")).not.toBeInTheDocument(); - }); - }); - - it("clears a prior connection result when its endpoint changes", async () => { - const user = userEvent.setup(); - mocks.checkConnection.mockResolvedValue({ - success: true, - message: "Text connection verified.", - }); - render(); - - await user.click(screen.getByRole("button", { name: "Test Text AI" })); - expect(await screen.findByText("Text connection verified.")).toBeInTheDocument(); - await user.clear(screen.getByRole("textbox", { name: "Base URL" })); - - expect(screen.queryByText("Text connection verified.")).not.toBeInTheDocument(); - }); - - it.each([ - ["ai.max_concurrent_jobs", "1.5", "Max concurrent jobs must be a positive whole number."], - ["subtitle_ai.batch_size", "2abc", "Subtitle batch size must be a positive whole number."], - [ - "subtitle_ai.context_neighbors", - "1.5", - "Subtitle context lines must be zero or a positive whole number.", - ], - [ - "subtitle_ai.asr_chunk_seconds", - "120seconds", - "Transcription chunk length must be between 60 and 600 seconds.", - ], - ])("rejects malformed integer input for %s", async (key, malformedValue, message) => { - const user = userEvent.setup(); - dirtyCount = 1; - values[key] = malformedValue; - render(); - - await user.click(screen.getByRole("button", { name: "Save Changes" })); - - expect(mocks.toastError).toHaveBeenCalledWith(message); - expect(mocks.save).not.toHaveBeenCalled(); - }); -}); diff --git a/web/src/pages/admin-settings/AIServicesSettings.tsx b/web/src/pages/admin-settings/AIServicesSettings.tsx deleted file mode 100644 index 79516773c..000000000 --- a/web/src/pages/admin-settings/AIServicesSettings.tsx +++ /dev/null @@ -1,609 +0,0 @@ -import { useState } from "react"; -import { - AudioLines, - ChevronDown, - CircleAlert, - CircleCheck, - ExternalLink, - Languages, -} from "lucide-react"; -import { toast } from "sonner"; - -import type { ConnectionCheckResponse } from "@/api/types"; -import { ConnectionCheckAction } from "@/components/admin/ConnectionCheckAction"; -import { Badge } from "@/components/ui/badge"; -import { Skeleton } from "@/components/ui/skeleton"; -import { useCheckAdminSettingsConnection } from "@/hooks/queries/admin/settings"; -import { useSettingsForm } from "@/hooks/useSettingsForm"; -import { QUOTA_PERIODS, QUOTA_PERIOD_WINDOW_LABELS } from "@/lib/quotaPeriods"; -import { cn } from "@/lib/utils"; - -import { SaveBar } from "./SaveBar"; -import { SettingField } from "./SettingField"; - -const TEXT_AI_KEYS = ["ai.base_url", "ai.chat_model", "ai.api_key"] as const; -const SPEECH_AI_KEYS = [ - "ai.base_url", - "ai.api_key", - "ai.asr_base_url", - "ai.asr_model", - "ai.asr_api_key", -] as const; -const LEGACY_AI_KEYS = [ - "subtitle_ai.base_url", - "subtitle_ai.api_key", - "subtitle_ai.chat_model", - "subtitle_ai.max_concurrent_jobs", -] as const; -const KEYS: string[] = [ - ...TEXT_AI_KEYS, - ...LEGACY_AI_KEYS, - "ai.asr_base_url", - "ai.asr_model", - "ai.asr_api_key", - "ai.max_concurrent_jobs", - "subtitle_ai.enabled", - "subtitle_ai.transcribe_enabled", - "subtitle_ai.batch_size", - "subtitle_ai.context_neighbors", - "subtitle_ai.asr_chunk_seconds", - "subtitle_ai.transcribe_quota_jobs", - "subtitle_ai.transcribe_quota_period", - "metadata_ai.enabled", - "metadata_ai.on_view", -]; - -const TRANSCRIPTION_PRESETS = [ - { - id: "self-hosted", - label: "Self-hosted", - description: - "Speaches or faster-whisper on your network. Replace the hostname with one reachable from the Silo container.", - baseUrl: "http://speaches:8000", - model: "deepdml/faster-whisper-large-v3-turbo-ct2", - }, - { - id: "groq-turbo", - label: "Groq - fast", - description: "Hosted whisper-large-v3-turbo. Requires a Groq API key.", - baseUrl: "https://api.groq.com/openai", - model: "whisper-large-v3-turbo", - }, - { - id: "groq-accurate", - label: "Groq - accurate", - description: "Hosted whisper-large-v3. Requires a Groq API key.", - baseUrl: "https://api.groq.com/openai", - model: "whisper-large-v3", - }, - { - id: "openai", - label: "OpenAI", - description: "Hosted whisper-1. The transcription key can inherit the Text AI key.", - baseUrl: "https://api.openai.com", - model: "whisper-1", - }, -] as const; - -const CHAT_ONLY_GATEWAY_HOSTS = ["openrouter.ai"]; - -function isChatOnlyGateway(rawURL: string): boolean { - const trimmed = rawURL.trim(); - if (!trimmed) return false; - try { - const host = new URL( - trimmed.includes("://") ? trimmed : `https://${trimmed}`, - ).hostname.toLowerCase(); - return CHAT_ONLY_GATEWAY_HOSTS.some( - (gateway) => host === gateway || host.endsWith(`.${gateway}`), - ); - } catch { - return false; - } -} - -function parseStrictInteger(rawValue: string): number | null { - const trimmed = rawValue.trim(); - if (!/^-?\d+$/.test(trimmed)) return null; - const parsed = Number(trimmed); - return Number.isSafeInteger(parsed) ? parsed : null; -} - -function SectionHeading({ - icon: Icon, - title, - description, - status, - statusTone = "neutral", -}: { - icon: typeof Languages; - title: string; - description: string; - status: string; - statusTone?: "ready" | "warning" | "neutral"; -}) { - return ( -
-
-
- -
-
-

{title}

-

- {description} -

-
-
- - {statusTone === "ready" ? ( - - ) : statusTone === "warning" ? ( - - ) : null} - {status} - -
- ); -} - -function RequirementNote({ - label, - ready, - detail, -}: { - label: string; - ready: boolean; - detail: string; -}) { - return ( -
- {ready ? ( - - ) : ( - - )} - - {label} - {detail} - -
- ); -} - -export default function AIServicesSettings() { - const form = useSettingsForm({ keys: KEYS }); - const textCheck = useCheckAdminSettingsConnection(); - const speechCheck = useCheckAdminSettingsConnection(); - const [textResult, setTextResult] = useState(null); - const [speechResult, setSpeechResult] = useState(null); - - if (form.isLoading) { - return ( -
- - - - - Loading AI settings -
- ); - } - - const value = (key: string, fallback = "") => form.getValue(key) || fallback; - const effectiveValue = (key: string, legacyKey: string, fallback: string) => - value(key, value(legacyKey, fallback)); - const textBaseURL = effectiveValue( - "ai.base_url", - "subtitle_ai.base_url", - "https://api.openai.com", - ); - const chatModel = effectiveValue("ai.chat_model", "subtitle_ai.chat_model", "gpt-4o-mini"); - const asrBaseURL = value("ai.asr_base_url"); - const asrModel = value("ai.asr_model", "whisper-1"); - const textReady = textBaseURL.trim() !== "" && chatModel.trim() !== ""; - const speechUsesTextEndpoint = asrBaseURL.trim() === ""; - const speechCheckable = - (asrBaseURL.trim() !== "" || textBaseURL.trim() !== "") && asrModel.trim() !== ""; - const speechCompatible = !isChatOnlyGateway(speechUsesTextEndpoint ? textBaseURL : asrBaseURL); - const speechReady = speechCheckable && speechCompatible; - const descriptionEnabled = value("metadata_ai.enabled", "false") === "true"; - - function setValue(key: string, nextValue: string) { - form.setValue(key, nextValue); - if (TEXT_AI_KEYS.includes(key as (typeof TEXT_AI_KEYS)[number])) { - setTextResult(null); - } - if (SPEECH_AI_KEYS.includes(key as (typeof SPEECH_AI_KEYS)[number])) { - setSpeechResult(null); - } - } - - async function checkTextConnection() { - try { - setTextResult( - await textCheck.mutateAsync({ - kind: "ai_chat", - body: form.buildConnectionCheckRequest([...TEXT_AI_KEYS]), - }), - ); - } catch (error) { - setTextResult({ - success: false, - message: error instanceof Error ? error.message : "Text AI connection check failed.", - }); - } - } - - async function checkSpeechConnection() { - try { - setSpeechResult( - await speechCheck.mutateAsync({ - kind: "ai_transcription", - body: form.buildConnectionCheckRequest([...SPEECH_AI_KEYS]), - }), - ); - } catch (error) { - setSpeechResult({ - success: false, - message: error instanceof Error ? error.message : "Speech-to-text connection check failed.", - }); - } - } - - async function save() { - const batchSize = parseStrictInteger(value("subtitle_ai.batch_size", "40")); - const contextLines = parseStrictInteger(value("subtitle_ai.context_neighbors", "2")); - const chunkSeconds = parseStrictInteger(value("subtitle_ai.asr_chunk_seconds", "600")); - const quotaJobs = Number.parseInt(value("subtitle_ai.transcribe_quota_jobs", "0"), 10); - const maxConcurrent = parseStrictInteger( - effectiveValue("ai.max_concurrent_jobs", "subtitle_ai.max_concurrent_jobs", "2"), - ); - - if (!textReady) { - toast.error("Text AI base URL and chat model are required."); - return; - } - if (maxConcurrent === null || maxConcurrent < 1) { - toast.error("Max concurrent jobs must be a positive whole number."); - return; - } - if (batchSize === null || batchSize < 1) { - toast.error("Subtitle batch size must be a positive whole number."); - return; - } - if (contextLines === null || contextLines < 0) { - toast.error("Subtitle context lines must be zero or a positive whole number."); - return; - } - if (chunkSeconds === null || chunkSeconds < 60 || chunkSeconds > 600) { - toast.error("Transcription chunk length must be between 60 and 600 seconds."); - return; - } - if (!Number.isInteger(quotaJobs) || quotaJobs < 0) { - toast.error("Transcription limit must be zero or a positive whole number."); - return; - } - await form.save(); - } - - function discard() { - form.discard(); - setTextResult(null); - setSpeechResult(null); - } - - return ( -
-
-

AI Services

-

- Configure text translation and speech-to-text independently, then enable only the features - that use them. -

-
- -
-
- -
- setValue("ai.base_url", next)} - hint="https://api.openai.com" - /> - setValue("ai.chat_model", next)} - hint="gpt-4o-mini, gemini-flash-latest, llama3.1" - /> - setValue("ai.api_key", next)} - sensitiveConfigured={ - form.sensitiveConfigured.includes("ai.api_key") || - form.sensitiveConfigured.includes("subtitle_ai.api_key") - } - hint="Optional for keyless local endpoints. Saved keys are reused for tests only when the endpoint host is unchanged." - /> - -
-
- -
- -
-
- {TRANSCRIPTION_PRESETS.map((preset) => { - const active = asrBaseURL === preset.baseUrl && asrModel === preset.model; - return ( - - ); - })} -
- setValue("ai.asr_base_url", next)} - hint="http://speaches:8000 or https://api.groq.com/openai" - /> - {speechUsesTextEndpoint && ( -
- - - Uses the Text translation endpoint and API key. This only works when that provider - implements OpenAI-compatible /audio/transcriptions with timestamped - segments. Test it before enabling audio generation. - -
- )} - setValue("ai.asr_model", next)} - hint="whisper-large-v3-turbo or whisper-1" - /> - setValue("ai.asr_api_key", next)} - sensitiveConfigured={form.sensitiveConfigured.includes("ai.asr_api_key")} - hint="Optional. A saved or inherited key is reused for tests only when the endpoint host is unchanged." - /> -

- For self-hosted services, use a hostname or IP reachable from the Silo container. - localhost - points back to Silo itself. -

- -
-
- -
-
-

Features

-

- Generated subtitles and translated metadata are saved once and served to every client - through Silo's normal pipelines. -

-
- -
-
- setValue("subtitle_ai.enabled", next)} - hint="Text AI required - Translates an existing text subtitle track. Whisper is not used." - /> - -
-
- setValue("subtitle_ai.transcribe_enabled", next)} - hint="Speech-to-text required - Uses Whisper to create timed subtitles from the selected audio track." - /> - -
-
- setValue("metadata_ai.enabled", next)} - hint="Text AI required - Translates overviews and taglines from the metadata editor or library refresh." - /> - -
-
- setValue("metadata_ai.on_view", next)} - disabled={!descriptionEnabled} - options={[ - { value: "off", label: "Off" }, - { value: "button", label: "Translate button on detail pages" }, - { value: "auto", label: "Automatic on view" }, - ]} - hint={ - descriptionEnabled - ? "Controls viewer-triggered description translation." - : "Inactive until Description translation is enabled." - } - /> -
-
-
- -
-
- -
-

Advanced

-

- Job concurrency, translation batching, transcription chunks, and account quotas. -

-
- - -
- setValue("ai.max_concurrent_jobs", next)} - hint="Shared by subtitle translation, speech-to-text, and description translation. Changing this value requires a server restart." - /> - setValue("subtitle_ai.batch_size", next)} - hint="Text cues sent in each translation request." - /> - setValue("subtitle_ai.context_neighbors", next)} - hint="Previous source cues included for scene continuity." - /> - setValue("subtitle_ai.asr_chunk_seconds", next)} - hint="60-600. Shorter chunks reduce timestamp drift but make more requests." - /> - setValue("subtitle_ai.transcribe_quota_jobs", next)} - hint="0 = unlimited. Profiles share their account's limit." - /> - setValue("subtitle_ai.transcribe_quota_period", next)} - options={QUOTA_PERIODS.map((period) => ({ - value: period, - label: `Per ${period} (rolling ${QUOTA_PERIOD_WINDOW_LABELS[period]})`, - }))} - hint="Rolling window used for the account limit." - /> -
-
-
-
- - - - void save()} - onDiscard={discard} - isSaving={form.isSaving} - restartRequired={form.restartRequired} - /> -
- ); -} diff --git a/web/src/pages/admin-settings/AdminSettingsLayout.test.tsx b/web/src/pages/admin-settings/AdminSettingsLayout.test.tsx index 863554108..628e0af67 100644 --- a/web/src/pages/admin-settings/AdminSettingsLayout.test.tsx +++ b/web/src/pages/admin-settings/AdminSettingsLayout.test.tsx @@ -16,10 +16,21 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/hooks/useSettingsForm", () => ({ useSettingsForm: () => ({ isLoading: true, - dirtyCount: 0, getValue: () => "", + setValue: () => {}, + resetValue: () => {}, + dirtyCount: 0, + dirtyKeys: [], + isDirty: () => false, + save: () => {}, + discard: () => {}, + isSaving: false, + restartRequired: false, sensitiveConfigured: [], sensitiveManagedByEnv: [], + sensitiveStatusReady: false, + sensitiveStatusError: null, + buildConnectionCheckRequest: () => ({}), }), })); @@ -64,7 +75,7 @@ describe("AdminSettingsLayout", () => { it("renders the grouped navigation sections", () => { const markup = renderLayout(); - for (const group of ["Server", "Media", "Connections", "Data"]) { + for (const group of ["Server", "Connections & Data"]) { expect(markup).toContain(`>${group}<`); } }); @@ -77,7 +88,7 @@ describe("AdminSettingsLayout", () => { expect( screen.queryByRole("navigation", { name: "Admin settings sections categories" }), ).not.toBeInTheDocument(); - for (const group of ["Server", "Media", "Connections", "Data"]) { + for (const group of ["Server", "Connections & Data"]) { expect(screen.getAllByRole("heading", { name: group })).toHaveLength(1); expect( screen.queryByRole("link", { name: new RegExp(`^${group}, \\d+ settings`) }), @@ -88,10 +99,10 @@ describe("AdminSettingsLayout", () => { it("uses one desktop grid and card geometry for every settings group", () => { const markup = renderLayout(); - expect(markup.match(/2xl:grid-cols-4/g)).toHaveLength(4); + expect(markup.match(/2xl:grid-cols-4/g)).toHaveLength(2); expect(markup).not.toContain("2xl:grid-cols-3"); - expect(markup.match(/lg:h-28/g)).toHaveLength(20); - expect(markup.match(/lg:line-clamp-3/g)).toHaveLength(20); + expect(markup.match(/lg:h-28/g)).toHaveLength(9); + expect(markup.match(/lg:line-clamp-3/g)).toHaveLength(9); }); it("renders every settings tab", () => { @@ -99,25 +110,14 @@ describe("AdminSettingsLayout", () => { for (const label of [ "General", - "Branding", - "Theming", - "Card Overlays", - "Scanner & Matcher", - "Search", - "Intro Markers", - "Subtitles", - "AI Services", + "Appearance", + "Security & Access", + "Library & Metadata", "Playback", - "Downloads", - "Watch Providers", "Integrations", - "Email", "Notifications", - "Compatibility Proxies", - "Rate Limiting", - "Database", - "Storage", - "Log Retention", + "Compatibility", + "Infrastructure", ]) { expect(markup).toContain(label); } @@ -126,7 +126,7 @@ describe("AdminSettingsLayout", () => { it("renders the settings index at the root and preserves tab deep links", () => { renderInteractiveLayout(); - expect(screen.getByRole("link", { name: /General.*Authentication/ })).toHaveAttribute( + expect(screen.getByRole("link", { name: /General.*Server identity/ })).toHaveAttribute( "href", "/admin/settings?tab=general", ); @@ -142,9 +142,11 @@ describe("AdminSettingsLayout", () => { vi.stubGlobal("scrollTo", scrollTo); renderInteractiveLayout(); - await userEvent.click(screen.getByRole("link", { name: /Database.*Postgres/ })); + await userEvent.click(screen.getByRole("link", { name: /Infrastructure.*Redis/ })); - const detailRegion = await screen.findByRole("region", { name: "Database settings" }); + const detailRegion = await screen.findByRole("region", { + name: "Infrastructure settings", + }); expect(scrollTo).toHaveBeenCalledWith(0, 0); expect(detailRegion).toHaveFocus(); }); @@ -152,9 +154,9 @@ describe("AdminSettingsLayout", () => { it("adds a mobile detail heading when the settings component has none", () => { vi.stubGlobal("scrollTo", vi.fn()); - renderInteractiveLayout("?tab=branding"); + renderInteractiveLayout("?tab=appearance"); - expect(screen.getByRole("heading", { name: "Branding", level: 2 })).toHaveFocus(); + expect(screen.getByRole("heading", { name: "Appearance", level: 2 })).toHaveFocus(); }); it("resets the scrolling detail pane when switching admin tabs", async () => { @@ -164,11 +166,13 @@ describe("AdminSettingsLayout", () => { const generalRegion = screen.getByRole("region", { name: "General settings" }); generalRegion.scrollTop = 400; - await userEvent.click(screen.getByRole("button", { name: "Database" })); + await userEvent.click(screen.getByRole("button", { name: /Infrastructure/ })); - const databaseRegion = await screen.findByRole("region", { name: "Database settings" }); - expect(databaseRegion.scrollTop).toBe(0); - expect(databaseRegion).toHaveFocus(); + const infrastructureRegion = await screen.findByRole("region", { + name: "Infrastructure settings", + }); + expect(infrastructureRegion.scrollTop).toBe(0); + expect(infrastructureRegion).toHaveFocus(); }); it("surfaces durable restart-required state above the active tab", () => { @@ -179,11 +183,38 @@ describe("AdminSettingsLayout", () => { expect(markup).toContain("Server restart required for saved settings to take effect."); }); - it("resolves the legacy jellyfin tab alias to Compatibility Proxies", () => { - const withAlias = renderLayout("?tab=jellyfin"); - const direct = renderLayout("?tab=compatibility-proxies"); + it("resolves every legacy tab id to the tab that absorbed it", () => { + const aliases: Record = { + jellyfin: "compatibility", + "compatibility-proxies": "compatibility", + branding: "appearance", + theming: "appearance", + overlays: "appearance", + "rate-limiting": "security", + scanner: "library", + search: "library", + intro: "library", + subtitles: "integrations", + ai: "integrations", + "watch-providers": "integrations", + downloads: "playback", + email: "notifications", + database: "infrastructure", + storage: "infrastructure", + "log-retention": "infrastructure", + }; + + for (const [legacy, current] of Object.entries(aliases)) { + expect(renderLayout(`?tab=${legacy}`)).toBe(renderLayout(`?tab=${current}`)); + } + }); + + it("badges Infrastructure as advanced in the settings nav", () => { + vi.stubGlobal("scrollTo", vi.fn()); - expect(withAlias).toBe(direct); + const markup = renderLayout("?tab=general"); + + expect(markup).toContain(">Advanced<"); }); it("filters admin settings sections from the search box", async () => { @@ -191,7 +222,7 @@ describe("AdminSettingsLayout", () => { await userEvent.type(screen.getByRole("searchbox", { name: "Search settings" }), "redis"); - expect(screen.getAllByRole("link", { name: /Database/ })).toHaveLength(1); + expect(screen.getAllByRole("link", { name: /Infrastructure/ })).toHaveLength(1); expect(screen.queryByRole("link", { name: /Playback/ })).not.toBeInTheDocument(); expect(screen.getByText("1 match")).toBeInTheDocument(); }); @@ -201,11 +232,11 @@ describe("AdminSettingsLayout", () => { await userEvent.type( screen.getByRole("searchbox", { name: "Search settings" }), - "pool max open", + "silenced log messages", ); - expect(screen.getAllByRole("link", { name: /Database/ })).toHaveLength(1); - expect(screen.queryByRole("link", { name: /General/ })).not.toBeInTheDocument(); + expect(screen.getAllByRole("link", { name: /General/ })).toHaveLength(1); + expect(screen.queryByRole("link", { name: /Playback/ })).not.toBeInTheDocument(); }); it("focuses admin settings search with Cmd+K", () => { diff --git a/web/src/pages/admin-settings/AdminSettingsLayout.tsx b/web/src/pages/admin-settings/AdminSettingsLayout.tsx index 958cfb009..59d07d303 100644 --- a/web/src/pages/admin-settings/AdminSettingsLayout.tsx +++ b/web/src/pages/admin-settings/AdminSettingsLayout.tsx @@ -12,31 +12,21 @@ import { import { ADMIN_SETTINGS_GROUPS, ADMIN_SETTINGS_NAV, + resolveAdminSettingsTabID, type AdminSettingsSearchItem, } from "@/lib/adminSettingsSearch"; import { cn } from "@/lib/utils"; import { useAdminServerStatus } from "@/hooks/queries/admin/settings"; -import EmailSettings from "./EmailSettings"; -import NotificationsAdminSettings from "./NotificationsAdminSettings"; import GeneralSettings from "./GeneralSettings"; +import AppearanceSettings from "./AppearanceSettings"; +import SecurityAccessSettings from "./SecurityAccessSettings"; +import LibraryMetadataSettings from "./LibraryMetadataSettings"; import PlaybackSettings from "./PlaybackSettings"; -import ScannerSettings from "./ScannerSettings"; -import SearchSettings from "./SearchSettings"; -import IntroSettings from "./IntroSettings"; -import SubtitlesSettings from "./SubtitlesSettings"; -import AIServicesSettings from "./AIServicesSettings"; -import RateLimitSettings from "./RateLimitSettings"; -import WatchProvidersSettings from "./WatchProvidersSettings"; import IntegrationsSettings from "./IntegrationsSettings"; +import NotificationsAdminSettings from "./NotificationsAdminSettings"; import CompatibilityProxiesSettings from "./CompatibilityProxiesSettings"; -import DatabaseSettings from "./DatabaseSettings"; -import StorageSettings from "./StorageSettings"; -import DownloadSettings from "./DownloadSettings"; -import LogRetentionSettings from "./LogRetentionSettings"; -import ThemeSettings from "./ThemeSettings"; -import BrandingSettings from "./BrandingSettings"; -import OverlaySettings from "./OverlaySettings"; +import InfrastructureSettings from "./InfrastructureSettings"; import { RestartServerButton } from "./RestartServerButton"; interface SettingsNav extends AdminSettingsSearchItem { @@ -50,25 +40,14 @@ interface SettingsNavGroup { const SETTINGS_COMPONENTS: Record = { general: GeneralSettings, - branding: BrandingSettings, - theming: ThemeSettings, - overlays: OverlaySettings, - scanner: ScannerSettings, - search: SearchSettings, - intro: IntroSettings, - subtitles: SubtitlesSettings, - ai: AIServicesSettings, + appearance: AppearanceSettings, + security: SecurityAccessSettings, + library: LibraryMetadataSettings, playback: PlaybackSettings, - downloads: DownloadSettings, - "watch-providers": WatchProvidersSettings, integrations: IntegrationsSettings, - email: EmailSettings, notifications: NotificationsAdminSettings, - "compatibility-proxies": CompatibilityProxiesSettings, - "rate-limiting": RateLimitSettings, - database: DatabaseSettings, - storage: StorageSettings, - "log-retention": LogRetentionSettings, + compatibility: CompatibilityProxiesSettings, + infrastructure: InfrastructureSettings, }; function settingsComponent(id: string) { @@ -89,7 +68,9 @@ const SETTINGS_NAV: SettingsNav[] = ADMIN_SETTINGS_NAV.map((item) => ({ component: settingsComponent(item.id), })); -const SHELL_HEADING_SETTINGS = new Set(["branding", "theming"]); +// Tabs whose component renders no heading of its own, so the shell supplies +// the mobile-visible one. +const SHELL_HEADING_SETTINGS = new Set(["appearance"]); export default function AdminSettingsLayout() { const [searchParams, setSearchParams] = useSearchParams(); @@ -98,7 +79,7 @@ export default function AdminSettingsLayout() { const activeHeadingRef = useRef(null); const { data: serverStatus } = useAdminServerStatus(); const rawActiveId = searchParams.get("tab"); - const activeId = rawActiveId === "jellyfin" ? "compatibility-proxies" : rawActiveId; + const activeId = resolveAdminSettingsTabID(rawActiveId); const filteredSettingsGroups = useMemo( () => filterSettingsSearchGroups(SETTINGS_GROUPS, settingsSearch), [settingsSearch], @@ -125,6 +106,13 @@ export default function AdminSettingsLayout() { const active = activeId ? SETTINGS_NAV.find((item) => item.id === activeId) : undefined; const ActiveComponent = active?.component; + // Rewrite a legacy `?tab=` id to the tab that absorbed it so the address bar, + // and anything the admin copies out of it, names a tab that still exists. + useEffect(() => { + if (!activeId || activeId === rawActiveId) return; + setSearchParams({ tab: activeId }, { replace: true }); + }, [activeId, rawActiveId, setSearchParams]); + useEffect(() => { if (!active) return; @@ -193,6 +181,13 @@ export default function AdminSettingsLayout() { label={item.label} icon={item.icon} active={item.id === active.id} + badge={ + item.badge ? ( + + {item.badge} + + ) : undefined + } onClick={() => setActiveId(item.id)} /> ))} diff --git a/web/src/pages/admin-settings/AppearanceSettings.test.tsx b/web/src/pages/admin-settings/AppearanceSettings.test.tsx new file mode 100644 index 000000000..aeb478a37 --- /dev/null +++ b/web/src/pages/admin-settings/AppearanceSettings.test.tsx @@ -0,0 +1,153 @@ +// @vitest-environment jsdom + +import { fireEvent, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const useSettingsFormMock = vi.fn(); + +vi.mock("@/hooks/useSettingsForm", () => ({ + useSettingsForm: (...args: unknown[]) => useSettingsFormMock(...args), +})); + +vi.mock("@/hooks/useRestartKeys", () => ({ + useRestartKeys: () => new Set(), +})); + +vi.mock("@/hooks/useBranding", () => ({ + useBranding: () => ({ + storageAvailable: true, + wordmarkUrl: null, + markUrl: null, + faviconUrl: null, + loginBgUrl: null, + }), +})); + +vi.mock("@/components/admin/BrandingAssetField", () => ({ + BrandingAssetField: ({ label }: { label: string }) =>
{label}
, +})); + +vi.mock("@/components/theme/TokenEditor", () => ({ + TokenEditor: ({ onSetVar }: { onSetVar: (token: "primary", value: string) => void }) => ( + + ), +})); + +vi.mock("@/components/theme/RawCssEditor", () => ({ + RawCssEditor: ({ value, onChange }: { value: string; onChange: (css: string) => void }) => ( +