diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..9ad62dc80 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,128 @@ +name: Native macOS Server + +on: + # Temporarily manual-only while GitHub's macOS runners cannot reliably + # create the VideoToolbox hardware compression session exercised below. + workflow_dispatch: + +concurrency: + group: macos-${{ github.ref }} + cancel-in-progress: true + +env: + GOPROXY: https://proxy.golang.org,direct + GOPRIVATE: github.com/Silo-Server/* + GONOSUMDB: github.com/Silo-Server/* + GOWORK: off + +permissions: + contents: read + +jobs: + build: + name: Build Darwin arm64 + runs-on: macos-15 + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install native dependencies + run: brew install vips ffmpeg + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + package_json_file: web/package.json + + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: web/pnpm-lock.yaml + + - name: Install frontend dependencies + working-directory: web + run: pnpm install --frozen-lockfile + + - name: Build frontend assets + working-directory: web + run: pnpm run build + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Validate SDK resolves from the module graph + run: | + sdk_module_json="$(mktemp)" + go list -m -json github.com/Silo-Server/silo-plugin-sdk > "$sdk_module_json" + if grep -q '"Replace":' "$sdk_module_json"; then + echo "silo-plugin-sdk resolved through a module replacement." + cat "$sdk_module_json" + exit 1 + fi + if grep -q '"Main": true' "$sdk_module_json"; then + echo "silo-plugin-sdk resolved from the local workspace instead of go.mod." + cat "$sdk_module_json" + exit 1 + fi + + - name: Test Go on macOS + run: go test ./... + + - name: Build native server archive + id: build + env: + BUILDINFO_PKG: github.com/Silo-Server/silo-server/internal/buildinfo + run: | + built_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + archive="silo-darwin-arm64-${GITHUB_SHA::8}.tar.gz" + package_dir="dist/silo-darwin-arm64" + mkdir -p "$package_dir" + + go build -trimpath \ + -ldflags "-X ${BUILDINFO_PKG}.revisionOverride=${GITHUB_SHA} -X ${BUILDINFO_PKG}.dirtyOverride=false -X ${BUILDINFO_PKG}.builtAtOverride=${built_at}" \ + -o "$package_dir/silo" ./cmd/silo/ + + cp LICENSE "$package_dir/LICENSE" + go version -m "$package_dir/silo" > "$package_dir/build-info.txt" + otool -L "$package_dir/silo" > "$package_dir/linked-libraries.txt" + tar -czf "dist/$archive" -C dist silo-darwin-arm64 + (cd dist && shasum -a 256 "$archive" > "$archive.sha256") + echo "archive=$archive" >> "$GITHUB_OUTPUT" + + - name: Verify native runtime and VideoToolbox + env: + ARCHIVE: ${{ steps.build.outputs.archive }} + run: | + file dist/silo-darwin-arm64/silo | grep -q 'Mach-O 64-bit executable arm64' + dist/silo-darwin-arm64/silo -h >/dev/null + grep -aFq "$GITHUB_SHA" dist/silo-darwin-arm64/silo + + ffmpeg -hide_banner -hwaccels > "$RUNNER_TEMP/ffmpeg-hwaccels.txt" 2>/dev/null + grep -Fxq 'videotoolbox' "$RUNNER_TEMP/ffmpeg-hwaccels.txt" + ffmpeg -hide_banner -encoders > "$RUNNER_TEMP/ffmpeg-encoders.txt" 2>/dev/null + grep -q 'h264_videotoolbox' "$RUNNER_TEMP/ffmpeg-encoders.txt" + ffmpeg -hide_banner -loglevel error \ + -f lavfi -i color=c=black:s=128x72:r=24 \ + -frames:v 12 -c:v h264_videotoolbox -f null - + + (cd dist && shasum -a 256 -c "$ARCHIVE.sha256") + + - name: Upload native server + uses: actions/upload-artifact@v4 + with: + name: silo-darwin-arm64-${{ github.sha }} + path: | + dist/${{ steps.build.outputs.archive }} + dist/${{ steps.build.outputs.archive }}.sha256 + if-no-files-found: error + retention-days: 14 diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 3ca06c805..3924e8fd9 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -16,6 +16,7 @@ import ( "os/signal" "path/filepath" "runtime/debug" + "slices" "sort" "strconv" "strings" @@ -53,6 +54,7 @@ import ( "github.com/Silo-Server/silo-server/internal/chapterthumbs" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/dashmetrics" "github.com/Silo-Server/silo-server/internal/database" "github.com/Silo-Server/silo-server/internal/diagnostics" "github.com/Silo-Server/silo-server/internal/downloads" @@ -80,6 +82,7 @@ import ( _ "github.com/Silo-Server/silo-server/internal/metadata/nfo" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/nodesessions" @@ -138,6 +141,146 @@ func resolveNodeIdentity() string { return h } +// nodeCapabilityFetcher adapts the authenticated node capability client to the +// node health sweep, which stores capability reports opaquely. +// +// What is persisted is the node's own response bytes, not this server's +// re-marshaling of the decoded struct. The two differ exactly when the node is +// newer than the API server reading it, which is every rolling upgrade: a +// re-marshal drops the fields this build has no struct member for, and stores +// the truncation under the node's hash. After the API is upgraded the sweep +// then sees the hashes agree and never refetches, leaving the durable inventory +// permanently missing fields the new code reads. The bytes are bounded and +// already parsed by the client, so storing them verbatim costs nothing but +// keeps the payload honest about the hash filed with it. +// +// The hash comes out of the payload itself, because only the node knows what it +// hashed. A report without one is refused rather than given a synthetic hash, +// which would make an old node look like it had capability tracking. +// The request bound comes from budget rather than a constant. A cold node's +// answer runs the whole FFmpeg probe matrix, and the size of that matrix is a +// function of how many hardware devices that node will actually probe — two +// render devices legitimately push the advertised budget past two minutes. +// Bounding every fetch at a fixed two minutes abandons such a node mid-probe +// and reports a fetch failure for a node operating inside its published +// contract. budget takes the node because the device set is a per-node +// property: an hw_device override is exactly a node that probes a different +// matrix from the one the cluster setting describes. +func nodeCapabilityFetcher(jwtSecret string, budget func(*nodepool.Node) time.Duration) nodepool.CapabilityFetcher { + client := &http.Client{} + return func(ctx context.Context, node *nodepool.Node) ([]byte, string, error) { + if node == nil { + return nil, "", fmt.Errorf("node capability request has no node") + } + ctx, cancel := context.WithTimeout(ctx, budget(node)) + defer cancel() + info, payload, status, err := transcodenode.FetchHWCapabilitiesPayload(ctx, client, node.URL, jwtSecret) + if err != nil { + return nil, "", err + } + if status != http.StatusOK { + return nil, "", fmt.Errorf("node capability request returned status %d", status) + } + return payload, info.CapabilityHash, nil + } +} + +// libraryPathProvider adapts the folder repository to the sampler's media-root +// provider. +// +// It runs on the sampling goroutine every interval, so it is bounded separately +// from the caller's context: a database that has stopped answering must cost one +// interval's disk sampling, not the sampler. +func libraryPathProvider(repo *catalog.FolderRepository) func(context.Context) []string { + if repo == nil { + return nil + } + return cachedLibraryPaths(func(ctx context.Context) ([]string, error) { + queryCtx, cancel := context.WithTimeout(ctx, libraryPathQueryTimeout) + defer cancel() + return repo.DistinctLibraryPaths(queryCtx) + }) +} + +// cachedLibraryPaths wraps a library-root query so a failed read reuses the last +// set that succeeded. +// +// Returning nothing on error is not the harmless degradation it looks like. The +// sampler treats the returned set as the whole truth: refreshDisks prunes every +// path outside it — dropping the cached capacity readings with it — and +// diskStats omits them from the sample. A two-second database hiccup would +// therefore blank every library mount from the admin resource panel and from +// Prometheus, and leave the next pass reporting them unavailable until fresh +// probes land, all while the mounts themselves are perfectly healthy. Only the +// query failed, so the previous answer is still the best one available. +// +// An empty result that the database actually returned is cached like any other: +// an operator who removed their last library has genuinely no roots. +func cachedLibraryPaths(query func(context.Context) ([]string, error)) func(context.Context) []string { + var ( + mu sync.Mutex + lastGood []string + ) + return func(ctx context.Context) []string { + paths, err := query(ctx) + mu.Lock() + defer mu.Unlock() + if err != nil { + slog.DebugContext(ctx, "library paths unavailable for resource sampling; reusing the last known set", + "component", "app", "error", err, "roots", len(lastGood)) + return slices.Clone(lastGood) + } + lastGood = slices.Clone(paths) + return paths + } +} + +// libraryPathQueryTimeout bounds the per-sample library root lookup. +const libraryPathQueryTimeout = 2 * time.Second + +// nodeCapabilityRequestTimeout is the floor under one capability request. +// +// The real bound is the node's own advertised probe budget, which +// nodeCapabilityProbeBudget computes from the configured hardware — that grows +// with the device count and passes two minutes at two devices. This floor covers +// the rest of what a fetch does (transport, a node answering from a warm cache +// but under load) and keeps a misconfigured or unreadable setting from +// producing an absurdly small bound. The fetch runs detached from the health +// sweep, so a generous bound costs the sweep nothing and lets a cold node's +// first report land instead of timing out. +const nodeCapabilityRequestTimeout = 2 * time.Minute + +// nodeCapabilityProbeBudget reports how long to allow one node's capability +// fetch. +// +// The cluster settings are read live on every call, so an operator adding a +// second render device does not have to restart the API for its fetches to stop +// being cut short. The node's own overrides win over them, because the worker +// builds its probe matrix from the policy it will actually run: a node that +// overrides hw_device with two devices needs the two-device budget even on a +// cluster configured with one, and it is precisely that node whose inventory +// would otherwise fail to refresh every sweep. +func nodeCapabilityProbeBudget(live func() *config.Config) func(*nodepool.Node) time.Duration { + return func(node *nodepool.Node) time.Duration { + hwAccel, hwDevice := "", "" + if live != nil { + if current := live(); current != nil { + hwAccel, hwDevice = current.Playback.HWAccel, current.Playback.HWDevice + } + } + // The same ladder every other caller prices a node's capability read + // with — its own advertisement, then its own policy, then this floor — + // so the sweep, the re-probe, and the two planning paths cannot end up + // allowing different amounts of time for one node's matrix. + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(hwAccel), + node.EffectiveHWDevice(hwDevice), + nodeCapabilityRequestTimeout, + ) + } +} + func clientIPResolverFromConfig(cfg *config.Config) (*clientip.Resolver, error) { if cfg == nil { return nil, fmt.Errorf("config is not loaded") @@ -793,12 +936,32 @@ func main() { streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, redisClient) streamTelemetryRegistry.Start(appCtx) + // Resolved before the watcher starts: NODE_URL is this process's + // stream_nodes identity, and the watcher needs it on its very first + // load to overlay the node's own acceleration overrides. + nodeURL := os.Getenv("NODE_URL") + nodeName := os.Getenv("NODE_NAME") + if nodeURL == "" { + nodeURL = "http://localhost" + cfg.Server.Listen + // The guess is this process's whole identity: it keys session + // bookkeeping *and* selects the stream_nodes row whose acceleration + // overrides this node adopts. Two nodes listening on the same port + // guess the same URL and would share both. + slog.Warn("NODE_URL not set, using listen address — session keys may collide across nodes, and this node adopts the acceleration overrides of whichever stream_nodes row carries that URL", + "node_url", nodeURL) + } + if nodeName == "" { + nodeName = mode + } + bootstrap := nodeconfig.BootstrapOverrides{ Listen: cfg.Server.Listen, Mode: cfg.Server.Mode, DatabaseURL: cfg.Database.URL, JFListen: cfg.JellyfinCompat.Listen, RedisURL: bc.RedisURL, + NodeURL: nodeURL, + NodeName: nodeName, } watcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, bootstrap) watcher.OnLoad(normalizeLoadedConfig) @@ -807,16 +970,6 @@ func main() { os.Exit(1) } - nodeURL := os.Getenv("NODE_URL") - nodeName := os.Getenv("NODE_NAME") - if nodeURL == "" { - nodeURL = "http://localhost" + cfg.Server.Listen - slog.Warn("NODE_URL not set, using listen address — session keys may collide across nodes") - } - if nodeName == "" { - nodeName = mode - } - tracker := nodesessions.NewTracker(redisClient, nodeURL, nodeName, mode) tracker.StartRefresh(appCtx) defer func() { @@ -856,6 +1009,12 @@ func main() { watcher.Config, nil, )) + // Keep the capability hash /health advertises current, so the API's + // sweep refetches this proxy's inventory only when it actually changed. + srv.StartCapabilitySnapshots(appCtx) + // Sample host and GPU resources so /health, /status and /metrics can + // report them without doing any work on the request. + srv.StartMetricsSampler(appCtx) handler = srv.Handler() } else { srv := transcodenode.NewServer(watcher, tracker) @@ -866,7 +1025,12 @@ func main() { // restart (the node hop token is recipe-less). Shares the offload Redis. srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0)) srv.SetStreamTelemetry(streamTelemetryRegistry) - srv.StartHardwareEncoderWarmup(appCtx) + // The first capability snapshot waits on warmup so it measures a + // primed encoder rather than racing it into a probe failure. + srv.StartCapabilitySnapshots(appCtx, srv.StartHardwareEncoderWarmup(appCtx)) + // Sample host and GPU resources so /health, /status and /metrics can + // report them without doing any work on the request. + srv.StartMetricsSampler(appCtx) // Reclaim orphaned transcode dirs at boot and hourly thereafter, bound // to appCtx so it stops on shutdown. srv.StartOrphanSweeper(appCtx) @@ -882,7 +1046,9 @@ func main() { // Hot-reload config watcher for integrated/api mode. Reloads on // EventSettingsChanged (Redis) with a 60s poll fallback, so settings // changes apply without restart even on Redis-less deployments. The - // watcher's config supersedes the startup snapshot from here on. + // watcher's config supersedes the startup snapshot from here on. No + // NodeURL: an API host is not a stream node and has no row whose + // acceleration overrides could apply to it. configWatcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, nodeconfig.BootstrapOverrides{ Listen: bc.Listen, Mode: bc.Mode, @@ -969,6 +1135,7 @@ func main() { OpsLogRepo: opsRepo, FFmpegLogSink: playback.NewSlogFFmpegLogSink(slog.Default(), nodeID), PublicURL: os.Getenv("SILO_PUBLIC_URL"), + CatalogSearchSettings: new(catalogSearchStartupSettings), RequestServerRestart: func(context.Context) error { if !restartRequested.CompareAndSwap(false, true) { return handlers.ErrServerRestartAlreadyRequested @@ -1060,6 +1227,7 @@ func main() { watchProviderRepo = watchsync.NewPostgresRepository(deps.DB, deps.SecretCipher) watchProviderService = watchsync.NewService(watchProviderRepo, watchProviderRegistry) deps.WatchProviderService = watchProviderService + deps.WatchProviderRegistry = watchProviderRegistry } // Initialize node pools for integrated/api modes. @@ -1086,9 +1254,39 @@ func main() { deps.NodePlanner = nodepool.NewPlanner(proxyPool, transcodePool) healthChecker := nodepool.NewHealthChecker(proxyPool, transcodePool, nodeRepo) + capabilityBudget := nodeCapabilityProbeBudget(configWatcher.Config) + healthChecker.SetCapabilityFetcher(nodeCapabilityFetcher(cfg.Auth.JWTSecret, capabilityBudget)) + // The sweep's backstop is derived from the same budget, so it can never + // be the deadline that fires first on a node with many devices. + healthChecker.SetCapabilityFetchBudget(capabilityBudget) + deps.NodeHealthChecker = healthChecker healthChecker.Start(appCtx) slog.Info("node pools initialized", "proxy_nodes", len(proxyNodes), "transcode_nodes", len(transcodeNodes)) + // The API host runs the same sampler the nodes do. It is not a + // registered stream node, so without this the machine serving admin + // requests — and, in integrated mode, transcoding — is the one host with + // no resource visibility. Unlike a node it also samples library roots: + // it is the process that knows what the library is, and its own view of + // a media mount is the one that is authoritative. + resourceSampler := nodemetrics.NewSampler(nodemetrics.Options{ + // Read per sample rather than captured: in integrated mode this host + // is the one transcoding, and playback.transcode_dir is + // hot-reloadable. A startup snapshot would keep reporting headroom on + // the volume it used to write to while the new one fills unwatched. + ScratchDir: func() string { + if live := configWatcher.Config(); live != nil { + return live.Playback.TranscodeDir + } + return cfg.Playback.TranscodeDir + }, + MediaRoots: libraryPathProvider(catalog.NewFolderRepository(pool)), + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: playback.SamplerDeviceIdentities, + }) + resourceSampler.Start(appCtx) + deps.ResourceSampler = resourceSampler + // Subscribe to node pool change events for multi-instance reload. _ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) { if event.Type == cache.EventNodePoolChanged { @@ -1132,7 +1330,13 @@ func main() { s.SetLiteraryWorkLinker(literaryWorkService) s.SetEbookEnrichmentQueue(ebooks.NewEnrichmentQueue(deps.DB)) deps.Scanner = s - deps.ProbeEnsurer = scanner.NewPlaybackProbeEnsurer(fileRepo, ffprobePath, cfg.Playback.FFmpegPath, 10*time.Second) + probeEnsurer := scanner.NewPlaybackProbeEnsurer(fileRepo, ffprobePath, cfg.Playback.FFmpegPath, 10*time.Second) + // Probe repair and the copy-safety scan follow playback.ffmpeg_path + // without a restart; the scanner's own ffprobe path above still does not. + configWatcher.OnChange(func(_, updated *config.Config) { + probeEnsurer.SetFFmpegPath(updated.Playback.FFmpegPath) + }) + deps.ProbeEnsurer = probeEnsurer slog.Info("scanner initialized") } @@ -1924,12 +2128,51 @@ func main() { } if deps.DB != nil { - adminStatsProvider, statsErr := handlers.NewAdminStatsProvider(appCtx, deps.DB, deps.EventBus) + adminStatsProvider, statsErr := handlers.NewAdminStatsProvider(appCtx, deps.DB, deps.EventBus, watchProviderRegistry) if statsErr != nil { log.Fatalf("failed to create admin stats provider: %v", statsErr) } defer adminStatsProvider.Close() deps.AdminStatsProvider = adminStatsProvider + + playbackActivityProvider, playbackActivityErr := handlers.NewAdminPlaybackActivityProvider(appCtx, deps.DB, deps.EventBus) + if playbackActivityErr != nil { + log.Fatalf("failed to create admin playback activity provider: %v", playbackActivityErr) + } + defer playbackActivityProvider.Close() + deps.AdminPlaybackActivityProvider = playbackActivityProvider + + topActivityProvider, topActivityErr := handlers.NewAdminTopActivityProvider(appCtx, deps.DB, deps.EventBus) + if topActivityErr != nil { + log.Fatalf("failed to create admin top activity provider: %v", topActivityErr) + } + defer topActivityProvider.Close() + deps.AdminTopActivityProvider = topActivityProvider + + timeseriesProvider, timeseriesErr := handlers.NewAdminTimeseriesProvider(appCtx, deps.DB, deps.EventBus) + if timeseriesErr != nil { + log.Fatalf("failed to create admin timeseries provider: %v", timeseriesErr) + } + defer timeseriesProvider.Close() + deps.AdminTimeseriesProvider = timeseriesProvider + + downloadsStatsProvider, downloadsStatsErr := handlers.NewAdminDownloadsStatsProvider(appCtx, deps.DB, deps.EventBus) + if downloadsStatsErr != nil { + log.Fatalf("failed to create admin downloads stats provider: %v", downloadsStatsErr) + } + defer downloadsStatsProvider.Close() + deps.AdminDownloadsStatsProvider = downloadsStatsProvider + + // The dashboard metrics sampler is the only writer of concurrent-stream + // and egress history. Proxy and transcode nodes serve bytes but do not + // own the catalog database, so only the API-facing modes sample. An + // unset SILO_MODE is the integrated default everywhere else in this + // file, so it samples too. + if mode == "integrated" || mode == "api" || mode == "" { + dashSampler := dashmetrics.NewSampler(deps.DB, deps.StreamTelemetry, nodeIdentity) + dashSampler.Start(appCtx) + defer dashSampler.Stop() + } } // Wire recommendations engine, worker, and ratings repo if enabled. @@ -2181,7 +2424,7 @@ func main() { metadata.NewArtworkRevisionGarbageCollector(deps.DB, deps.S3Public), )) } - catalogSearchIndexer := catalog.NewCatalogSearchIndexer(deps.DB, settingsRepo) + catalogSearchIndexer := catalog.NewCatalogSearchIndexerFromSettings(deps.DB, settingsRepo, catalogSearchStartupSettings) taskMgr.Register(tasks.NewSyncCatalogSearchIndexTask(catalogSearchIndexer)) taskMgr.Register(tasks.NewRebuildCatalogSearchIndexTask(catalogSearchIndexer)) taskMgr.Register(tasks.NewCatalogSearchEventRetentionTask(catalog.NewSearchIndexEventRepository(deps.DB))) @@ -2229,6 +2472,10 @@ func main() { if deps.NodeRepo != nil { preparer.SetOriginLookup(deps.NodeRepo) } + // Prepared downloads cache a node's inventory separately from + // protocol-v3 planning, so the router's invalidation has to reach + // both. Set before NewRouter, which composes them. + deps.NodeCapabilityInvalidator = preparer.InvalidateNodeCapabilities artifactMgr := downloads.NewArtifactManager( downloads.NewArtifactRepository(deps.DB), downloads.NewRepository(deps.DB), @@ -2756,9 +3003,8 @@ func main() { if watchProviderService != nil { compatDeps.WatchScrobbler = watchProviderService } - compatSearchService := catalog.NewCatalogSearchService( - appCtx, - settingsRepo, + compatSearchService := catalog.NewCatalogSearchServiceFromSettings( + catalogSearchStartupSettings, itemRepo, catalog.NewSearchIndexEventRepository(deps.DB), deps.CatalogSearchVectorizer, diff --git a/cmd/silo/main_test.go b/cmd/silo/main_test.go index c9b6cce07..04070dea4 100644 --- a/cmd/silo/main_test.go +++ b/cmd/silo/main_test.go @@ -2,10 +2,12 @@ package main import ( "context" + "encoding/json" "errors" "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -13,6 +15,7 @@ import ( pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" "github.com/Silo-Server/silo-server/internal/api" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/plugins" "github.com/Silo-Server/silo-server/internal/tonemap" @@ -294,3 +297,191 @@ func TestReloadWatchSyncPluginProvidersDropsStaleProvidersOnCapabilityReadFailur t.Fatalf("stale provider %q remained registered", provider.Key()) } } + +// The sampler treats the returned root set as the whole truth: paths outside it +// are pruned — losing their cached capacity readings — and omitted from the +// sample. Returning nothing on a transient database error would therefore blank +// every library mount from the admin panel and from Prometheus, and leave the +// next pass reporting them unavailable until fresh probes land, all while the +// mounts are healthy. +func TestCachedLibraryPathsReusesTheLastGoodSetOnError(t *testing.T) { + var ( + paths []string + err error + ) + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { return paths, err }) + + paths = []string{"/mnt/movies", "/mnt/shows"} + if got := provider(context.Background()); !slices.Equal(got, paths) { + t.Fatalf("first read = %v, want %v", got, paths) + } + + paths, err = nil, errors.New("database is not answering") + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies", "/mnt/shows"}) { + t.Fatalf("read after an error = %v, want the last good set", got) + } + + // Recovery replaces it rather than merging. + paths, err = []string{"/mnt/movies"}, nil + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies"}) { + t.Fatalf("read after recovery = %v, want the fresh set", got) + } +} + +// An empty set the database actually returned is a real answer: an operator who +// removed their last library has no roots, and holding the old ones would keep +// reporting mounts the deployment no longer has. +func TestCachedLibraryPathsCachesADeliberateEmptyResult(t *testing.T) { + var ( + paths = []string{"/mnt/movies"} + err error + ) + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { return paths, err }) + provider(context.Background()) + + paths = nil + if got := provider(context.Background()); len(got) != 0 { + t.Fatalf("read = %v, want the deliberate empty result", got) + } + // And that empty result is what a later failure falls back to. + err = errors.New("database is not answering") + if got := provider(context.Background()); len(got) != 0 { + t.Fatalf("read after an error = %v, want the cached empty result", got) + } +} + +// The cache is what a failed read falls back to, so a caller scribbling on the +// slice it was handed must not be able to corrupt it — in either direction: the +// value returned from a successful read, or the one returned from the fallback +// itself. +func TestCachedLibraryPathsDoesNotShareItsCachedSlice(t *testing.T) { + failing := false + provider := cachedLibraryPaths(func(context.Context) ([]string, error) { + if failing { + return nil, errors.New("database is not answering") + } + return []string{"/mnt/movies"}, nil + }) + + provider(context.Background())[0] = "/tmp/clobbered" + + failing = true + fallback := provider(context.Background()) + if !slices.Equal(fallback, []string{"/mnt/movies"}) { + t.Fatalf("fallback = %v, want the cache untouched by the caller's mutation", fallback) + } + + fallback[0] = "/tmp/clobbered-again" + if got := provider(context.Background()); !slices.Equal(got, []string{"/mnt/movies"}) { + t.Fatalf("fallback = %v, want the cache untouched by a mutation of an earlier fallback", got) + } +} + +// The stored capability payload has to be the node's own bytes. +// +// Re-marshaling the decoded struct drops every field this build has no member +// for, which is exactly what happens during a rolling upgrade where a node is +// newer than the API reading it — and the truncation is then stored under the +// node's own hash. After the API is upgraded, the sweep sees the hashes agree +// and never refetches, so the durable inventory stays missing fields the new +// code reads until something unrelated moves the hash. +func TestNodeCapabilityFetcherStoresTheNodesOwnBytes(t *testing.T) { + const body = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"capability_hash":"sha256:abc","a_field_this_build_has_never_heard_of":{"nested":[1,2,3]}}` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/hw-capabilities" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + payload, hash, err := nodeCapabilityFetcher("secret", nodeCapabilityProbeBudget(nil))( + context.Background(), &nodepool.Node{ID: 1, URL: server.URL}) + if err != nil { + t.Fatalf("nodeCapabilityFetcher: %v", err) + } + if hash != "sha256:abc" { + t.Fatalf("hash = %q, want the node's own", hash) + } + + var stored map[string]any + if err := json.Unmarshal(payload, &stored); err != nil { + t.Fatalf("stored payload is not valid JSON: %v (%s)", err, payload) + } + if _, ok := stored["a_field_this_build_has_never_heard_of"]; !ok { + t.Fatalf("a field this build does not know was dropped, but its hash was kept: %s", payload) + } + if stored["resolved"] != "nvenc" { + t.Fatalf("resolved = %v, want the report's own value", stored["resolved"]) + } +} + +// A cold node's answer runs the whole FFmpeg probe matrix, and that matrix grows +// with the configured device count: a node with two render devices legitimately +// advertises a request budget past two minutes. Bounding every fetch at a fixed +// two minutes abandons such a node mid-probe and reports a failure for a node +// operating inside its published contract. +func TestNodeCapabilityProbeBudgetTracksTheConfiguredDevices(t *testing.T) { + single := &config.Config{} + single.Playback.HWAccel, single.Playback.HWDevice = "qsv", "/dev/dri/renderD128" + pair := &config.Config{} + pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" + + clusterNode := &nodepool.Node{ID: 1, URL: "http://gpu-1"} + oneDevice := nodeCapabilityProbeBudget(func() *config.Config { return single })(clusterNode) + twoDevices := nodeCapabilityProbeBudget(func() *config.Config { return pair })(clusterNode) + + if want := playback.CapabilityRequestTimeout("qsv", pair.Playback.HWDevice); twoDevices != want { + t.Fatalf("two-device budget = %v, want the node's own advertised %v", twoDevices, want) + } + if twoDevices <= nodeCapabilityRequestTimeout { + t.Fatalf("two-device budget = %v, want more than the %v floor — that is the case that was being cut short", + twoDevices, nodeCapabilityRequestTimeout) + } + if twoDevices <= oneDevice { + t.Fatalf("two-device budget %v is not above the one-device %v; the budget does not track the matrix", + twoDevices, oneDevice) + } + + // A node's own override wins over the cluster setting, because the worker + // probes the policy it will actually run. This is the case a cluster-wide + // read cannot see: one device configured centrally, two on this node. + twoDeviceOverride := pair.Playback.HWDevice + overridden := &nodepool.Node{ID: 1, URL: "http://gpu-1", HWDeviceOverride: &twoDeviceOverride} + if got := nodeCapabilityProbeBudget(func() *config.Config { return single })(overridden); got != twoDevices { + t.Fatalf("overridden node budget = %v, want the two-device %v its own policy needs", got, twoDevices) + } + + // An override set to the empty string means "inherit", not "no devices". + empty := "" + inheriting := &nodepool.Node{ID: 1, URL: "http://gpu-1", HWDeviceOverride: &empty, HWAccelOverride: &empty} + if got := nodeCapabilityProbeBudget(func() *config.Config { return pair })(inheriting); got != twoDevices { + t.Fatalf("inheriting node budget = %v, want the cluster's %v", got, twoDevices) + } + + // Nothing configured, no live config, or no node at all still gets a usable + // bound rather than zero. + if got := nodeCapabilityProbeBudget(nil)(clusterNode); got < nodeCapabilityRequestTimeout { + t.Fatalf("budget with no configuration = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) + } + if got := nodeCapabilityProbeBudget(func() *config.Config { return nil })(nil); got < nodeCapabilityRequestTimeout { + t.Fatalf("budget with a nil config and nil node = %v, want at least the %v floor", got, nodeCapabilityRequestTimeout) + } +} + +// The backstop the health sweep puts around the same fetch must never be the +// thing that cuts it short, or the budget above is decorative. +func TestCapabilityFetchBackstopExceedsTheAdvertisedBudget(t *testing.T) { + pair := &config.Config{} + pair.Playback.HWAccel, pair.Playback.HWDevice = "qsv", "/dev/dri/renderD128,/dev/dri/renderD129" + budget := nodeCapabilityProbeBudget(func() *config.Config { return pair })(&nodepool.Node{ID: 1, URL: "http://gpu-1"}) + + if nodepool.CapabilityRefreshTimeout <= budget { + t.Fatalf("health sweep backstop %v does not exceed the %v a two-device node advertises", + nodepool.CapabilityRefreshTimeout, budget) + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 0f574fb82..551d3df8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,7 +79,17 @@ services: - ${SILO_DATA_ROOT:-/opt/silo}/compat:/var/lib/silo/compat - ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode - ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro + # Only load-bearing when the Docker host is itself an LXC container, where + # these three files describe the physical machine and the LXC's own cap + # sits on a cgroup this container cannot see. lxcfs virtualizes them to the + # LXC's limits, so passing that view in is what makes CPU, load, and memory + # report this container's share instead of the whole machine's. Harmless + # everywhere else: on bare metal or a VM they are the same files Silo would + # read anyway. Each is used independently, so dropping one only reverts + # that reading. See docs/wiki/deployment/docker.md. - /proc/meminfo:/host/proc/meminfo:ro + - /proc/stat:/host/proc/stat:ro + - /proc/loadavg:/host/proc/loadavg:ro depends_on: postgres: condition: service_healthy @@ -105,6 +115,12 @@ services: # - "${PROXY_PORT:-8083}:8080" # volumes: # - ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro + # # See the note on the integrated service above: a node reports its own + # # CPU, load, and memory rather than the physical machine's only with + # # these mounted, and the Nodes page reads exactly what the node reports. + # - /proc/meminfo:/host/proc/meminfo:ro + # - /proc/stat:/host/proc/stat:ro + # - /proc/loadavg:/host/proc/loadavg:ro # depends_on: # postgres: # condition: service_healthy @@ -134,6 +150,12 @@ services: # - ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro # - ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins # - ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode + # # See the note on the integrated service above: a node reports its own + # # CPU, load, and memory rather than the physical machine's only with + # # these mounted, and the Nodes page reads exactly what the node reports. + # - /proc/meminfo:/host/proc/meminfo:ro + # - /proc/stat:/host/proc/stat:ro + # - /proc/loadavg:/host/proc/loadavg:ro # depends_on: # postgres: # condition: service_healthy diff --git a/docs/admin-api.md b/docs/admin-api.md index f29fb8b46..c5adc76f3 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -1,14 +1,639 @@ # Admin API -Server-administration endpoints under `/api/v1/admin`. Every route here requires -an authenticated account with the server-wide `admin` role — the same -authorization as `/api/v1/admin/sessions` — and none of them are part of the -client-facing contract that third-party apps build against. +Server-administration endpoints under `/api/v1/admin`. Every `/api/v1/admin` +route requires an authenticated account with the server-wide `admin` role — the +same authorization as `/api/v1/admin/sessions` — and none of them are part of +the client-facing contract that third-party apps build against. A few +deliberately public reads outside `/api/v1/admin` (marked `public` in the route +tables) are documented beside the admin writes they pair with. This document is new and covers only the routes listed below. The rest of the admin surface predates it and is currently documented by the code and by the design documents under `docs/design/`. +## Branding assets + +Uploadable images white-label the server: the sidebar wordmark, the square +mark (collapsed sidebar and installed PWA), optional light-theme variants of +both, the browser favicon, and the login background. Each is stored in the public S3 bucket and referenced from a +`server_settings` row, so uploads return `503 unavailable` until +`s3.public_bucket` is configured. + +| Route | Auth | Purpose | +| --------------------------------------------- | ------ | -------------------------------------------------------------------- | +| `POST /api/v1/admin/branding/assets/{kind}` | admin | Upload (multipart, field name `file`). Replaces whatever is stored. | +| `DELETE /api/v1/admin/branding/assets/{kind}` | admin | Clear the asset. `204`, and clearing an unset asset is not an error. | +| `GET /api/v1/branding/assets/{kind}` | public | Serve the stored bytes. Content-addressed, so `immutable` cached. | +| `GET /api/v1/theme/branding` | public | Current branding, including each asset URL (omitted when unset). | + +Public reads are deliberately unauthenticated: branding has to apply on the +login page, before anyone has a session. + +`{kind}` is one of `wordmark`, `wordmark_light`, `mark`, `mark_light`, `favicon`, `login_bg` — the light variants follow their base kind's processing. Uploads are +processed per kind — the numbers below are the contract the admin UI quotes back +to the operator, and they live in `internal/branding/assets.go`: + +| Kind | Accepts | Max upload | Stored as | +| ---------- | ------------------- | ---------- | ------------------------------------------------------------------------------------------------------------- | +| `wordmark` | PNG, JPEG, WebP | 8 MB | WebP, aspect preserved, capped at 640px wide. Narrower art is not enlarged. | +| `mark` | PNG, JPEG, WebP | 8 MB | WebP, center-cropped to a square, then forced to exactly 512×512 (smaller art is upscaled). | +| `favicon` | PNG, WebP, ICO, SVG | 1 MB | Byte-for-byte as uploaded, so `.ico` and `.svg` keep working in browsers that will not render a WebP favicon. | +| `login_bg` | PNG, JPEG, WebP | 12 MB | WebP, aspect preserved, capped at 2560px wide. Clients display it cover-cropped. | + +There is one stored variant per kind, not a responsive set: the PWA manifest +advertises the single 512px mark at both 192×192 and 512×512, and native clients +read the same URLs as the web app. Recommend source art at or above the stored +size — anything larger is downscaled, anything smaller is either left small +(wordmark, login background) or upscaled (mark). + +Failure modes: `400 bad_request` for an unknown kind, a missing `file` field, or +a content type the kind does not accept; `413 too_large` past the cap; +`503 unavailable` when asset storage is not configured. + +Uploaded SVG favicons are admin-controlled but served from the app origin, so +every asset response carries `X-Content-Type-Options: nosniff` and a sandboxing +`Content-Security-Policy` — a directly-navigated SVG cannot run script in the +viewer's session. + +## Server status and restarts + +Some settings are only read at startup. Two routes carry that contract: + +| Route | Auth | Purpose | +|---|---|---| +| `GET /api/v1/admin/server/status` | admin | Process start time and pending-restart state. | +| `GET /api/v1/admin/settings/restart-keys` | admin | The compiled registry of setting keys that only take effect after a restart (`internal/config/restart_keys.go`). | + +`GET /api/v1/admin/server/status` response: + +| Field | Type | Meaning | +|---|---|---| +| `started_at` | RFC3339 string | When this process started. | +| `restart_required` | bool | A restart-required change was saved. Latches true for the life of the process; a real restart clears it by starting a new process. | +| `restart_required_at` | RFC3339 string | When the flag first latched. Omitted until then. | +| `restart_required_reason` | string | The reason of the **last** restart-required save only — later saves overwrite it. | +| `restart_required_reasons` | string[] | Every distinct reason since boot, first-seen order. Settings saves record one `setting:` entry per restart-required key, so a client can scope a pending restart to the subsystem it belongs to. | +| `restart_mark_count` | int | Increments on every restart-required save. Because the boolean latches, this counter is the only signal that a **new** requirement arrived — the admin UI re-arms its dismissed restart banner on it. | +| `restart_requested`, `restart_requested_at` | bool, RFC3339 string | An in-app restart was requested, and when. | + +## Catalog search status + +`GET /api/v1/admin/catalog/search/status` reports the configured search +provider, the provider currently answering requests, Meilisearch health, index +state, semantic readiness, and links to the search maintenance tasks. + +`active_provider` describes the route requests actually take; it is not merely +the configured provider. `degraded` and the optional `degraded_reason` explain +temporary fallback or keyword-only operation. `index.rebuild_required` is true +when the active index does not match the current settings. Background search +maintenance runs at startup and every minute: it rebuilds a missing or stale +index, then resumes incremental event sync. When the prior index is known to +have the same document and media scope, Meilisearch continues serving keyword +search while the replacement is built; otherwise searches use PostgreSQL. + +## `GET /api/v1/admin/nodes` + +Lists every registered stream node — proxy and transcode alike — with its +configuration, last health result, and last stored hardware inventory. + +Always `200 OK` with a JSON array. + +| Field | Type | Meaning | +|---|---|---| +| `id`, `name`, `type`, `url` | int, string, string, string | Identity. `type` is `proxy` or `transcode`. `url` is the backend address: what the API server dials for health checks, capability fetches, and dispatch, and what a proxy dials to reach a transcode node — a private/internal address is fine and keeps that traffic off the public network. | +| `public_url` | string \| null | Client-facing base URL, when it differs from `url`. Stream and download URLs handed to players are built on it. Only meaningful on proxy nodes — clients never talk to transcode nodes. Absent or `null` means clients use `url`, which must then be publicly reachable. | +| `enabled` | bool | Whether the node is eligible for selection at all. | +| `healthy` | bool | Result of the last health check. | +| `active_jobs`, `egress_kbps` | int | Last health-reported load. `egress_kbps` is a rolling average and is currently non-zero for proxy nodes only. | +| `group` | string \| null | Co-location group. A group is only eligible while every enabled member is healthy. | +| `max_jobs`, `max_bandwidth_kbps` | int \| null | Capacity caps. `null` means unlimited. | +| `last_health_check` | RFC3339 string \| null | When the node was last checked. | +| `created_at` | RFC3339 string | When the node was registered. | +| `capabilities` | object | The node's last stored capability report — the same body `GET /hw-capabilities` returns on the node. Omitted until one has been stored. | +| `capabilities_hash` | string | Identity of that report, as computed by the node. Omitted with `capabilities`. | +| `advertised_capabilities_hash` | string | The hash the node named on its last health check. It differs from `capabilities_hash` while a refetch is outstanding or failing — the one case a recent `last_health_check` cannot rule out, since that check keeps succeeding while the refetch does not. Derived per sweep rather than stored, so it is **absent** until the first check after an API restart; **present and empty** when the node answered but named no hash at all, as a build predating capability reports does. Absent says nothing about the stored report; empty says the node is no longer confirming it. | +| `capabilities_refreshed_at` | RFC3339 string | When the report was fetched. This is the age of the *inventory*, not of the health check: an unchanged node keeps a report from hours ago. | +| `physical_gpu_keys` | string[] | Stable identities of the GPUs behind this node, derived from `capabilities` (see below). Omitted when the node reports no identifiable GPU. | +| `last_stats` | object | The node's most recent host resource sample — `{"system": …, "gpu": […]}` in the shape below. Omitted when the node reported none. | +| `hw_accel_override`, `hw_device_override` | string | This node's own acceleration policy (see below). Omitted when the node inherits the cluster-wide settings, which is the normal case. | +| `capability_drift` | string | Human-readable note describing how the node's hardware got worse at the last capability refetch. Omitted when the last refetch found no regression (see below). | +| `capability_drift_baseline` | object | What that note is waiting on — `{"backends": ["nvenc"], "devices": [{"uuid": "GPU-8a7b…", "aliases": ["GPU-8a7b…", "0000:03:00.0", "/dev/dri/renderD128"]}]}`. Never present without `capability_drift`; absent with it only for a note written before this field existed (see below). Each device carries every stable name it answered to, so it is recognized if it returns renumbered; `uuid` is held apart because it is the only name that can prove a *different* card, a replacement in the same slot inheriting both the slot and the render path. Either key is omitted when empty. | + +### Acceleration overrides + +`hw_accel_override` and `hw_device_override` override the cluster-wide +`playback.hw_accel` and `playback.hw_device` settings for one node. +`hw_accel_override` takes the same values as the cluster setting — `auto`, +`qsv`, `vaapi`, `nvenc`, `none`. Absent means inherit; there is no separate +"inherit" value to set. + +They exist for a heterogeneous deployment: one CPU-only node in a QSV cluster +sets `none` for itself instead of forcing every node onto the lowest common +denominator. A homogeneous deployment should leave both unset and configure +`playback.hw_accel` once. + +Repointing a node's `url` to a different machine clears the identity-bound +state on that row — `capabilities`, `capabilities_hash`, +`capabilities_refreshed_at`, `last_stats`, and the drift note with its baseline +— because all of it describes the worker the old address reached, and the pools +are reloaded from the row immediately. The replacement is treated as newly +registered until its first health check and capability fetch. + +A node finds its own row by URL first: `NODE_URL` on the node is matched +against `stream_nodes.url`, ignoring a trailing slash on either side. Set +`NODE_URL` explicitly on every node. Without it a node guesses +`http://localhost:` and adopts whatever row carries that URL, which on a +multi-node deployment can be a different machine's policy. + +If the URL does not match, the node falls back to `NODE_NAME` against the +registered name. This covers split-horizon topologies where `stream_nodes.url` +was registered as a public address the node's own `NODE_URL` never equals — +with `public_url` carrying the client-facing address, `url` can simply be the +node's internal address and match `NODE_URL` directly, which is the +recommended shape. `name` +carries no unique constraint, so an ambiguous match — more than one row +sharing that name — identifies nothing and adopts neither row's overrides; +registered names should be unique per node, and `NODE_NAME` should equal the +registered name. A node whose row *stops* matching — renaming it in the admin +form while the worker's `NODE_NAME` still holds the old value — keeps the last +overrides it read rather than reverting to the cluster settings, since the API +goes on dispatching that row's backend and a row that has gone is not evidence +an operator cleared the override. Fix the mismatch: the node adopts whatever it +finds on its next poll. + +The node overlays its row onto the cluster-wide playback settings on every +config reload, so the override is what that node probes with, advertises in +`capabilities.resolved`, and falls back to when a start request names no +backend. The API dispatches remote transcodes with the node's +`hw_accel_override` in preference to its own cluster setting, so the request +agrees with what the node would have run anyway. Dispatch reads the override +column, not `capabilities.resolved`: a node inheriting `auto` is dispatched +`auto` so it resolves against live hardware at session start rather than +against a snapshot. + +**A changed override applies without a restart, but not all at once.** An update +that actually moves either override asks the node to re-read its configuration +before the pools are reloaded, so the node adopts the new device before this +server begins dispatching the new backend. Without that ordering, changing both +at once — QSV on a render node to NVENC on a CUDA index, say — would pair the new +backend with the old device until the node's own poll caught up. + +That reload is non-destructive: sessions already transcoding are untouched, and +an edit that leaves both overrides where they were makes no call at all. The +API also drops its own cached view of what the node can do, so the next session +is planned against the new backend's tone-map executors rather than the previous +one's. It is +also best effort — a node that is unreachable still applies the change on its +next config reload (within 60 seconds). When it does not confirm, the policy is +published anyway and a warning names the node: withholding it would leave a +stored override never reaching dispatch, since nothing else re-reads the column. +Until that node's poll catches up its backend comes from this server while its +device comes from its own configuration, so a start dispatched to it in that +window can pair the two wrongly and fail. Either way the node re-advertises +`capabilities.resolved` at its next capability snapshot (every 15 minutes). Two +things do wait for a restart: the hardware encoder warmup that ran at boot, +which stays primed for the old backend, and sessions already transcoding, which +keep the backend they started with. Restart the node when you want all four in +agreement immediately. + +### `last_stats` + +Written by the same 30-second health check that writes `active_jobs`, so it is +exactly as old as `last_health_check` and never fresher. It is the current +sample only: nothing here is a time series, and operators who want history +scrape the node's own `GET /metrics` (unauthenticated, on the node's listener, +same `streamapp_node_*` gauges, with disk series labeled by role rather than by +path). A sample larger than 32 KiB is dropped rather than stored — the health +verdict is what routes streams, and no honest sample comes close to that. + +Cgroup correction alone is not always enough: a Docker container nested inside +an LXC container sees no limit on its own cgroup (the LXC's cap lives on an +ancestor cgroup outside its namespace), so `cpu_pct`, `cores`, `load1`, and the +memory fields below read as the bare-metal host's totals unless the deployment +bind-mounts lxcfs's virtualized `/proc` files in — see the LXC section of +[docs/wiki/deployment/docker.md](wiki/deployment/docker.md#node-metrics). + +`last_stats.system`: + +| Field | Type | Meaning | +|---|---|---| +| `cpu_pct` | int | Aggregate busy percentage across all cores over the last sampling interval (5s), 0-100. Idle and iowait both count as not busy. Under a cgroup this is the container's own consumption against its own quota, not the host's. | +| `load1` | float | 1-minute load average. Unlike `cpu_pct` it also counts tasks blocked on storage, so a node stuck on I/O looks idle in one and busy in the other. Always host-wide: the kernel keeps no per-cgroup load average. | +| `cores` | int | CPUs this process may run on — the cgroup's CPU quota rounded up where one is set, otherwise every CPU the kernel reports. This is what `cpu_pct` is normalized against and what `load1` must be read relative to. | +| `mem_used_mb`, `mem_total_mb` | int | Memory. Under a cgroup with a concrete limit these are the cgroup's limit and working set (page cache excluded); otherwise both are the host's. The pair always comes from one domain — a container with no limit publishes a readable working set, and reporting that against host RAM would read as idle on a machine that is nearly out of memory. | +| `disks` | object[] | Sampled mounts, transcode scratch first, deduplicated by filesystem and capped at 8 — unmeasurable paths included, so the array never grows with the library count. The cap is on what is *probed*, not only on what is reported: each mount costs a `statfs` goroutine per interval that a dead network mount parks indefinitely. Roots past the cap are not sampled, and the number left out is logged (`component=nodemetrics`) rather than left to look like a clean bill of health. A second ceiling bounds probes outstanding at once across every path ever offered, so reconfiguring library roots while mounts are wedged cannot accumulate parked goroutines. | +| `net_rx_bps`, `net_tx_bps` | int | Aggregate throughput in **bits** per second, loopback excluded. In a container this is the container's own network namespace. | + +Each entry in `disks`: + +| Field | Type | Meaning | +|---|---|---| +| `path` | string | Where the mount is. Absent from a node's `last_stats`: the node reports it on its own bearer-authed `/status`, and the API host reports its own on `GET /admin/system/resources`, but a node's `/health` takes no credential and withholds it. Use `role`. | +| `role` | string | What the mount is for: `scratch`, or `library-N` positionally per media root. Assigned when the sample is built, so it names the same mount on `/health`, `/status`, `/admin/system/resources` and the `streamapp_node_disk_*` series, and it stays with the mount even when a probe cannot measure it. | +| `used_gb`, `total_gb` | float | Capacity in GiB. `used_gb` counts filesystem-reserved blocks, as `df` does. `total_gb` is the capacity usable by the node process — used plus still-available — so it reads lower than the device's nameplate size on a volume that reserves blocks for root, and `used_gb`/`total_gb` is the ratio `df` prints as Use%. | +| `stale` | bool | The numbers are real but carried over from an earlier pass because the current probe has not returned — the normal reading for a network mount whose server went away. Omitted when false. | +| `unavailable` | bool | The path has never been measured on this node (it does not exist here, or the first probe is still hanging). `used_gb`/`total_gb` are meaningless. Omitted when false. | +| `scratch` | bool | This is the node's transcode working directory. Set on at most one entry; a media root sharing that volume is deduplicated onto it. Omitted when false. | + +`scratch` exists because this server does not know a node's transcode directory — +the node does. It is the one mount whose filling up breaks transcoding rather +than browsing, so the entry has to identify itself. Node selection reads it: +see "Scratch admission" below. It is also what labels the node's own +`streamapp_node_disk_*` series `scratch` instead of `library-N`. + +### Scratch admission + +A transcode writes HLS segments to its node's scratch volume for the whole life +of the session, so admitting one onto a nearly full node produces a stream that +dies mid-playback — after the client has already committed to it. Transcode +selection therefore skips a node whose `scratch` entry reports **95% or more** +used, and prefers a node with headroom even when the full one carries fewer +jobs. + +The exclusion is soft in two directions: + +- If it would leave no eligible candidate at all, it is ignored and the ordinary + least-jobs selection stands. Degraded service beats no service, and 95% was + not chosen to be a kill switch for a whole cluster. +- A node whose fill cannot be read is never excluded: no sample, no `scratch` + entry (a node predating the flag), an unmeasurable path, or numbers the node + itself marked `stale`. Taking capacity away on a fill we cannot read would be + worse than the failure it prevents. + +Each transition into pressure is logged once per node (`component=nodepool`, +"scratch volume nearly full"), not once per session start. + +Nothing else routes on `last_stats`, and the guard applies to playback and +local-egress transcode selection only — not to proxy selection, and not to the +non-streaming transcode reservations used by prepared downloads. + +Each entry in `last_stats.gpu`: + +| Field | Type | Meaning | +|---|---|---| +| `device` | string | The render node path (`/dev/dri/renderD128`), or `cuda:N` for an NVIDIA GPU with no readable DRM node. | +| `vendor` | string | `intel`, `nvidia` or `amd`. Omitted when sysfs names a vendor we do not recognize. | +| `sessions` | int | GPU workloads this node currently has pinned to the device. It comes from the playback device balancer, so it is exact for Silo's own work and blind to any other tenant's. With no `playback.hw_device` configured the workload is counted against the render device the transcode will actually open — the one auto-detection verified the backend on, or the first available render node when the backend was named explicitly and no detection walk ran. It goes uncounted only on a host with no render device at all. | +| `video_busy_pct`, `render_busy_pct` | int | Engine busy percentages over the sampling interval. | +| `total_busy_pct` | int | Whole-GPU utilization *including other tenants*. | +| `vram_used_mb`, `vram_total_mb` | int | GPU memory. | +| `source` | string | What produced the numbers: `fdinfo`, `nvidia-smi`, `fdinfo+nvidia-smi`, or `unavailable`. | + +Every measurement field above is omitted when nothing measured it, and +availability is per field rather than per device: absent is not zero and must +not be rendered as an idle GPU. A card can answer for some columns and not +others — `nvidia-smi` prints `[N/A]` for an engine a GPU cannot report while +still giving real memory figures, and a device reached only through `nvidia-smi` +has no render-engine reading at all — so read each field's presence, not the +device's. + +`source` is what tells an operator how far to trust the busy percentages that +are present. `fdinfo` is the unprivileged DRM baseline and covers **only this +node's own ffmpeg children** — a GPU shared with anything outside Silo reads as +less busy than it is. `nvidia-smi` is whole-GPU. `unavailable` means nothing +could measure the device this interval, so it carries no percentages at all. + +A node reports these fields in its own `/health` and `/status`; the API stores +them opaquely and parses only what it routes on. No GPU field is one of those — +nothing in node selection reads `last_stats.gpu`. The one part that is read is +the `scratch` disk entry, described under "Scratch admission" above. + +`last_stats` comes from `/health`, which takes no credential, so it carries no +filesystem paths — disk entries are named by `role`. GPU `device` values are +kept: a render node or a CUDA index is a fact about the hardware rather than +about this deployment, and the unauthenticated `/metrics` already labels its +per-GPU series with the same value. + +Capability reports are refreshed by the background health sweep, not by this +read: a node advertises a `capabilities_hash` in its own health response, and +only a hash that differs from the stored one triggers a refetch. A node running +a build from before capability snapshots advertises no hash and therefore +carries none of the four fields above. A failed refetch keeps the previous +report rather than clearing it — a node that cannot be reached is not evidence +that its hardware changed. The refetch itself runs outside the sweep's own wait, +one at a time per node, so a slow capability probe cannot delay the health +cadence of the other nodes; a new report can therefore land shortly after the +check that noticed the change rather than with it. + +An operator who cannot wait for the sweep — or whose node will never advertise a +changed hash because its probe results are cached for its process lifetime — uses +`POST /api/v1/admin/nodes/{id}/reprobe`, which stores the new report before it +answers. + +### `capability_drift` + +Set when a capability refetch shows the node's hardware got **worse** than the +report it replaced: a backend that used to pass its FFmpeg probe and now fails, +or a render device that is gone. It reads like +`verified hardware backends lost: qsv; render devices gone: /dev/dri/renderD128; +resolved backend qsv -> none`, and is capped at 512 characters. + +It exists because that regression is otherwise only a log line, and the node +stays `healthy` throughout: a driver that stopped working silently turns a GPU +transcoder into a CPU one, which shows up to users as slow or failing playback +long before anyone reads a log. + +Semantics worth knowing: + +- Setting it is a comparison; clearing it is not. The note appears when a refetch + loses something, and it records what it lost in `capability_drift_baseline`. + Clearing requires that specific hardware back: every backend in the baseline + verifying again, and every device in it answering to one of its recorded + aliases. A refetch that finds nothing *newly* lost leaves the note alone, + because a delta against an already-degraded report always finds nothing — a + reboot moves `boot_id`, a reworded FFmpeg failure moves the probe reason, and + either would otherwise erase a standing regression. Three cases make the + baseline necessary rather than pedantic, and none of them are caught by + looking at the current report alone: a GPU that disappeared completely leaves + no candidate backend to fail; a multi-GPU node that lost one card keeps + probing the survivor perfectly cleanly; and adding an unrelated GPU grows the + inventory without repairing anything. Successive losses accumulate, so two + cards going one at a time must both return. +- A note carried over from before the baseline existed has nothing recorded to + wait for, and a clean report clears it. +- Only a backend that was *probed and failed* counts as lost. A backend simply + absent from the report was not asked about — detection probes the backends the + configured `hw_device` gives it candidates for — so repointing a node from a + QSV render path to an NVENC index is not a regression. Hardware actually + disappearing shows up in the device inventory, which is the host's own and + owes nothing to the configuration: `render_devices` for cards with a DRM node, + and `nvidia_gpu_uuids` for those without one, which is the ordinary shape of + an NVENC container. A card that vanishes from either is a loss. +- A backend reported as `skipped` neither sets the note nor holds it open. + Skipping means no probe ran because the node cannot open the backend's + configured devices, which is a statement about access rather than about + hardware — the GPU column reports that state separately. +- Only a loss is reported. Added hardware is not drift. +- A node's first stored report carries none — there is nothing to compare it + against. +- It is written in the same statement as `capabilities` and `capabilities_hash`, + so it always describes the report stored beside it. +- Nothing routes on it. Node selection reads `healthy`, capacity, capability + eligibility, and the scratch guard above — never this field. + +Refetches only happen when a node advertises a changed `capabilities_hash`, so a +node whose GPU broke while its process kept running may report nothing new: the +probe results are cached for its process lifetime. `POST +/api/v1/admin/nodes/{id}/reprobe` is what forces the question. + +### `physical_gpu_keys` + +One key per GPU in the stored report, deduplicated and sorted. From each render +device: + +- the device's `gpu_uuid` when present (NVIDIA's permanent GPU identity, which + follows the card between slots and hosts), otherwise +- `|`, because a PCI slot only means the same hardware + within one boot of one kernel. + +Plus every entry in `nvidia_gpu_uuids`, which is what covers a card with no +readable DRM node — the ordinary NVIDIA container, where NVENC works and +`render_device_details` is empty. A uuid is host-independent, so a card reported +both ways yields one key, and a container that sees only `/dev/nvidia*` and one +that also sees `/dev/dri` recognize the same physical GPU. + +A device with neither identity contributes no key rather than a synthetic one, +and so does a slot on a host that reported no `boot_id`: `boot_id` detection is +best-effort, and an unscoped slot is not an identity, since every host with an +Intel iGPU has one at `0000:00:02.0`. Two nodes sharing a key are backed by the +same physical GPU — the case that makes per-node capacity accounting wrong, and +which no single node's report can express. The keys are derived from the stored report on every read, so they are +present as soon as a report is, including immediately after an API restart. + +Caveats on what a key can prove: + +- A key is only stable within one boot of the host it came from. `boot_id` + changes on reboot, so a fallback key does too, and the same card looks like a + different GPU until every node on that host has re-reported. An NVIDIA + `gpu_uuid` has no such limit. +- Intel and AMD GPUs passed through to separate VMs cannot be correlated at + all: each guest reports its own `boot_id` and its own PCI topology, so two + guests on one card produce two unrelated keys. Sharing there is invisible to + the server, and stays a matter for how the host is partitioned. + +Node selection uses the same keys as a tie-breaker: among transcode nodes that +are otherwise level on effective job count, the one whose physical GPU group — +itself plus every pooled transcode node sharing a key with it — carries the +fewest jobs wins. It never overrides the job count itself or the soft affinity +that keeps a session on its current node, and it does not apply to proxy +selection, which is round-robin and does no GPU work. + +## `POST /api/v1/admin/nodes` + +Registers a node. Body: `name`, `type` (`proxy` or `transcode`), `url`, and the +optional `public_url`, `group`, `max_jobs`, `max_bandwidth_kbps`. A +non-positive cap and an empty group mean "unlimited" and "ungrouped"; an empty +`public_url` means clients use `url`. + +`201 Created` with the created node in the same shape as one list entry (with +no capability fields yet — nothing has been fetched). `400 Bad Request` when a +required field is missing or `type` is not one of the two allowed values. The +node pools are reloaded afterwards. + +## `PUT /api/v1/admin/nodes/{id}` + +Updates a node's mutable fields. Every field is optional; an omitted field is +left unchanged. An empty-string `group` clears the group, and a non-positive +`max_jobs` or `max_bandwidth_kbps` clears that cap. + +`public_url` follows the same convention as the overrides below: `null` or an +empty string clears it, sending clients back to `url`; an omitted field leaves +it alone. + +`hw_accel_override` and `hw_device_override` are writable here. Either `null` +or an empty string clears one, restoring inheritance of the cluster-wide +setting; an omitted field leaves it alone. Clearing an override is a real +change with a real effect, so it is deliberately expressible rather than being +indistinguishable from omission. + +`200 OK` with the updated node, `404 Not Found` for an unknown id, +`400 Bad Request` when `hw_accel_override` is not one of `auto`, `qsv`, +`vaapi`, `nvenc`, `none` (matched case-insensitively and stored lowercase, as +`playback.hw_accel` is). The node pools are reloaded afterwards, so remote +dispatch honors a new override immediately; the target node itself picks it up +on its next config reload — see "Acceleration overrides" above for what waits +for a restart. + +Capability fields are not writable here. They are owned by the health sweep, +because only the node can say what hardware it has. + +## `DELETE /api/v1/admin/nodes/{id}` + +Removes a node. `204 No Content`, or `404 Not Found` for an unknown id. The +node pools are reloaded afterwards. Sessions already streaming from the node +are not torn down by this call. + +## `POST /api/v1/admin/nodes/{id}/check` + +Runs one health check against a node immediately and persists the result, for +an admin who does not want to wait for the next 30-second sweep. + +Always `200 OK`; an unreachable node is reported as `healthy: false` rather +than as an error status. `404 Not Found` for an unknown id. + +| Field | Type | Meaning | +|---|---|---| +| `healthy` | bool | The node answered its health endpoint. | +| `active_jobs`, `egress_kbps` | int | What it reported. Zero when unhealthy. | +| `capabilities_hash` | string | The hash the node advertised on this check. Omitted when the node reports none. | + +The check also persists the node's resource sample, so `last_stats` on the list +response reflects this check immediately. The sample itself is not echoed here. + +This is the node's *current* hash, not the stored one. A value here that +differs from the `capabilities_hash` in the list response means the background +sweep has a refetch pending; this route does not fetch capabilities itself. + +## `POST /api/v1/admin/nodes/{id}/reprobe` + +Tells one node to discard its cached hardware-probe verdicts and re-verify +against live hardware, then refetches and stores the resulting inventory +immediately. + +This is the answer to hardware that stopped working underneath a running node. A +node caches a **successful** probe for its whole process lifetime — re-verifying +per request would put FFmpeg execs on the playback path — so a GPU that has since +been removed, or whose driver was replaced with one that cannot encode, keeps +reporting `verified: true` until the node restarts. That is not visible in a +health check, because the node is healthy either way. Use it after installing or +downgrading a GPU driver, after changing which devices a node's container can +open, after replacing an FFmpeg build in place, and to confirm a +`capability_drift` note is still true. + +The reverse needs no action: a **failed** GPU probe carries a 15-second negative +TTL and is retried on its own, so a repaired driver flips `verified` to `true`, +changes the node's `capabilities_hash`, and is refetched within one snapshot +interval. The exception is the tone-map matrix, which caches any non-empty +inventory for the process lifetime, so a node whose GPU was broken at start can +stay software-only for tone mapping until it is re-probed or restarted. + +Body: none. Always `200 OK`; a node that refused or could not be reached is +reported in the body rather than as an HTTP error status, matching +`{id}/check` and `{id}/force-reload`. `404 Not Found` for an unknown id. + +| Field | Type | Meaning | +|---|---|---| +| `node_id`, `node_name` | int, string | The node this action ran against. | +| `status` | string | `ok` or `error`. | +| `error` | string | Why the node failed. Omitted on success. | +| `resolved` | string | The backend the node picked after re-probing. Omitted on failure. | +| `capability_hash` | string | The snapshot the node published. Compare it against `capabilities_hash` from the list response taken *before* the call to see whether anything changed. Omitted on failure. | +| `capabilities_refreshed` | bool | Whether this server also stored the node's new inventory before answering. | + +`capabilities_refreshed: false` with `status: ok` means the node re-probed but +the stored row has not caught up yet — a refresh for that node was already +running, or this deployment has no health sweep. The next sweep stores it. + +A node whose probe could not complete answers `status: error` and **keeps its +previous capability report**: an unfinished probe is not evidence the hardware +changed, and publishing a partial one would announce a change that did not +happen. Nothing is stored in that case, so a failed re-probe never degrades what +the list shows. + +A node that is **transcoding refuses**, also as `status: error`, and keeps its +report. Every hardware probe ends in a real encode on the GPU; a card at its +concurrent encoder-session limit fails that encode with an error nothing can +distinguish from a missing device, and the resulting `verified: false` would be +stored as a hardware regression for a GPU that is at that moment encoding. +Disable the node or wait for it to drain, then re-probe. + +The call can take a while. The node is given the probe budget it advertises in +its own report (`probe_request_timeout_ms`, up to five minutes); a node that has +never been inventoried gets 150 seconds. The capability refetch that follows adds +up to two minutes. The connection's write deadline is extended to cover both, so +this route can legitimately outlive the API listener's ordinary 120-second +`WriteTimeout`; a client should allow for that rather than treating a long wait +as a hung request. This re-probes only — it does not reload configuration or tear +anything down. + +Under the hood this is a bearer-authenticated `POST +/admin/reprobe-capabilities` on the node's own listener, which both transcode +nodes and proxy nodes serve. That route is internal to the cluster and is not +part of any client contract. + +## `GET /api/v1/admin/system/hw-accel` + +Reports GPU hardware and acceleration capability. With healthy transcode nodes +registered it probes each of them; with none it probes this host. The top-level +fields are the first node that answered (or the local probe), and `nodes` +carries one entry per healthy node. + +`playback.hw_device` is one cluster-wide value, so the per-node inventories are +what an operator needs to see that a device path exists on every node before +pinning one. + +Always `200 OK`. A node that failed its probe is reported in `nodes` with an +`error` rather than dropped, so a hardware problem is visible instead of silent. + +Top-level (and each node's own report): + +| Field | Type | Meaning | +|---|---|---| +| `resolved` | string | The backend that would actually be used: `nvenc`, `qsv`, `vaapi`, or `none`. An explicitly configured backend wins even when its probe failed — read `detected_backends` for why. | +| `render_devices` | string[] | Every accessible `/dev/dri/renderD*` path. | +| `render_device_details` | object[] | One entry per device (see below). | +| `intel_detected` | bool | An Intel GPU is present in the inventory. | +| `detected_backends` | object[] | One entry per backend that had candidate hardware, with the outcome of its FFmpeg verification (see below). | +| `boot_id` | string | The host's kernel boot identity (Linux only). Pairs with a device's `pci_address` to distinguish the same GPU from the same slot after a reboot. | +| `nvidia_gpu_uuids` | string[] | Every GPU `nvidia-smi` reports on this host, sorted. Independent of `render_device_details`, because a card is not always reachable through a DRM node — an NVIDIA container is routinely given `/dev/nvidia*` and the toolkit with no `/dev/dri` at all. Omitted where `nvidia-smi` is absent. | +| `capability_hash` | string | `sha256:` over this report's hardware identity and capability fields — not over `source`, `node_url`, the probe budget, or itself. Two reports of unchanged hardware hash identically regardless of probe order. | +| `source` | string | `local` for a probe of this host. | +| `node_url` | string | Set on a node's report. | +| `transformations`, `tone_map_capabilities` | object[] | What this host can execute, as advertised to the planner. | + +`render_device_details` entries: + +| Field | Type | Meaning | +|---|---|---| +| `path` | string | The `/dev/dri` path. Assigned by enumeration order, so it moves when hardware is added or removed. | +| `pci_address` | string | The device's PCI slot (e.g. `0000:03:00.0`), read from sysfs. Omitted when the device has no PCI identity. | +| `gpu_uuid` | string | NVIDIA's permanent GPU identity. Reported only for NVIDIA devices on hosts with `nvidia-smi` installed; omitted otherwise. | +| `description` | string | Short human label, e.g. `NVIDIA GPU (0x2204)`. | + +`detected_backends` entries: + +| Field | Type | Meaning | +|---|---|---| +| `backend` | string | `nvenc`, `qsv`, or `vaapi`. | +| `verified` | bool | At least one candidate device passed a real single-frame encode, not just an FFmpeg build-flag listing. | +| `devices` | string[] | Every candidate considered for this backend. | +| `device` | string | The candidate whose probe passed. Empty for NVENC, which addresses its GPU through CUDA rather than a render node. | +| `reason` | string | Why verification failed, attributed per device when several were tried. | + +Each entry in `nodes` carries `node_url` and `node_name` plus either that +node's `resolved`, `render_devices` and `render_device_details`, or an `error` +explaining why it could not be probed. The full report for one node — including +`detected_backends`, `boot_id` and `capability_hash` — is what +`GET /api/v1/admin/nodes` stores per node in `capabilities`. + +## `GET /api/v1/admin/system/resources` + +Reports the **API host's own** current resource sample — the counterpart to the +per-node `last_stats` above. + +The API host is not a registered stream node, so without this route the one +machine an operator cannot see is the machine serving the request (and, in +integrated mode, doing the transcoding). Unlike a node, this host also samples +the configured library roots: it is the process that knows what the library is, +and its view of a media mount is the authoritative one. + +Always `200 OK`. It reads a snapshot the sampler already published, so it costs +nothing and cannot hang regardless of what a mount or a GPU query is doing. + +| Field | Type | Meaning | +|---|---|---| +| `available` | bool | This host can be sampled. False on a non-Linux host, before the first sample lands, or when no sampler is running — in which case the fields below are absent. | +| `sampled_at` | RFC3339 string | When the sample was taken. Omitted when there is none. | +| `system` | object | Same shape as `last_stats.system` above. | +| `gpu` | object[] | Same shape as `last_stats.gpu` above. | + +Sampling is Linux-only: `available: false` on macOS or Windows is expected and +is not an error. History and alerting are Prometheus's job — the same numbers +are exposed as `streamapp_node_*` gauges on this process's existing `/metrics` +endpoint, with one deliberate difference: `/metrics` is unauthenticated, so its +disk series are labeled `mount="scratch"` / `mount="library-N"` and the library +paths themselves appear only here, behind admin auth. + ## `GET /api/v1/admin/stream-telemetry/parity` Returns the merged stream-telemetry view beside the two legacy live-session @@ -32,51 +657,549 @@ Always `200 OK`. "Nothing to compare" is expressed in the body rather than as an error status, because an empty report with a success status would read as agreement. +| Field | Type | Meaning | +| --------- | ------ | ------------------------------------------------------------------------------------ | +| `enabled` | bool | Stream telemetry is running in this process. | +| `reason` | string | Present when there is nothing to compare (telemetry disabled, or no view built yet). | +| `view` | object | State of the merged view the comparison was built from. | +| `sources` | array | One report per legacy projection. Empty when `enabled` is false. | + +`view`: + +| Field | Type | Meaning | +| ------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `available` | bool | A merged view exists. | +| `built_at` | RFC3339 string | When it was built. Omitted if never. | +| `age_ms`, `stale` | int, bool | Age of the cached view, and whether it exceeded the TTL. | +| `build_took_ms` | int | Cost of the last rebuild. | +| `refreshes`, `failures`, `last_error` | int, int, string | Cache counters since process start. | +| `complete` | bool | No publisher was stale, degraded or truncated. | +| `incomplete_reasons` | string[] | Why `complete` is false — e.g. `missing_publisher`, `publisher_truncated`, `decode_errors`, `truncated`. | +| `missing_publishers` | string[] | Publisher ids present in the roster but with no usable snapshot. | +| `clock_skew_suspected` | bool | A publisher stamped a time in the future. A clock running _behind_ is indistinguishable from a stalled publisher in one sample; compare `publishers` sequence across two reads to tell them apart. | +| `publishers` | string[] | `=`, where state is `fresh`, `degraded`, `stale` or `departed`. | +| `session_count`, `transfer_count` | int | Sizes of the merged view. | + +Each entry in `sources`: + +| Field | Type | Meaning | +| ----------- | -------- | -------------------------------------------- | +| `source` | string | `playback_sessions_sync` or `node_sessions`. | +| `available` | bool | The projection could be read. | +| `error` | string | Why it could not. | +| `notes` | string[] | Caveats that apply to this comparison. | +| `report` | object | The diff, when available. | + +`report`: + +| Field | Type | Meaning | +| --------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `telemetry_count`, `legacy_count`, `in_both` | int | Session counts on each side and their intersection. | +| `agrees` | bool | Same session set, and no field both sides express disagrees. Read `fields_absent` before treating this as clearance to cut over. | +| `telemetry_only`, `legacy_only` | string[] | Session ids present on one side only, capped. | +| `telemetry_only_truncated`, `legacy_only_truncated` | int | How many ids the cap dropped. | +| `mismatches` | object[] | Per-session field disagreements, capped. | +| `mismatches_truncated` | int | How many the cap dropped. | +| `fields_absent` | object | Per field, sessions both sides know where one side carries no value. A gap in a projection, not a disagreement. | + +A single report samples three independently updated stores, so one-sided +differences are normal and are not on their own evidence of a defect. Repeated +agreement over time is what the legacy-retirement project is gated on. + +## `/api/v1/admin/dashboard/layout` + +The admin dashboard is a widget grid each admin arranges for themselves. The +arrangement is stored per **account** (`users.id`), not per household profile, +so the same admin sees the same dashboard in every browser they log in from. + +The server stores the document verbatim and validates only that the body is at +most 16 KiB and that `layout` is a JSON object. Widget ids, column spans and row +heights are the admin web client's vocabulary: it already sanitizes what it +loads — dropping widgets it does not know, clamping each axis to that widget's +range, and filling in the default height for an entry saved before row heights +existed — so a second copy of that schema on the server would only be another +place to update whenever a widget is added. That also means a layout written by +a newer build degrades gracefully on an older one instead of being rejected. + +Writes are last-write-wins. The layout is one admin's own blob, so a race +between two of their tabs can cost only the older arrangement; `updated_at` is +returned so a compare-and-set could be layered on later without a contract +change. + +The web client keeps a copy in `localStorage` for instant paint and offline use, +adopts the server document when it arrives, and — the first time it finds no +server document but does have a local one — uploads that local layout once. + +### `GET /api/v1/admin/dashboard/layout` + +`200 OK`. Both fields are `null` when this admin has never saved a layout; that +is the normal first-load answer, not an error. + | Field | Type | Meaning | |---|---|---| -| `enabled` | bool | Stream telemetry is running in this process. | -| `reason` | string | Present when there is nothing to compare (telemetry disabled, or no view built yet). | -| `view` | object | State of the merged view the comparison was built from. | -| `sources` | array | One report per legacy projection. Empty when `enabled` is false. | +| `layout` | object \| null | The stored document, exactly as it was written. | +| `updated_at` | RFC3339 string \| null | When it was last written. | -`view`: +```json +{ + "layout": { + "version": 1, + "entries": [{ "id": "libraries", "span": 7, "rows": 4 }] + }, + "updated_at": "2026-08-26T10:00:00Z" +} +``` + +### `PUT /api/v1/admin/dashboard/layout` + +Body: `{"layout": {…}}`. Responds `204 No Content` on success, and +`400 bad_request` when the body is not valid JSON, when `layout` is absent or +`null`, when `layout` is not a JSON object, or when the body exceeds 16 KiB. + +### `DELETE /api/v1/admin/dashboard/layout` + +Resets this admin to the default arrangement. `204 No Content`, and idempotent: +deleting a layout that is not there succeeds. + +## `GET /api/v1/admin/dashboard/capabilities` + +Feature detection for the admin dashboard surface. Per the v1 rules a new +feature is detected rather than inferred from a server version, and every field +here is additive: a server that has this endpoint answers `true` for all of +them, and a server that predates the dashboard answers `404`. That is how a +client tells "this deployment is older than my build" from "the request failed". + +| Field | Meaning | +|---|---| +| `server_layouts` | `GET`/`PUT`/`DELETE /admin/dashboard/layout` store the widget arrangement per admin account. | +| `timeseries` | `GET /admin/stats/timeseries` serves sampled concurrent-stream and egress history. | +| `playback_activity` | `GET /admin/stats/playback-activity` serves the rolling playback activity aggregate. | +| `top_activity` | `GET /admin/stats/top-activity` serves the leaderboards. | +| `health` | `GET /admin/server/status` carries the additive `health` object. | +| `log_level_list` | `GET /admin/logs/app` accepts a multi-level filter. | +| `watch_providers` | `GET /admin/stats` carries the per-provider `watch_providers` array. | +| `downloads_stats` | `GET /admin/stats/downloads` serves the offline-download aggregate, and timeseries points carry the additive `download_egress_kbps` split. | + +```json +{ + "server_layouts": true, + "timeseries": true, + "playback_activity": true, + "top_activity": true, + "health": true, + "log_level_list": true, + "watch_providers": true, + "downloads_stats": true +} +``` + +## `GET /api/v1/admin/stats` + +Library, user, and playback totals for the dashboard, plus one entry per watch +provider. Cached in-process for 15s and bypassed with `?refresh=1`. | Field | Type | Meaning | |---|---|---| -| `available` | bool | A merged view exists. | -| `built_at` | RFC3339 string | When it was built. Omitted if never. | -| `age_ms`, `stale` | int, bool | Age of the cached view, and whether it exceeded the TTL. | -| `build_took_ms` | int | Cost of the last rebuild. | -| `refreshes`, `failures`, `last_error` | int, int, string | Cache counters since process start. | -| `complete` | bool | No publisher was stale, degraded or truncated. | -| `incomplete_reasons` | string[] | Why `complete` is false — e.g. `missing_publisher`, `publisher_truncated`, `decode_errors`, `truncated`. | -| `missing_publishers` | string[] | Publisher ids present in the roster but with no usable snapshot. | -| `clock_skew_suspected` | bool | A publisher stamped a time in the future. A clock running *behind* is indistinguishable from a stalled publisher in one sample; compare `publishers` sequence across two reads to tell them apart. | -| `publishers` | string[] | `=`, where state is `fresh`, `degraded`, `stale` or `departed`. | -| `session_count`, `transfer_count` | int | Sizes of the merged view. | +| `total_items`, `total_files`, `total_users` | int | Catalog and account totals. | +| `total_movies`, `total_movie_files`, `total_shows`, `total_show_files` | int | Per-kind catalog totals. | +| `active_streams` | int | Playback sessions currently synced as live. | +| `total_storage_bytes` | int | Sum of every scanned media file's size. | +| `watch_providers` | object[] | One entry per watch provider, ordered by `provider`. Always an array, never null. | -Each entry in `sources`: +`watch_providers` covers the union of the providers registered in the watchsync +registry — built-in and plugin-contributed alike, so a provider installed by a +plugin appears as soon as it registers, with zeros — and any provider that has +rows in the watch-provider tables. The second half of that union keeps history +visible after a provider's plugin is uninstalled; such an entry carries +`"registered": false` and falls back to its key as the display name. + +Each entry: | Field | Type | Meaning | |---|---|---| -| `source` | string | `playback_sessions_sync` or `node_sessions`. | -| `available` | bool | The projection could be read. | -| `error` | string | Why it could not. | -| `notes` | string[] | Caveats that apply to this comparison. | -| `report` | object | The diff, when available. | +| `provider` | string | Provider key (`trakt`, `simkl`, `mdblist`, a plugin's key). | +| `display_name` | string | Human name from the registry, or the key when the provider is not registered. | +| `registered` | bool | False when the provider only exists in stored rows. | +| `scrobbling` | bool | The provider declares the scrobble-playback capability. | +| `exporting` | bool | The provider declares the export-watched capability. | +| `connected_profiles` | int | Profiles with a connection to this provider. | +| `enabled_profiles` | int | Connected profiles with at least one sync direction enabled. | +| `export_enabled_profiles`, `scrobble_enabled_profiles` | int | Connected profiles with that toggle on. | +| `last_sync_completed_at` | string | RFC3339 timestamp of the newest completed sync run. Omitted when there is none. | +| `sync_runs_24h`, `sync_errors_24h` | int | Sync runs started in the last 24h, and how many of those failed. | +| `imported_watched_24h`, `imported_progress_24h`, `exported_watched_24h` | int | Rows moved by those runs. | +| `pending_exports`, `failed_exports` | int | Queued history exports by status, all-time. | +| `open_scrobbles` | int | Scrobble sessions started but not yet stopped. | +| `scrobbles_24h` | int | Scrobble sessions touched in the last 24h. | -`report`: +```json +{ + "total_items": 4821, + "total_files": 5310, + "total_users": 6, + "total_movies": 1980, + "total_shows": 212, + "active_streams": 3, + "total_storage_bytes": 91234567890, + "watch_providers": [ + { + "provider": "mdblist", + "display_name": "MDBList", + "registered": true, + "scrobbling": false, + "exporting": false, + "connected_profiles": 0, + "enabled_profiles": 0, + "export_enabled_profiles": 0, + "scrobble_enabled_profiles": 0, + "sync_runs_24h": 0, + "sync_errors_24h": 0, + "imported_watched_24h": 0, + "imported_progress_24h": 0, + "exported_watched_24h": 0, + "pending_exports": 0, + "failed_exports": 0, + "open_scrobbles": 0, + "scrobbles_24h": 0 + }, + { + "provider": "trakt", + "display_name": "Trakt", + "registered": true, + "scrobbling": true, + "exporting": true, + "connected_profiles": 2, + "enabled_profiles": 2, + "export_enabled_profiles": 2, + "scrobble_enabled_profiles": 1, + "last_sync_completed_at": "2026-03-01T12:00:00Z", + "sync_runs_24h": 5, + "sync_errors_24h": 0, + "imported_watched_24h": 30, + "imported_progress_24h": 4, + "exported_watched_24h": 12, + "pending_exports": 0, + "failed_exports": 0, + "open_scrobbles": 1, + "scrobbles_24h": 9 + } + ] +} +``` + +`watch_providers` replaced the Trakt-hardcoded `watch_provider_activity` object, +which was removed pre-lock; see the removals table in +[architecture/v1-scope.md](architecture/v1-scope.md). + +## `GET /api/v1/admin/stats/timeseries` + +Sampled history for the concurrent-streams and egress charts. Cached in-process +for 30s, dropped early on playback or admin activity, and bypassed with +`?refresh=1`. + +| Parameter | Type | Meaning | +|---|---|---| +| `hours` | int | Window length. Default 24, clamped to 1..744 (31 days, the retention window). A non-numeric value is `400 bad_request`. | +| `refresh` | bool | Bypass the cache for this read. | + +Neither series can be reconstructed after the fact — live sessions leave no +per-minute trace once they end, and node egress is a rolling average that each +health check overwrites — so a sampler (`internal/dashmetrics`) writes them as +they happen, once a minute, into `dashboard_metric_samples`. Samples older than +31 days are deleted. + +Reads bucket those minutes down so a response stays under ~750 points at any +window. `resolution_seconds` reports the bucket that was used — read it rather +than assuming the sampler's minute: + +| Requested window | `resolution_seconds` | +|---|---| +| ≤ 2 hours | 60 | +| ≤ 48 hours | 300 | +| ≤ 336 hours (14 days) | 1800 | +| wider | 7200 | + +A bucket wider than a minute reports the **peak** minute of each column, never +an average: these charts are read to answer "how bad did it get", and a mean +would erase exactly that. Stream counts and egress are maxed independently, so +a bucket's columns may come from different minutes within it. + +Each minute holds up to two kinds of row. The `shared` row is the cluster-wide +snapshot: stream counts by play method, plus the egress reported by enabled, +healthy stream nodes. Every replica tries to write it and the first one to land +wins, so the values for a minute come from whichever replica got there first — +they differ only by sub-second timing. A `proc:` row per API process +carries the viewer egress that process served, measured from stream telemetry; +without it a deployment with no stream nodes would chart zero egress forever. +Relay traffic is excluded, so bytes a proxy node passes through the API node are +not counted twice. + +Stream counts in a point therefore come from the shared row, while `egress_kbps` +sums every source for a minute before the peak minute of the bucket is taken. +Precision is mixed by design: node egress is a 30-second rolling average and +process egress is an exact byte delta. + +`egress_kbps` keeps its pre-split meaning — the total viewer egress across +every source — so a chart drawn from it alone stays truthful. +`download_egress_kbps` is the additive file-transfer subset of that total: +offline and direct downloads, ebook reads, and ABS file fetches, measured as +the actual bytes each API process wrote (including partial range-request +bodies). Node egress cannot be split and counts entirely outside the subset. +The sampler keeps the subset ≤ the total per minute, and each field takes its +own per-bucket peak, so neither can exceed the total and their difference is +never negative. But past the two-hour display resolution the two maxima are +preserved independently and may come from different minutes: subtracting them +does not yield any minute's playback rate. Chart the total and the download +subset as separate series rather than deriving a playback series. Samples +written before the split report `0` — read that as "not measured yet", not +"no downloads". + +A bucket with no sample in it is absent from `points` rather than zero — a gap +(a restart, a stopped server) and an idle bucket are different facts. Stream +telemetry being disabled means no `proc:` rows, not an error. +`oldest_sample_at` is `null` until the first sample exists, which is how a +fresh install renders "collecting data" instead of an empty chart. + +```json +{ + "resolution_seconds": 300, + "from": "2026-08-25T12:00:00Z", + "to": "2026-08-26T12:00:00Z", + "oldest_sample_at": "2026-08-24T09:31:00Z", + "points": [ + { + "t": "2026-08-26T11:55:00Z", + "streams": 3, + "direct": 1, + "remux": 0, + "transcode": 2, + "egress_kbps": 48211, + "download_egress_kbps": 6100 + } + ] +} +``` + +## `GET /api/v1/admin/stats/playback-activity` + +Bucketed playback starts split by play method, plus reliability scalars, for the +admin dashboard. Answers are cached in-process for 60s and dropped early when +the shared event bus reports playback or admin activity; `?refresh=1` drops the +cache before reading. + +| Parameter | Type | Meaning | +|---|---|---| +| `hours` | int | Window length. Default 24, clamped to 1..744. A non-numeric value is `400 bad_request`. | +| `refresh` | bool | Bypass the cache for this read. | + +Buckets are hourly up to a 48-hour window and daily beyond it; `bucket_seconds` +is `3600` or `86400` accordingly. A bucket's `hour` field is its start instant +at either width — it keeps that name because it is the same fact, and +`bucket_seconds` already says how wide the bucket is. + +Sessions come from `playback_history_admin` (which only gains a row when a +session finalizes) unioned with the live sessions table, so the current hour is +not under-counted. A live session cannot already be in history, so nothing is +counted twice. Live sessions with no recorded start — reconstructed after a +restart — are dated by their last update instead. + +`from` and `to` are the queried window on the database clock — the clock the +bucket filter ran against. Clients should anchor their zero-fill grid on `to` +rather than their own clock: a browser a minute behind the server around a +boundary would otherwise discard the newest bucket. + +`buckets` contains only buckets that saw a session; the client zero-fills the +window on the `bucket_seconds` grid so a quiet server draws empty columns rather +than a shorter chart. Everything in `reliability` is computed over the whole +requested window. `completion_rate` is +`completed_sessions / finalized_sessions`: live sessions are excluded from both +sides, because a session that is still playing has not failed to complete. + +`profiles_active_24h` is a fixed rolling-24h figure that ignores `hours` — it +answers "who watched today" whatever window the chart beside it is showing. It +counts distinct (account, profile) pairs in +`user_watch_history` over a rolling 24 hours, excluding history that was +imported or synced from a watch provider (`import`, `trakt`, `simkl`, +`mdblist`), so it means "watched on this server". Marked-watched (`manual`) +rows are counted: they are on-server actions. + +**Not reported:** time-to-first-frame and failed-start counts. Nothing records +a playback *start* event today, so both would have to be inferred from log +parsing. They need start-event capture in playback first, and are deliberately +absent rather than approximated. + +```json +{ + "hours": 24, + "bucket_seconds": 3600, + "from": "2026-08-25T10:41:03Z", + "to": "2026-08-26T10:41:03Z", + "buckets": [{ "hour": "2026-08-26T10:00:00Z", "direct": 4, "remux": 1, "transcode": 2 }], + "reliability": { + "sessions_started": 42, + "transcode_starts": 11, + "finalized_sessions": 38, + "completed_sessions": 27, + "completion_rate": 0.7105, + "unique_profiles": 9 + }, + "profiles_active_24h": 9 +} +``` + +## `GET /api/v1/admin/stats/top-activity` + +Most-watched titles and most-active profiles over a multi-day window. Cached +for 5 minutes — a seven-day ranking barely moves within minutes — with the same +`?refresh=1` escape hatch. + +| Parameter | Type | Meaning | +|---|---|---| +| `days` | int | Window length. Default 7, clamped to 1..30. | +| `limit` | int | Rows per list. Default 10, clamped to 1..25. | +| `refresh` | bool | Bypass the cache for this read. | + +`plays` on both lists counts `user_watch_history` rows with the same source +exclusions as `profiles_active_24h` above, so marking something watched counts +as a play. Episodes are rolled up to their series, so a season binge reads as +one show and a title's `media_item_id` is a series content id for TV. + +`total_seconds` is **watched time**, summed from finalized playback sessions +(`playback_history_admin.watched_seconds`) that *ended* inside the same window +— the same stop instant `watched_at` records, so plays and watch time see the +same sessions — not the runtime of what was played. Watch history records the media's full duration, +so summing that would report three hours for a movie someone abandoned after a +minute. An entry that was only ever marked watched has no sessions and reports +`0`. Because `watched_seconds` records a session's final absolute position, a +resumed session would claim the already-watched stretch again, so each +session's contribution is capped at its wall-clock length; the figure is an +estimate until playback records true elapsed viewing time. + +Profile display names live in the per-user stores rather than in watch history, +so they are read back from that profile's most recent `playback_history_admin` +row; a profile that has only ever marked things watched falls back to its +profile id. Ties are broken on a stable key (`media_item_id`, or +`user_id`/`profile_id`) so equal rows keep their order between refreshes. No +poster URLs are returned — the bar-list widgets do not need them, and it keeps +the query cheap. + +Both lists are `[]` on a server with no history, never `null`. + +```json +{ + "days": 7, + "limit": 10, + "titles": [ + { + "media_item_id": "…", + "title": "…", + "media_type": "series", + "plays": 18, + "total_seconds": 54120 + } + ], + "profiles": [ + { + "user_id": 3, + "username": "quick", + "profile_id": "p1", + "profile_name": "Quick", + "plays": 12, + "total_seconds": 40100 + } + ] +} +``` + +## `GET /api/v1/admin/stats/downloads` + +Offline-download aggregate for the dashboard's downloads widget. Cached +in-process for 60s, dropped early on admin activity from the shared event bus, +and bypassed with `?refresh=1`. + +| Parameter | Type | Meaning | +|---|---|---| +| `limit` | int | Rows in `top_users`. Default 10, clamped to 1..25. A non-numeric value is `400 bad_request`. | +| `refresh` | bool | Bypass the cache for this read. | + +The aggregate reads the `downloads` table, which carries two lifecycles: a +**managed device entry** (a device keeps the item offline; `device_id` set) and +a **one-shot web download** (`device_id` null, pruned over time). "Active" +means a managed entry whose status is `queued`, `preparing`, `ready`, +`downloading`, or `completed` — anything that has not ended in failure, +cancellation, or revocation. The headline numbers and `top_users` count active +managed entries only; the 24-hour counters cover both lifecycles, so one-shot +web downloads show up there. | Field | Type | Meaning | |---|---|---| -| `telemetry_count`, `legacy_count`, `in_both` | int | Session counts on each side and their intersection. | -| `agrees` | bool | Same session set, and no field both sides express disagrees. Read `fields_absent` before treating this as clearance to cut over. | -| `telemetry_only`, `legacy_only` | string[] | Session ids present on one side only, capped. | -| `telemetry_only_truncated`, `legacy_only_truncated` | int | How many ids the cap dropped. | -| `mismatches` | object[] | Per-session field disagreements, capped. | -| `mismatches_truncated` | int | How many the cap dropped. | -| `fields_absent` | object | Per field, sessions both sides know where one side carries no value. A gap in a projection, not a disagreement. | +| `users_with_downloads` | int | Distinct accounts (login accounts, not household profiles) with at least one active managed download. | +| `active_downloads` | int | Active managed entries. A series batch contributes one entry per episode. | +| `total_bytes` | int | Sum of `file_size` over completed managed entries — bytes sitting on devices as far as the server can know without devices reporting back. | +| `downloads_started_24h` | int | Rows created in the last 24 hours, both lifecycles. | +| `downloads_completed_24h` | int | Rows that reached `completed` in the last 24 hours, both lifecycles. | +| `limit` | int | The clamped `top_users` size the response was built with. | +| `top_users` | object[] | Accounts ranked by active managed downloads; `[]` when nobody downloads, never `null`. | -A single report samples three independently updated stores, so one-sided -differences are normal and are not on their own evidence of a defect. Repeated -agreement over time is what the legacy-retirement project is gated on. +Each `top_users` entry: `user_id`, `username`, `downloads` (active managed +entries), and `total_bytes` (completed managed entries only, like the headline). + +A deployment with the downloads feature disabled answers all zeros rather than +an error — the table exists on every deployment. + +```json +{ + "users_with_downloads": 2, + "active_downloads": 14, + "total_bytes": 52613349376, + "downloads_started_24h": 3, + "downloads_completed_24h": 2, + "limit": 10, + "top_users": [ + { "user_id": 3, "username": "quick", "downloads": 11, "total_bytes": 41234567890 }, + { "user_id": 5, "username": "kid", "downloads": 3, "total_bytes": 11378781486 } + ] +} +``` + +## `GET /api/v1/admin/server/status` — `health` + +The status route carries an additive `health` object for the dashboard health +strip. Every field the route already returned is unchanged; only `health` is +new, and the example below is trimmed to the fields it discusses: + +```json +{ + "started_at": "2026-08-26T09:00:00Z", + "restart_required": false, + "health": { + "postgres": { "configured": true, "ok": true, "latency_ms": 1.42 }, + "redis": { "configured": true, "ok": true, "latency_ms": 0.31 }, + "errors_24h": 4, + "warnings_24h": 12 + } +} +``` + +Each component reports `configured` first: `false` means this deployment runs +without that service — a supported single-node shape for Redis — and `ok` and +`latency_ms` are then absent, so "not present" and "present but broken" do not +look the same on the strip. Latency is the round trip of one ping, in +milliseconds with two decimals, bounded by a 2s timeout: a wedged dependency is +reported as `ok: false` rather than holding the route open. + +`errors_24h` / `warnings_24h` count `operational_logs` rows at those levels over +a rolling 24 hours, cached for 30s. A server with operational logging disabled +reports zeros and logs a warning; this route never fails over a secondary +number. + +Version, uptime and node health are not repeated here. The client composes them +from `GET /admin/system/build`, `started_at` above, and `GET /admin/nodes`. + +## `GET /api/v1/admin/logs/app` — `level` + +`level` accepts a comma-separated list, so one request can ask for several +levels at once (`?level=error,warn`). Values are trimmed, lowercased and +de-duplicated; a single value behaves exactly as before. The same parsing +applies to the log-stream WebSocket, so a stream filtered on two levels +delivers both. diff --git a/docs/architecture/admin-settings-ux.md b/docs/architecture/admin-settings-ux.md new file mode 100644 index 000000000..947628cc7 --- /dev/null +++ b/docs/architecture/admin-settings-ux.md @@ -0,0 +1,159 @@ +# Admin settings UX + +Admin settings are organized by admin intent ("I want subtitles to download +automatically"), not by subsystem. `/admin/settings` is the **Overview**: +server health across the top and one live card per settings group. Twelve +standalone pages hang off it: General, Storage & Database, Appearance, +Security & Access, Library & Metadata, Playback, Downloads, Subtitles & +Metadata, Watch Providers, AI Services, Notifications, and Compatibility. The +global admin sidebar has one Settings destination; the Overview owns the +settings information architecture. Old `?tab=` URLs and retired page ids from +earlier layouts (including `integrations`, now split into Subtitles & Metadata, +Watch Providers, and AI Services) redirect to the page that absorbed them +rather than 404ing. +`⌘K` (`AdminSectionCommandDialog`) is mounted in `AdminLayout` so search works +from every admin page, not just the Dashboard. + +## Visual system + +One page is on screen at a time, and each thing on screen carries one signal. +The admin settings detail view deliberately mirrors the user settings page +(`SettingsLayout`): one `surface-panel-lg` shell with a `SideNavItem` rail on +the left and the page content on the right, so the two settings surfaces read +as the same product. The Overview stays the category directory: it explains +each group's scope, and every category has its own `/admin/settings/:page` +route. The rail (`SettingsPageRail`, rendered by the settings shell) lists +every settings page with the open one marked; an All settings link above the +shell leads back to the directory. The rail is desktop-only — on smaller +screens the Overview is the directory, exactly as on the user side. The +Overview shows a health tile only for a tile in `warn` or `off`. The +**Setup & health** section explains that it holds recommendations and +configuration problems; an empty checklist reads "No action needed" and +names the conditions that will appear there. Below it is one card per +settings group. Each card explains the group's scope and names the sections +inside it. Live state stays in the health area instead of reducing a +multi-provider group to one misleading summary. + +A category page opens with `SettingsPageHeader`: the title, and page actions +if it has any. No breadcrumb, no lede, no status strip. Below it, settings +are hairline-ruled rows inside `FieldGroup`s — thin wrappers over the shared +`SettingsGroup` panel the user settings pages use — one panel per group, +never per field. The Advanced tier stays inline as one disclosure row per +group. A row can carry its `server_settings` key as a mono caption under the +label (`SettingField`'s `settingKey`) so an admin can match the UI to the +API and environment overrides, and a violet dot marks unsaved edits on the +row and on the group heading (`dirty`, driven from `form.isDirty`). A +description under a field label is the exception, not the rule: one short +sentence, and only when the label alone is ambiguous. Units live beside the +control (`SettingField`'s `unit`), not in the label. When every field in a +group needs a restart, the group says so once (`FieldGroup restartAll`) and +the fields inside drop their chips. Provider credentials are `ProviderTile`s +that expand in place to Test before saving; their border is neutral in every +state and the state is a dot plus a word in the header. Provider setup lives on +a provider page (Subtitles & Metadata, Watch Providers), not on the page that +owns the feature: Library & Metadata decides *whether* Silo looks for intro and +credits markers, while *which provider answers, in what order, and on what +terms* is a tile beside the subtitle and metadata providers, with a cross-link +each way. A tile only reads "Connected" when the provider could actually serve +a request — its configuration saved and the provider switched on — so an +installed plugin whose API key was never entered reads "Needs setup". Staged edits raise +one floating save pill (`SaveBar`) and arm the shell's unsaved-changes prompt; +the restart prompt is a single `RestartBanner` (`web/src/components/admin/`) +rendered by the admin shell (`AdminLayout`), never per page. A restart is owed +by the server, not by the page that asked for it, so the banner sits in the +flow at the top of the content column on *every* admin page — dashboard, +users, tasks, settings — and follows the admin around until they restart or +dismiss it. + +## Three tiers, and how to pick one for a new setting + +Every admin setting is one of: + +- **Essential** — shown by default, no disclosure needed. Target at most ~8 + essential controls per page above the fold. A setting is Essential only if a + household admin on a single-node install would plausibly need it without + being told to look for it (on/off toggles for a whole feature, the handful + of values that make the feature usable at all). +- **Advanced** — correct but not essential; collapsed by default behind one + `AdvancedSection` disclosure per page (or per `FieldGroup` on a dense page). + Open state persists in `localStorage` and auto-expands when a dirty or + invalid field lives inside it. Tuning knobs, alternate backends, + and anything whose default is good enough that most admins never touch it + belong here. +- **Hidden** — no UI at all, on any page. The setting is still a normal + `server_settings` row: readable and writable through the admin settings API + and environment configuration exactly as before this reorganization. Use + Hidden for legacy key families kept for compatibility, settings that only + make sense with expert knowledge of the codebase, or values better derived + automatically (e.g. from node pool capacity) than hand-set. + +The tier is a UI-only decision. It must never change a key's validation, +default resolution, or API visibility — moving a setting to Hidden is +reversible by adding UI back, not by a data migration. + +## Shared primitives + +Reuse these instead of adding a bespoke variant per page: + +- `SettingField` / `FieldGroup` / `SaveBar` (`web/src/pages/admin-settings/`) + and `useSettingsForm` (`web/src/hooks/`) — the one save model. Every page + batches edits and commits them through one `SaveBar` with Discard; provider + credentials are the only exception, and only because they need + Test-before-commit, which is `ProviderTile` rather than a bespoke card per + provider. +- `UnsavedChangesGuard` (`web/src/components/`) plus `useReportUnsavedChanges` + (`web/src/hooks/useUnsavedChanges.ts`) — the one unsaved-edits prompt. A form + only reports that it is dirty; the settings shell mounts the guard once and + blocks router navigation (rail, back link, admin sidebar, browser back) with + a confirmation. `useSettingsForm` keeps a `beforeunload` listener for tab + close and reload, which the router never sees. Blocking is `useBlocker`, + which is why `App.tsx` mounts a data router (`createBrowserRouter` + + `RouterProvider`) — a page must never grow its own prompt, its own blocker, + or its own draft store. +- `SettingsPageHeader` (`web/src/components/settings/`) — the one way a + section names itself. Live state belongs on the Overview, not repeated as a + strip on every page. +- `SettingsPageRail` (`web/src/components/settings/`) — the one sibling nav, + rendered once by the settings shell from `ADMIN_SETTINGS_NAV` using the + shared `SideNavSection`/`SideNavItem` primitives. Pages never render their + own nav or add entries directly to the rail. +- `SettingsGroup` (`web/src/components/settings/`) — the one settings panel, + shared with the user settings pages; admin pages reach it through the + `FieldGroup` wrapper, which layers on the restart-all line, the + unsaved-edits dot, and the restart context. +- `AdvancedSection` — the one collapsible-disclosure primitive for the + Advanced tier. Do not add another `
`, another bespoke collapsible + component, or a per-page expand/collapse toggle. +- `SecretField` — the one credential control: an always-editable password + input whose masked placeholder stands in for the saved value. Typing stages + a replacement; emptying the input keeps the saved secret, so no ordinary save + erases one by accident. Clearing is always a deliberate act, and every + surface has exactly one way to do it: either a page-level action (Disconnect, + Clear credentials) or, where the page has none, the field's own opt-in + `onClear`/`cleared` affordance, which stages the empty write for the save bar + and can be taken back with "Keep saved value" or Discard. +- `LimitField` — the one "Unlimited" checkbox pattern, replacing "0 = unlimited" + hint text conventions. +- A restart badge on `SettingField` itself, sourced from + `config.RestartRequired` (`internal/config/restart_keys.go`), not hand-copied + into hint text or inferred by a page-local heuristic. `RestartRequired` is + the single source of truth for which keys need a process restart to take + effect; a new field's badge must read that function (directly, or via a + manifest/meta endpoint built on top of it) rather than duplicating its + judgment. + +## Deferred out of this reorganization + +These were identified during the review but deliberately left for later work, +not folded into this pass: + +- Key renames (e.g. un-namespaced `allow_4k_transcode`, + `enable_transcode_throttle`, `transcode_throttle_seconds` moving under + `playback.*`). +- Introducing `server.public_url` as one canonical public URL that + `jellyfin_compat.public_url` and friends would derive from. +- Deleting the legacy `s3.operational_*` rows that a past migration copied and + never removed. + +Each still applies via its existing key and behavior; only the UI +reorganization and tiering in this document shipped now. diff --git a/docs/architecture/email.md b/docs/architecture/email.md index baa1c8d3d..5172e6050 100644 --- a/docs/architecture/email.md +++ b/docs/architecture/email.md @@ -38,7 +38,7 @@ low): | `email.from_address` | — | required | | `email.from_name` | `Silo` | | -Admin UI: Admin Settings → Connections → Email, including a synchronous test +Admin UI: Admin Settings → Notifications → Email, including a synchronous test send (`POST /api/v1/admin/email/test`). ## Adding a consumer diff --git a/docs/architecture/v1-scope.md b/docs/architecture/v1-scope.md index 039c2d94b..3ba10cdd5 100644 --- a/docs/architecture/v1-scope.md +++ b/docs/architecture/v1-scope.md @@ -39,6 +39,7 @@ justification and falls back to the Deprecation/Sunset flow like anything else. | `409 protocol_disabled` on `POST /api/v1/playback/route-events`, and the `"enabled": false` shape of `GET /api/v1/playback/capability` | Playback protocol v3, [spec](playback-protocol-v3.md) | Both described a server with v3 switched off. With v3 the only playback protocol that state cannot exist — "disabled" would mean "no playback at all". The `enabled` field itself is kept and is constant `true`, so clients that feature-detect against it keep working; only the negative shape and the status code go. | | The draft-v3 platform-specific wire vocabulary: `ClientPlaybackContextV3.features`, `.platform`, and `.engines`; `PlanV3.engine`; `output_route_generation` in start, replan, output-context, and route-event bodies; Android device/build fields (`brand`, `device`, `product`, `soc_*`, `build_*`, `security_patch`, `sdk_int`, `abis`); and the `media3_only` / `detailed_decode_capabilities` feature tokens | Platform-neutral playback protocol v3, [spec](playback-protocol-v3.md) | These names exposed one client's implementation as the cross-platform contract. Before v1 lock they are replaced by neutral delivery classes, evidence tiers, `device.platform` / `device.os_version` / bounded `platform_details`, opaque `output_context_id`, and top-level feature advertisement. Carrying both drafts through lock would force every client to translate Media3-specific aliases indefinitely and leave two conflicting sources of capability truth. | | Accepting `access_group_id` alongside the admin role on `POST /api/v1/admin/users`, `PUT /api/v1/admin/users/{id}`, and `POST /api/v1/admin/invitations` — all three now reject the combination with `422` (`ErrAdminGrouped`, "Admin accounts cannot belong to an access group") | Admin-ungrouped constraint, 2026-08-22 | Admins are never grouped: the household access-group ceiling has no meaning for an account that already has server-wide admin rights, and silently accepting a group on an admin account left a stored value that read as a policy nobody enforced. Rejecting the combination at write time is cheaper to carry than a deprecation window for a field whose only valid value on an admin account was already `null`. | +| The `watch_provider_activity` object on `GET /api/v1/admin/stats`, with its `trakt_connected_profiles`, `trakt_enabled_profiles`, `trakt_export_enabled`, and `trakt_scrobble_enabled` fields | Watch-provider dashboard widget, 2026-08-28 | Every field was hardcoded to Trakt while the watch-provider subsystem is pluggable: Simkl, MDBList, and any plugin provider sync through the same tables and were invisible in it. The replacement is the additive `watch_providers` array, one entry per registered provider, advertised by `watch_providers` on `GET /admin/dashboard/capabilities`. The object had exactly one consumer — the admin web dashboard, which ships with the server — so a deprecation window would only preserve a shape that can never describe a second provider. | Feature-detection precedent: clients discover which metadata providers (including the built-in NFO provider, #216) apply to a library type via diff --git a/docs/wiki/admin/monitoring-nodes.md b/docs/wiki/admin/monitoring-nodes.md new file mode 100644 index 000000000..36d7b6dec --- /dev/null +++ b/docs/wiki/admin/monitoring-nodes.md @@ -0,0 +1,374 @@ +--- +title: Monitoring Stream Nodes +description: How to read the admin Nodes page, when to re-probe a node's hardware, and how scratch-disk pressure affects transcode placement. +summary: How to read a node's Acceleration, Load, and Capacity blocks, the GPU re-probe action, scratch admission, and scraping node metrics with Prometheus. +tags: + - silo + - docs + - wiki + - nodes + - gpu + - monitoring +audience: + - operator +last_reviewed: 2026-08-27 +related: + - ../index.md + - ../deployment/docker.md +--- + +# Monitoring Stream Nodes + +**Settings -> Nodes** lists every registered proxy and transcode node with its +current health, its verified GPU capability, and its host resource usage. This +page explains what each reading means, which of them affect where playback is +placed, and what to do when the page disagrees with the hardware you know is in +the machine. + +Everything here applies to a distributed deployment. A single-node install has no +registered nodes, and its own resource usage appears on the admin dashboard +instead. + +## Reading a node + +Each node renders as one unit: a header naming the node, its state, its +co-location group, and its URL — with the enable switch and the actions beside +them — over three labelled blocks. **Acceleration** is what the node's FFmpeg +verified it can do, **Load** is the host underneath it, and **Capacity** is what +it is carrying against any caps it was given. Readings measured against a +ceiling draw a meter; a reading with no ceiling, or one nothing measured, draws +none — a bar at zero would read as "measured and idle", which is the one thing +an unmeasured value is not. + +When any node carries a group label, a **Groups** chip row above the sections +filters both of them at once — a group's proxy sits in one section and its +transcode nodes in the other, and the filter is how to see the pairing. A +group's chip carries an amber dot while an enabled member is unhealthy, because +that takes the whole group out of service: its transcode nodes stop taking +work, and a group with proxies of its own never falls back to another group's. + +### State + +The rail on a unit's left edge, the dot in its header, and one label carry the +node's state. **Disabled** is the administrator's switch: a disabled node is in +no pool, is never selected, stops counting against its co-location group, and +renders dimmed — its readings are whatever they were when it left the rotation. +**Healthy** and **Unhealthy** are the result of the last check. Every node is +polled every 30 seconds; a node that does not answer is routed around +immediately, and existing streams on it fail over on their next segment +request. **Checked**, in the Capacity block, is how long ago that poll ran — +hover for the exact time. + +Health and capability are independent. A node whose GPU driver broke stays +perfectly healthy: it still answers, it just encodes in software now. That is +what the Acceleration block and the drift badge exist to surface. + +### Acceleration + +The Acceleration block reports what the node's FFmpeg could actually *do* the last time +it was asked, not what its configuration names. Silo verifies hardware by running +a real single-frame encode on each candidate device, so the states are evidence, +not guesses: + +| Reading | Meaning | +|---|---| +| **QSV / VAAPI / NVENC**, green | The backend passed its FFmpeg probe on this node. Hover for the device it passed on. | +| **QSV / VAAPI / NVENC**, amber | The backend is configured and in use, but its probe *failed*. Hover for the reason FFmpeg gave. Transcodes will attempt it anyway, because an explicitly configured backend is honored verbatim. | +| **QSV / VAAPI / NVENC**, plain | The backend is in use but the node reported no verification for it — normal when a backend is configured on a node with no candidate devices to probe. | +| **SW** | No hardware backend verified: this node encodes in software. | +| **SW**, "devices not accessible" on hover | Every candidate device was *skipped* rather than probed, because this process cannot open any of them. This is the normal reading for a proxy node reading a cluster-wide `playback.hw_device` that points at the transcode nodes' cards. It is not a driver failure. | + +Below the badge, each render device the node can see is listed with its live +video-engine busyness — a meter and a session count per device — where a +measurement source exists; an unmeasured device shows a dash and no meter. + +**`stale`** in the Acceleration block means no health check has *confirmed* this inventory +for more than ten minutes. Checks run every 30 seconds and every response carries +the node's current capability hash, so roughly twenty checks in a row have to go +missing before the marker appears — it says the confirming sweep stopped, not +that the report is old. An old report is normal and is deliberately not flagged: +a node recomputes its snapshot every 15 minutes, the server refetches only when +the advertised hash changes, and a node whose GPU has not changed in a week is +serving a week-old report that is still true. An unhealthy node is never marked +stale, since it cannot confirm a report at all. + +**`Shared GPU`** marks a node whose physical card is also visible to another +registered node — two containers on one GPU, most often. Silo detects this from +each report's device identity (NVIDIA GPU UUID where available, otherwise the PCI +slot scoped to the host's boot id) and uses it when placing work: among transcode +nodes level on job count, the one whose *card* carries the fewest jobs wins. +Without that, spreading sessions across node records that share silicon would not +spread the work at all. + +**Drift**, an amber badge, means a capability refetch found the node's hardware +got *worse* than the report it replaced: a backend that used to pass its probe +now fails, or a render device is gone. Hover for the note. The badge records what it +lost and stays until exactly that comes back — the backend verifying again, the +card answering to one of the identities it had. It is not erased by a refetch +that merely loses nothing further, so a reboot or a reworded FFmpeg error cannot +make a standing regression look repaired; nor by the surviving card on a +multi-GPU node probing cleanly; nor by an unrelated GPU being added, which grows +the inventory without repairing anything. Cards lost one at a time must all +return. Re-probing the node is the +direct way to ask whether it is still true. It is a warning, not a routing input +— nothing in node selection reads it. + +### Load and Capacity + +**Load** is CPU, memory, the fullest sampled disk, and network throughput, +sampled by the node itself every five seconds and carried on its health +response. It is the current sample only; Silo keeps no history (see [Scraping +node metrics](#scraping-node-metrics) below). Network draws no meter: the +sampler reports bytes moved, never the link's speed, so there is no ceiling to +draw it against. + +**Capacity** is the node's concurrency (transcodes, or relayed streams on a +proxy) and, on proxy nodes, measured egress — each against its configured cap +where one is set. An uncapped reading shows the bare number and no meter, since +any bar would be measured against a ceiling the page invented. The readings +tint once a cap is reached, which is when the planner routes new work +elsewhere. + +An unhealthy node shows a dash rather than its last numbers: the sample predates +the check that failed, and a frozen CPU percentage looks exactly like a live one. +A dash on a *healthy* node means the node reported no sample — sampling is +Linux-only, and a node running an older build reports none. + +The disk reading is the fullest mount the node can see, and it tints when that +mount passes 85% used. A mount whose server stopped responding shows its last +good numbers rather than blocking the health response; a path the node cannot see +at all is reported as unmeasurable rather than as an empty disk. + +In a container these numbers are the *container's*, corrected against its cgroup +— but only where the cgroup actually caps something. A CPU quota or cpuset the +size of the whole machine restricts nothing, and both are ordinary: an +unconstrained container inherits a cpuset holding every online CPU, and a +deployment sized to the box writes a matching quota. Silo reads those as +uncapped and keeps reporting the host, because on a shared machine the CPU a +neighbor burns is CPU this node cannot have. A real cap — two cores of a +sixty-four core host — switches both the busy figure and the core count to the +cgroup's. This is also true on an LXC host running Docker nested inside it, which +needs three bind-mounts to read its own limits instead of the physical +machine's. See the LXC +notes in [Deploy Silo with Docker](../deployment/docker.md#node-metrics), +including the caveat that lxcfs only virtualizes `/proc/loadavg` when it runs +with loadavg accounting enabled (`lxcfs -l`; off by default on Proxmox), so +`load1` can remain the physical host's while CPU and memory are correct. + +## Re-probing a node's hardware + +Each node's header has a **Re-probe** action beside Check. It tells that node to throw +away its cached hardware verdicts, re-verify against live hardware, and hand the +fresh inventory straight back to the server. + +It exists because a *successful* probe is cached for the node's whole process +lifetime. That is deliberate — re-verifying on every playback request would put +FFmpeg executions on the playback path — but it means a node goes on reporting +hardware that has since stopped working: + +- **After a GPU driver upgrade, downgrade, or reinstall.** A card that worked + when the node started keeps reading as verified until the node restarts, even + once the new driver cannot encode a frame. +- **After changing which devices a container can open** (removing a `/dev/dri` + passthrough, switching the NVIDIA overlay, changing group membership). +- **After replacing an FFmpeg build in place**, where the path did not change. +- **To confirm a drift badge is still true**, rather than a transient failure + during a driver reload. + +The opposite direction needs no action. A *failed* GPU probe is cached for only +15 seconds, so a card that was broken when the node started and has since been +repaired verifies on its own; that flips the node's capability hash and the +server picks it up within one 15-minute snapshot. The exception is the tone-map +matrix, which caches any non-empty result permanently: a host whose GPU was +broken at start can stay software-only for tone mapping until it is re-probed or +restarted. + +Re-probe does not restart anything and does not reload configuration. It is +**refused on a node that is transcoding**: every probe ends in a real encode on +the GPU, and a card at its concurrent session limit fails that encode with an +error nothing can tell apart from a broken driver — which would flag working +hardware as a regression and take the node's tone-map inventory down with it. +Disable the node or wait for it to drain, then re-probe. "Transcoding" covers +everything that opens an encoder on that node — playback transcodes, +reconstructed sessions, prepared downloads, and hardware chapter-thumbnail +extraction — and the exclusion runs both ways: while a re-probe is in progress +the node refuses new GPU work with a 503 rather than queueing it, so the API +places that session elsewhere, and its own scheduled capability snapshot stands +down until the re-probe finishes. A scheduled snapshot does not refuse while +transcodes run, though: a node under sustained load would otherwise never +refresh its inventory at all. On an idle node the call can take a couple of +minutes, because it pays the full cold probe cost on purpose. + +Two outcomes are worth knowing: + +- On success the row's capability report, verified backends, and drift note are + all updated before the action returns — you are not waiting for the next + 30-second sweep. +- If the node's probe cannot finish, the action reports an error and the node + **keeps its previous report**. An unfinished probe is not evidence that the + hardware changed, so nothing is overwritten. Re-running it after the node + settles is safe. + +Force reload is a different tool: it makes a node re-read its configuration (and +tears down its sessions to do so). Use it after changing a node's acceleration +overrides; use Re-probe after changing the hardware or the driver under it. It +has no button today — it is API-only, `POST +/api/v1/admin/nodes/{id}/force-reload`. + +## Scratch admission + +A transcode writes HLS segments to its node's scratch directory for the entire +life of a session. A node that is nearly full therefore does not fail fast: it +accepts the session, streams for a while, and then dies with a write error after +the client has already committed to it. + +To avoid that, transcode selection **skips a node whose scratch volume is 95% or +more full**, preferring a node with headroom even when the full one is carrying +fewer jobs. Each time a node crosses into that state the server logs it once +(`component=nodepool`, "scratch volume nearly full") — once per transition, not +once per session. + +The rule is deliberately forgiving in two ways: + +- If skipping would leave *no* usable transcode node, the rule is ignored and + selection proceeds normally. Degraded playback beats no playback, and 95% was + never meant to darken a whole cluster. The log says which of the two happened: + a node that was genuinely kept out reads "excluded from selection", while a + node the guard had to admit anyway reads "still selected because no eligible + node has scratch headroom" and is accompanied by "transcode scratch guard + ignored". The second is an outage in progress — sessions are landing on a disk + that will fail mid-stream — and is worth paging on where the first is not. +- A node whose scratch fill cannot be read is never skipped: no sample, an + unmeasurable path, or numbers the node itself flagged as carried over from an + earlier probe. Removing capacity on a reading we do not have would be worse + than the failure it prevents. + +If you see this in your logs, the fix is on the node: raise the volume's size, +lower segment retention, or clear stale artifacts from the transcode directory. + +## Scraping node metrics + +Every Silo process — the API host and each node — publishes its own sample as +`streamapp_node_*` gauges on `/metrics`, on the same listener it serves traffic +from. It is unauthenticated, matching the API listener, because a scrape target +that needs a credential is a scrape target that goes unmonitored; what it exposes +is host resource counters, not media. + +Disk series are labeled by **role**, not by path — `mount="scratch"` and +`mount="library-1"`, `library-2`, ... — so an anonymous scrape cannot enumerate +where your media lives. A node's `/health` is unauthenticated for the same +reason and reports the same roles without paths, which is what the Nodes page +draws from. The real paths are behind admin authentication on +`GET /api/v1/admin/system/resources` and behind a bearer token on each node's +`/status`. Library ordering is stable for a given configuration, but it is +positional: adding a library root can renumber the series after it, so alert on +`mount="scratch"` by name and on the library mounts by aggregate. A mount that +goes unavailable keeps its number rather than renumbering the ones after it. + +At most eight mounts are sampled per host, scratch first. The cap bounds +probing, not just reporting — an unresponsive network mount parks a `statfs` +call that cannot be interrupted — so roots past it are not sampled, and the +process logs how many were left out (`component=nodemetrics`). + +A minimal scrape config, with one job per role: + +```yaml +scrape_configs: + - job_name: silo-api + metrics_path: /metrics + static_configs: + - targets: ["silo-api.internal:8080"] + labels: + silo_role: api + + - job_name: silo-nodes + metrics_path: /metrics + static_configs: + - targets: + - "transcode-1.internal:8081" + - "transcode-2.internal:8081" + labels: + silo_role: transcode + - targets: + - "proxy-1.internal:8082" + labels: + silo_role: proxy +``` + +Replace the hostnames and ports with your own; the metrics path is the same on +every role. The two alerts worth having first are the scratch volume filling and +a node's GPU going quiet while its CPU does not: + +```yaml +groups: + - name: silo-nodes + rules: + - alert: SiloScratchVolumeFilling + expr: | + streamapp_node_disk_used_bytes{mount="scratch"} + / streamapp_node_disk_total_bytes{mount="scratch"} > 0.9 + for: 15m + annotations: + summary: "Silo scratch volume above 90% on {{ $labels.instance }}" + + - alert: SiloNodeCPUSaturated + expr: streamapp_node_cpu_percent > 90 + for: 15m + annotations: + summary: "Silo node CPU pegged on {{ $labels.instance }} — check the Acceleration block for a failed probe" + + - alert: SiloDiskMeasurementStale + expr: streamapp_node_disk_stale == 1 + for: 15m + annotations: + summary: "Silo has not measured {{ $labels.mount }} on {{ $labels.instance }} for a while — its used/total figures are carried over" +``` + +That last one matters more than it looks. A mount whose probe stops returning +keeps exporting its last real used and total bytes, because dropping the series +would blank the panel for a network mount that is merely slow to answer — which +happens routinely. The numbers are genuine, only old, so `SiloScratchVolumeFilling` +above still fires on a volume that was nearly full when measurement stopped. What +it cannot do is notice a volume that stopped answering at 40% and has been +filling since. `streamapp_node_disk_stale` is what closes that gap: it is `1` +whenever the used and total beside it are carried over, and `0` when they are +current. A mount Silo has never measured at all exports nothing — not even a +staleness series, since there would be no numbers for it to qualify. + +A GPU nothing could measure exports no engine gauges at all rather than zeros — +a Prometheus sample carries no `source` to qualify them, and a zero would read as +idle on a card that may be busy and merely unobservable. Its session count still +ships, since that comes from Silo's own accounting rather than from a driver. + +The GPU gauges (`streamapp_node_gpu_video_busy_percent`, +`streamapp_node_gpu_render_busy_percent`, `streamapp_node_gpu_busy_percent`, +`streamapp_node_gpu_sessions`, `streamapp_node_gpu_vram_used_bytes`, +`streamapp_node_gpu_vram_total_bytes`) are labeled by `device`. On Intel and AMD +the engine gauges measure Silo's own FFmpeg processes only, so a card shared with +anything outside Silo reads as less busy than it is; on NVIDIA they come from +`nvidia-smi` and are whole-GPU. A device that nothing could measure this interval +publishes no VRAM series rather than a zero. + +`streamapp_node_gpu_busy_percent` is the card's own utilization, other tenants +included, and it ships wherever a source reports one — `nvidia-smi` today. It is +the series to alert on for a shared GPU: the engine gauges can show Silo idle +while the card it is planned onto is saturated by something else. + +If `nvidia-smi` fails five samples running, Silo stops calling it — a host +without the NVIDIA toolkit would otherwise spawn a doomed subprocess every few +seconds forever. It is not retired for good: one probationary call goes out every +ten minutes, so a driver reset or a toolkit installed after startup is picked up +on its own. `POST /api/v1/admin/nodes/{id}/reprobe` puts it back in service +immediately, which is the faster path when you have just fixed the driver +yourself. + +## Source References + +- `internal/nodepool` — health sweep, capability refetch, drift detection, and + transcode placement including the scratch guard. +- `internal/nodemetrics` — host and GPU sampling, and the `streamapp_node_*` + collector. +- `internal/playback/gpudetect.go` — the hardware verification probes behind the + Acceleration block. +- [Admin API](../../admin-api.md) — the `GET /api/v1/admin/nodes` field table and + the re-probe endpoint. diff --git a/docs/wiki/deployment/docker.md b/docs/wiki/deployment/docker.md index fb5676293..0cc811f25 100644 --- a/docs/wiki/deployment/docker.md +++ b/docs/wiki/deployment/docker.md @@ -209,6 +209,98 @@ NVIDIA_GPU_COUNT=1 Windows uses `;` instead of `:` between entries in `COMPOSE_FILE`. +## Node metrics + +Every Silo process — the API host and each proxy or transcode node — samples its +own CPU, memory, disk, network and GPU usage every five seconds. The numbers +appear on the admin Nodes page, in each node's `/health` and `/status` +responses, and as `streamapp_node_*` gauges on that process's `/metrics` +endpoint. `/metrics` is unauthenticated on node listeners, matching the API +listener; it exposes host resource counters, not media. Disk series are labeled +by role — `mount="scratch"`, `mount="library-1"` — rather than by path, so an +anonymous scrape cannot enumerate where your media lives. A node's `/health` is +unauthenticated for the same reason and withholds paths on the same terms. The +paths themselves are reported by the admin-authenticated +`GET /api/v1/admin/system/resources` and by each node's bearer-authed `/status`. + +At most eight mounts are sampled per host — the transcode scratch directory +first, then library roots in order. The cap bounds probing, not just reporting: +each mount costs a `statfs` call per interval, and one on an unresponsive +network mount cannot be interrupted. A deployment with more roots than that logs +how many go unsampled rather than quietly reporting a subset. + +Sampling is current-sample only. Silo stores no history — point Prometheus at +`/metrics` if you want trends or alerts. + +**Works out of the box.** System metrics and per-device GPU busyness need no +extra packages, no privileges and no configuration. GPU usage comes from DRM +fdinfo, which the kernel exposes for any process holding a `/dev/dri` device, so +the standard VA-API overlay above is enough for Intel (i915, xe) and AMD +(amdgpu). It measures Silo's own ffmpeg processes only: a GPU shared with +something outside Silo will read as less busy than it is. Each device reports a +`source` field saying which measurement it used. + +**NVIDIA needs the container toolkit.** The proprietary driver implements no +fdinfo, so `nvidia-smi` is the only signal — and it is a whole-GPU one, so it +also sees other tenants. It is already required for NVENC, so the NVIDIA overlay +above gives you GPU utilization, encoder/decoder utilization and VRAM with no +extra step. If the binary is missing or fails repeatedly, that device reports +`source: unavailable` and nothing else degrades. + +**Whole-GPU Intel sampling is not implemented yet.** Reading Intel utilization +across all tenants needs `intel_gpu_top`, which requires `CAP_PERFMON` (or root) +plus a permissive `kernel.perf_event_paranoid` — privileges a plain `/dev/dri` +passthrough container does not have and should not be given by default. Until +that lands, Intel GPUs report the fdinfo baseline only, and `total_busy_pct` is +absent rather than zero. + +**What the numbers mean inside a container.** `/proc/stat` and `/proc/meminfo` +describe the *host*, not the container, so Silo corrects both against the +container's cgroup. Memory reports the cgroup limit and working set (page cache +excluded), which is what the kernel will OOM-kill against. CPU reports the +cgroup's own consumption against its own quota, so a container limited to +`cpus: 2` on a 64-core host reads 100% when it is pegged — not the 3% of the +host machine that same work amounts to — and `cores` is the quota, not the +host's core count. `/proc/net/dev` is +already per-namespace, so network throughput is the container's own traffic. +Disk figures come from `statfs` on the paths the container can see — the +transcode scratch directory on every node, plus the configured library roots on +the API host. A mount that stops responding reports its last good numbers marked +`stale` and never delays a health response; a path a node cannot see at all is +reported `unavailable` rather than as an empty disk. + +**LXC hosts running Docker nested inside them.** The cgroup correction above +only works when the limit is visible on *this* container's own cgroup. On an +LXC host, a Docker container nested inside the LXC sees the raw kernel's +`/proc/stat`, `/proc/loadavg`, and `/proc/meminfo` — the physical machine's +totals, not the LXC's — while its own cgroup shows no limit at all, because the +LXC's cap lives on an ancestor cgroup outside the nested container's namespace. +lxcfs, which every LXC container mounts for exactly these files, virtualizes +them to the LXC's own limits, so bind-mounting that virtualized view into the +Docker container fixes it: + +```yaml +volumes: + - /proc/meminfo:/host/proc/meminfo:ro + - /proc/stat:/host/proc/stat:ro + - /proc/loadavg:/host/proc/loadavg:ro +``` + +No setting turns this on: the sampler uses each file under `/host/proc` when it +is present and falls back to its own `/proc` when it is not, per file, so the +mounts take effect on the next sample and removing one reverts it. + +These three mounts are only useful when the Docker host is itself an LXC/lxcfs +container — a bare-metal or VM Docker host has nothing extra to gain from them, +since its own `/proc` is already correct or already cgroup-corrected. Without +them, a node running nested this way reports the bare-metal host's CPU, memory, +and load totals instead of its own. + +One caveat: lxcfs only virtualizes `/proc/loadavg` when it runs with loadavg +accounting enabled (its `-l` flag; on Proxmox, `lxcfs` defaults to off). With +it off, cores and memory are container-scoped but `load1` remains the physical +host's — visibly higher than the container's core count under host contention. + ## Optional Meilisearch PostgreSQL full-text search needs no extra service. To offer Meilisearch as an @@ -233,9 +325,15 @@ alternative provider: 3. In **Admin > Settings > Search**, select Meilisearch, set the URL to `http://meilisearch:7700`, enter the same key as the API key, test the connection, and save. -4. Restart Silo and rebuild the catalog search index from the same page. +4. Restart Silo. Background search maintenance builds the catalog search index + automatically and retries failures every minute. The Search status panel + shows whether Meilisearch, keyword-only Meilisearch, or PostgreSQL is serving + requests while the build runs; the manual rebuild action remains available. Silo continues to use PostgreSQL full-text search until Meilisearch is selected. +Settings that change the index format, including enabling meaning-based search, +also trigger an automatic background rebuild after restart. A compatible older +Meilisearch index keeps serving keyword results while its replacement is built. ## External PostgreSQL and Redis diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 28b5cc90a..7d870035e 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -37,6 +37,8 @@ rewriting. synced library collections sourced from TMDB, Trakt, and MDBList. - [Local NFO Metadata](admin/nfo-local-metadata.md) - Supported NFO sidecar fields, how they merge with online providers, and the naming-supplies-structure contract. +- [Monitoring Stream Nodes](admin/monitoring-nodes.md) - Reading the Nodes page columns, re-probing + a node's GPU after a driver change, scratch-disk admission, and scraping node metrics. ## Deployment diff --git a/internal/adminjob/library_refresh_language_test.go b/internal/adminjob/library_refresh_language_test.go index 311b6ff48..6be6a0dbc 100644 --- a/internal/adminjob/library_refresh_language_test.go +++ b/internal/adminjob/library_refresh_language_test.go @@ -31,6 +31,7 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { matchedID := fmt.Sprintf("lang-matched-%d", suffix) missingLogoID := fmt.Sprintf("logo-missing-%d", suffix) tvdbLogoID := fmt.Sprintf("logo-tvdb-%d", suffix) + legacyTVDBLogoPathID := fmt.Sprintf("logo-tvdb-path-%d", suffix) var folderID int if err := pool.QueryRow(ctx, ` @@ -41,7 +42,7 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { t.Fatalf("seed folder: %v", err) } t.Cleanup(func() { - _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{mismatchID, matchedID, missingLogoID, tvdbLogoID}) + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{mismatchID, matchedID, missingLogoID, tvdbLogoID, legacyTVDBLogoPathID}) _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) }) @@ -52,6 +53,7 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { {matchedID, "da", "/l.png", "tmdb://logo/matched.png"}, {missingLogoID, "da", "", ""}, {tvdbLogoID, "da", "/l.png", "tvdb://artwork/illustrated.png"}, + {legacyTVDBLogoPathID, "da", "tvdb/123/logo/clear-art.png", "tmdb://logo/legacy-path.png"}, } { if _, err := pool.Exec(ctx, ` INSERT INTO media_items ( @@ -77,7 +79,7 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { t.Fatalf("ListLibraryItems: %v", err) } - var sawMismatch, sawMatched, sawMissingLogo, sawTVDBLogo bool + var sawMismatch, sawMatched, sawMissingLogo, sawTVDBLogo, sawLegacyTVDBLogoPath bool for _, item := range items { switch item.ContentID { case mismatchID: @@ -88,6 +90,8 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { sawMissingLogo = true case tvdbLogoID: sawTVDBLogo = true + case legacyTVDBLogoPathID: + sawLegacyTVDBLogoPath = true } } if !sawMismatch { @@ -102,6 +106,9 @@ func TestQuickRefreshListsLanguageOrArtworkIncompleteItems(t *testing.T) { if !sawTVDBLogo { t.Errorf("quick refresh must include an item whose existing logo came from TVDB clear-art") } + if !sawLegacyTVDBLogoPath { + t.Errorf("quick refresh must include an item whose legacy logo path identifies TVDB clear-art") + } } func TestQuickRefreshIgnoresHistoricalSecondaryStaleIDs(t *testing.T) { diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 1a650092a..0ccc05e60 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -20,6 +20,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" "golang.org/x/sync/errgroup" "github.com/Silo-Server/silo-server/internal/access" @@ -116,13 +117,22 @@ type ImpersonationService interface { // AdminHandler handles admin-only HTTP endpoints for user management, // session listing, unmatched files, and system stats. type AdminHandler struct { - userRepo UserRepository - pool *pgxpool.Pool - SessionsLoader *PlaybackSessionsLoader - storeProv userstore.UserStoreProvider - accountProvisioner *auth.AccountProvisioner - DetailSvc *catalog.DetailService - StatsSource AdminStatsSource + userRepo UserRepository + pool *pgxpool.Pool + SessionsLoader *PlaybackSessionsLoader + storeProv userstore.UserStoreProvider + accountProvisioner *auth.AccountProvisioner + DetailSvc *catalog.DetailService + StatsSource AdminStatsSource + // WatchProviders lists the registered watch providers for the uncached + // stats path. Nil on a deployment without the watchsync registry, which + // degrades to "only providers with stored activity". + WatchProviders WatchProviderLister + PlaybackActivitySource AdminPlaybackActivitySource + TopActivitySource AdminTopActivitySource + TimeseriesSource AdminTimeseriesSource + DownloadsStatsSource AdminDownloadsStatsSource + RedisClient *redis.Client // health reporting only; nil means this deployment runs without Redis Config *config.Config EventBus cache.EventBus EventsHub *evt.Hub @@ -140,6 +150,19 @@ type AdminHandler struct { OnServerSettingUpdated func(ctx context.Context, key, value string) RestartStatus *ServerRestartStatusTracker CatalogSearchStatus catalog.CatalogSearchStatusProvider + // logLevelCounts caches the 24h error/warning tallies served on + // /admin/server/status. The dashboard polls that route every 15s, and the + // counts are only ever read as a rough signal, so re-counting per request + // would be pure waste. Nil when the handler was built without the + // constructor (tests): the counts are then simply uncached. + logLevelCounts *cache.TTLCache[adminLogLevelCounts] + // PublicStorageConfigured reports whether the public object-storage client + // is active in this process — the same condition that gates branding asset + // uploads (branding.Service.HasStorage) and the metadata image cacher, both + // of which are only wired when the public S3 client exists. A nil func + // means "not configured". See publicBucketConfigured for the full rule, + // which also accepts a bucket that is saved but not live yet. + PublicStorageConfigured func() bool } // NewAdminHandler creates a new AdminHandler backed by the given @@ -154,6 +177,7 @@ func NewAdminHandler( pool: pool, storeProv: storeProv, accountProvisioner: auth.NewAccountProvisioner(userRepo, storeProv), + logLevelCounts: cache.NewTTLCache[adminLogLevelCounts](), } } @@ -1341,7 +1365,7 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) { } resp = stats } else if h.pool != nil { - stats, err := queryAdminStats(r.Context(), h.pool) + stats, err := queryAdminStats(r.Context(), h.pool, h.WatchProviders) if err != nil { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get stats") return @@ -1628,6 +1652,22 @@ func (h *AdminHandler) HandleGetEffectiveSettings(w http.ResponseWriter, r *http writeJSON(w, http.StatusOK, effective) } +type restartKeysResponse struct { + Keys []string `json:"keys"` + Prefixes []string `json:"prefixes"` +} + +// HandleGetRestartKeys handles GET /admin/settings/restart-keys. The registry +// is compiled into the binary (internal/config), so the response only changes +// across deploys; the admin UI caches it and uses it to badge the fields whose +// saved value waits on a restart. +func (h *AdminHandler) HandleGetRestartKeys(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, restartKeysResponse{ + Keys: config.RestartRequiredKeys(), + Prefixes: config.RestartRequiredPrefixes(), + }) +} + type sensitiveStatusResponse struct { Configured []string `json:"configured"` ManagedByEnv []string `json:"managed_by_env,omitempty"` @@ -1684,8 +1724,10 @@ func (h *AdminHandler) HandleGetSensitiveStatus(w http.ResponseWriter, r *http.R type adminSettingResponse struct { Key string `json:"key"` Value string `json:"value"` - // RestartRequired reports whether the saved value only takes effect - // after a server restart (set on update responses only). + // RestartRequired reports whether the value only takes effect after a + // server restart. It is populated from the compiled restart-key registry + // on read responses as well as on updates, so the admin UI never has to + // hand-copy the list into hint text. RestartRequired bool `json:"restart_required,omitempty"` } @@ -2205,7 +2247,11 @@ func (h *AdminHandler) HandleGetSetting(w http.ResponseWriter, r *http.Request) } if value, ok := h.BootstrapSensitiveValues[key]; ok && value != "" { - writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: value}) + writeJSON(w, http.StatusOK, adminSettingResponse{ + Key: key, + Value: value, + RestartRequired: config.RestartRequired(key), + }) return } @@ -2219,7 +2265,11 @@ func (h *AdminHandler) HandleGetSetting(w http.ResponseWriter, r *http.Request) return } - writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: value}) + writeJSON(w, http.StatusOK, adminSettingResponse{ + Key: key, + Value: value, + RestartRequired: config.RestartRequired(key), + }) } type updateSettingRequest struct { @@ -2236,7 +2286,107 @@ type updateSettingsResponse struct { RestartRequiredKeys []string `json:"restart_required_keys,omitempty"` } -func (h *AdminHandler) normalizeBatchSetting(ctx context.Context, key, value string) (string, string, error) { +// settingMetadataCacheImages copies provider artwork into the public bucket, so +// it cannot be turned on without one. settingPublicBucketLegacy is the +// pre-rename alias config.db_loader still falls back to. +const ( + settingMetadataCacheImages = "metadata.cache_images" + settingPublicBucket = "s3.public_bucket" + settingPublicBucketLegacy = "s3.operational_bucket" +) + +// errCodeStorageUnavailable is the API error code for a setting that needs +// object storage this deployment has not configured. +const errCodeStorageUnavailable = "storage_unavailable" + +// errPublicStorageUnavailable is returned when a write would leave +// metadata.cache_images enabled with no public bucket anywhere: the image cacher +// is wired off the public S3 client, so caching could never start. +var errPublicStorageUnavailable = errors.New( + "S3 image caching requires a configured public storage bucket: metadata.cache_images cannot be " + + "enabled while s3.public_bucket is empty (Infrastructure \u2192 Public storage)") + +// publicBucketConfigured reports whether a public object-storage bucket exists +// from the server's point of view. A bucket that is only saved counts: an admin +// editing an inactive-but-saved deployment is one restart away. Only the live +// client proves caching starts immediately, so the UI still badges the pending +// restart — but the API must not block a legitimate save. +// +// This is the single-key endpoint's view: it writes one setting against +// whatever is already stored. The batch endpoint uses +// prospectivePublicBucketConfigured instead, because a bucket written or +// cleared by the same request has not reached the store yet. +func (h *AdminHandler) publicBucketConfigured(ctx context.Context) bool { + if h == nil { + return false + } + if h.PublicStorageConfigured != nil && h.PublicStorageConfigured() { + return true + } + for _, key := range []string{settingPublicBucket, settingPublicBucketLegacy} { + if h.BootstrapSensitiveConfigured[key] && + strings.TrimSpace(h.BootstrapSensitiveValues[key]) != "" { + return true + } + if h.SettingsRepo == nil { + continue + } + if stored, err := h.SettingsRepo.Get(ctx, key); err == nil && strings.TrimSpace(stored) != "" { + return true + } + } + return false +} + +// prospectivePublicBucketConfigured reports whether the settings a batch is +// about to persist still describe a public bucket. effective is the stored +// state overlaid with the batch and the environment and run through +// config.EffectiveAdminSettings, so the legacy s3.operational_bucket fallback +// LoadFromDB applies is already folded into settingPublicBucket; changed is the +// batch itself. +// +// A bucket key the batch does not mention leaves the live public client as +// evidence, because the bucket may come from a source the settings store cannot +// see. An explicitly empty bucket in the batch is a clear, not an absence: the +// live client only reflects what this process booted with, so it cannot vouch +// for storage the saved settings no longer describe. +func (h *AdminHandler) prospectivePublicBucketConfigured(effective, changed map[string]string) bool { + if h == nil { + return false + } + if strings.TrimSpace(effective[settingPublicBucket]) != "" { + return true + } + for _, key := range []string{settingPublicBucket, settingPublicBucketLegacy} { + if _, cleared := changed[key]; cleared { + return false + } + } + return h.PublicStorageConfigured != nil && h.PublicStorageConfigured() +} + +// validateProspectiveImageCaching rejects a batch whose final state leaves image +// caching enabled with nowhere to write. Both directions matter: enabling +// caching while clearing the bucket in the same request, and clearing the bucket +// while stored settings already have caching on. Either way the image cacher +// cannot start after the next restart. +func (h *AdminHandler) validateProspectiveImageCaching(effective, changed map[string]string) error { + // ParseBool matches config.LoadFromDB, which reads the stored value the same + // way; anything it rejects is not a deployment running with caching on. + enabled, _ := strconv.ParseBool(strings.TrimSpace(effective[settingMetadataCacheImages])) + if !enabled { + return nil + } + if h.prospectivePublicBucketConfigured(effective, changed) { + return nil + } + return errPublicStorageUnavailable +} + +func (h *AdminHandler) normalizeBatchSetting( + ctx context.Context, + key, value string, +) (string, string, error) { if strings.HasPrefix(key, "ratelimit.") { return "", "bad_request", fmt.Errorf("%s is managed by /admin/rate-limits/config", key) } @@ -2260,7 +2410,7 @@ func (h *AdminHandler) normalizeBatchSetting(ctx context.Context, key, value str case diagnostics.KeyUploadsEnabled: if normalized == "true" { if err = h.validateDiagnosticsUploadsEnabled(ctx); err != nil { - return "", "storage_unavailable", err + return "", errCodeStorageUnavailable, err } } case diagnostics.KeyMaxBundleBytes, @@ -2503,6 +2653,7 @@ func (h *AdminHandler) HandleUpdateSettings(w http.ResponseWriter, r *http.Reque after map[string]string effectiveChanges map[string]bool validationErr error + validationCode string ) err := updateServerSettingsAtomically(r.Context(), h.SettingsRepo, func(stored map[string]string) (map[string]string, error) { @@ -2511,13 +2662,22 @@ func (h *AdminHandler) HandleUpdateSettings(w http.ResponseWriter, r *http.Reque prospective[key] = value } activeProspective := h.activeAdminSettings(prospective) + before := h.effectiveAdminSettings(stored) + after = h.effectiveAdminSettings(prospective) + // Cross-field checks run against the complete prospective state, so a + // value the batch clears is gone even when the store still has it and + // the current process is still running on it. + if err := h.validateProspectiveImageCaching(after, normalized); err != nil { + validationErr = err + validationCode = errCodeStorageUnavailable + return nil, err + } validationSnapshot := adminSettingsValidationSnapshot(activeProspective, normalized) if err := validateProspectiveAdminSettings(validationSnapshot, h.RedisBootstrapAvailable); err != nil { validationErr = err + validationCode = "invalid_settings" return nil, err } - before := h.effectiveAdminSettings(stored) - after = h.effectiveAdminSettings(prospective) writes := make(map[string]string, len(normalized)) effectiveChanges = make(map[string]bool, len(normalized)) for key, value := range normalized { @@ -2532,7 +2692,7 @@ func (h *AdminHandler) HandleUpdateSettings(w http.ResponseWriter, r *http.Reque return writes, nil }) if validationErr != nil { - writeError(w, http.StatusBadRequest, "invalid_settings", validationErr.Error()) + writeError(w, http.StatusBadRequest, validationCode, validationErr.Error()) return } if err != nil { @@ -2560,8 +2720,11 @@ func (h *AdminHandler) HandleUpdateSettings(w http.ResponseWriter, r *http.Reque restartKeys = append(restartKeys, key) } } - if len(restartKeys) > 0 { - h.markServerRestartRequired("server_settings") + // Per-key reasons ("setting:") so the admin UI can scope a pending + // restart to the subsystem the key belongs to instead of warning on every + // tile for any settings save. + for _, restartKey := range restartKeys { + h.markServerRestartRequired("setting:" + restartKey) } writeJSON(w, http.StatusOK, updateSettingsResponse{ Values: responseValues, @@ -2657,6 +2820,11 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques return } req.Value = strconv.FormatBool(enabled) + case settingMetadataCacheImages: + if req.Value == "true" && !h.publicBucketConfigured(r.Context()) { + writeError(w, http.StatusBadRequest, errCodeStorageUnavailable, errPublicStorageUnavailable.Error()) + return + } case diagnostics.KeyUploadsEnabled: enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value)) if err != nil { @@ -2666,7 +2834,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques req.Value = strconv.FormatBool(enabled) if enabled { if err := h.validateDiagnosticsUploadsEnabled(r.Context()); err != nil { - writeError(w, http.StatusBadRequest, "storage_unavailable", err.Error()) + writeError(w, http.StatusBadRequest, errCodeStorageUnavailable, err.Error()) return } } @@ -2839,6 +3007,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques after map[string]string effectiveChanged bool validationErr error + validationCode string ) err := updateServerSettingsAtomically(r.Context(), h.SettingsRepo, func(stored map[string]string) (map[string]string, error) { @@ -2847,14 +3016,28 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques // This legacy route can only change one key, so enforcing every // cross-field invariant would make paired settings impossible to // establish or clear one write at a time. Per-key validation above - // remains strict; Redis transport is the one durable prerequisite - // that may not be broken by a single-key write. + // remains strict; the durable prerequisites are the exception — a + // single-key write may not break them. if key == "redis.url" { if err := config.ValidateRedisRateLimitTransport( h.activeAdminSettings(prospective), h.RedisBootstrapAvailable, ); err != nil { validationErr = err + validationCode = "invalid_settings" + return nil, err + } + } + // Image caching's bucket is the other durable prerequisite: clearing + // it here while metadata.cache_images is stored on would leave the + // cacher unable to start after restart. Disable caching first. + if key == settingPublicBucket || key == settingPublicBucketLegacy { + if err := h.validateProspectiveImageCaching( + h.effectiveAdminSettings(prospective), + map[string]string{key: req.Value}, + ); err != nil { + validationErr = err + validationCode = errCodeStorageUnavailable return nil, err } } @@ -2868,7 +3051,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques return nil, nil }) if validationErr != nil { - writeError(w, http.StatusBadRequest, "invalid_settings", validationErr.Error()) + writeError(w, http.StatusBadRequest, validationCode, validationErr.Error()) return } if err != nil { @@ -2886,7 +3069,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques } restartRequired := effectiveChanged && config.RestartRequired(key) if restartRequired { - h.markServerRestartRequired("server_settings") + h.markServerRestartRequired("setting:" + key) } if sensitiveSettingKeys[key] { writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, RestartRequired: restartRequired}) diff --git a/internal/api/handlers/admin_cache_images_setting_test.go b/internal/api/handlers/admin_cache_images_setting_test.go new file mode 100644 index 000000000..2d9f3eab0 --- /dev/null +++ b/internal/api/handlers/admin_cache_images_setting_test.go @@ -0,0 +1,319 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-chi/chi/v5" +) + +// metadata.cache_images copies provider artwork into the public bucket, so both +// write paths reject enabling it when no public bucket exists anywhere. A saved +// but not-yet-active bucket counts: the setup wizard configures the bucket and +// enables caching in one batch, and the UI badges the pending restart. + +func cacheImagesHandler(settings *fakeServerSettingsStore, storage bool) *AdminHandler { + h := &AdminHandler{SettingsRepo: settings} + if storage { + h.PublicStorageConfigured = func() bool { return true } + } + return h +} + +func updateCacheImagesBatch(h *AdminHandler, values string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h.HandleUpdateSettings(rec, httptest.NewRequest( + http.MethodPut, + "/admin/settings", + strings.NewReader(`{"values":{`+values+`}}`), + )) + return rec +} + +func updateSingleSetting(h *AdminHandler, key, value string) *httptest.ResponseRecorder { + router := chi.NewRouter() + router.Put("/admin/settings/{key}", h.HandleUpdateSetting) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest( + http.MethodPut, + "/admin/settings/"+key, + strings.NewReader(`{"value":"`+value+`"}`), + )) + return rec +} + +func updateCacheImagesSingle(h *AdminHandler, value string) *httptest.ResponseRecorder { + return updateSingleSetting(h, "metadata.cache_images", value) +} + +func assertStorageUnavailable(t *testing.T, rec *httptest.ResponseRecorder) { + t.Helper() + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + var body errorResponse + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if body.Error != "storage_unavailable" { + t.Fatalf("error code = %q, want storage_unavailable; body=%#v", body.Error, body) + } + if !strings.Contains(body.Message, "S3 image caching requires a configured public storage bucket") { + t.Fatalf("error message = %q", body.Message) + } +} + +func TestCacheImagesEnableRequiresPublicBucket(t *testing.T) { + const cacheImagesTrue = `"metadata.cache_images":"true"` + + t.Run("batch with active storage", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), cacheImagesTrue) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("batch with saved bucket but inactive store", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{"s3.public_bucket": "silo-public"}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, false), cacheImagesTrue) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // The setup wizard writes both in one request, before any restart. + t.Run("batch configuring the bucket in the same request", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, false), + cacheImagesTrue+`,"s3.public_endpoint":"https://s3.example.com","s3.public_bucket":"silo-public"`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" || + settings.values["s3.public_bucket"] != "silo-public" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // The batch is validated inside the settings transaction, so it reads the + // stored values but must not write any of them. + t.Run("batch with no bucket anywhere", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, false), cacheImagesTrue) + assertStorageUnavailable(t, rec) + if settings.setManyCalls != 0 || settings.setCalls != 0 { + t.Fatalf("write attempted: setMany=%d set=%d", settings.setManyCalls, settings.setCalls) + } + if _, stored := settings.values["metadata.cache_images"]; stored { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // Clearing the bucket in the same batch must not count as configuring one. + t.Run("batch clearing the bucket while enabling", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, false), + cacheImagesTrue+`,"s3.public_bucket":""`) + assertStorageUnavailable(t, rec) + }) + + // The batch persists atomically, so a bucket it clears is gone even though + // the store — and the process still running on it — have one right now. + t.Run("batch clearing a stored bucket while enabling", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{"s3.public_bucket": "silo-public"}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), + cacheImagesTrue+`,"s3.public_bucket":""`) + assertStorageUnavailable(t, rec) + if settings.setManyCalls != 0 || settings.setCalls != 0 { + t.Fatalf("write attempted: setMany=%d set=%d", settings.setManyCalls, settings.setCalls) + } + if settings.values["s3.public_bucket"] != "silo-public" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // The dual: caching is already on and the batch never resubmits it, so only + // the complete prospective state catches the storage disappearing. + t.Run("batch clearing the bucket while caching stays on", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_bucket": "silo-public", + }} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), `"s3.public_bucket":""`) + assertStorageUnavailable(t, rec) + if settings.values["s3.public_bucket"] != "silo-public" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("batch clearing the public endpoint and bucket while caching stays on", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_endpoint": "https://s3.example.com", + "s3.public_bucket": "silo-public", + }} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), + `"s3.public_endpoint":"","s3.public_bucket":""`) + assertStorageUnavailable(t, rec) + if settings.values["s3.public_bucket"] != "silo-public" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // Turning caching off in the same batch is the documented way out. + t.Run("batch disabling caching while clearing the bucket", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_endpoint": "https://s3.example.com", + "s3.public_bucket": "silo-public", + }} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), + `"metadata.cache_images":"false","s3.public_endpoint":"","s3.public_bucket":""`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "false" || + settings.values["s3.public_bucket"] != "" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // LoadFromDB falls back to the pre-rename bucket key, so clearing only the + // canonical one still leaves the cacher a bucket. + t.Run("batch clearing the bucket while the legacy bucket remains", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_endpoint": "https://s3.example.com", + "s3.public_bucket": "silo-public", + "s3.operational_bucket": "silo-legacy", + }} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, true), `"s3.public_bucket":""`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["s3.public_bucket"] != "" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("batch disable with no bucket", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{"metadata.cache_images": "true"}} + rec := updateCacheImagesBatch(cacheImagesHandler(settings, false), `"metadata.cache_images":"false"`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "false" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single with active storage", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesSingle(cacheImagesHandler(settings, true), "true") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single with saved bucket but inactive store", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{"s3.public_bucket": "silo-public"}} + rec := updateCacheImagesSingle(cacheImagesHandler(settings, false), "true") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single with an environment-supplied bucket", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + h := cacheImagesHandler(settings, false) + h.BootstrapSensitiveConfigured = map[string]bool{"s3.public_bucket": true} + h.BootstrapSensitiveValues = map[string]string{"s3.public_bucket": "silo-public"} + rec := updateCacheImagesSingle(h, "true") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "true" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single with no bucket anywhere", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + rec := updateCacheImagesSingle(cacheImagesHandler(settings, false), "true") + assertStorageUnavailable(t, rec) + if settings.setCalls != 0 { + t.Fatalf("Set calls = %d, want 0", settings.setCalls) + } + if _, stored := settings.values["metadata.cache_images"]; stored { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single disable with no bucket", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{"metadata.cache_images": "true"}} + rec := updateCacheImagesSingle(cacheImagesHandler(settings, false), "false") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["metadata.cache_images"] != "false" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + // The legacy single-key route must hold the same line as the batch: + // clearing the bucket while caching is stored on strands the cacher. + t.Run("single bucket clear while caching is on", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_bucket": "silo-public", + }} + rec := updateSingleSetting(cacheImagesHandler(settings, true), "s3.public_bucket", "") + assertStorageUnavailable(t, rec) + if settings.values["s3.public_bucket"] != "silo-public" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single bucket clear while caching is off", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "s3.public_bucket": "silo-public", + }} + rec := updateSingleSetting(cacheImagesHandler(settings, false), "s3.public_bucket", "") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["s3.public_bucket"] != "" { + t.Fatalf("stored values = %#v", settings.values) + } + }) + + t.Run("single bucket change to a new value while caching is on", func(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{ + "metadata.cache_images": "true", + "s3.public_bucket": "silo-public", + }} + rec := updateSingleSetting(cacheImagesHandler(settings, false), "s3.public_bucket", "silo-art") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if settings.values["s3.public_bucket"] != "silo-art" { + t.Fatalf("stored values = %#v", settings.values) + } + }) +} diff --git a/internal/api/handlers/admin_dashboard_layout.go b/internal/api/handlers/admin_dashboard_layout.go new file mode 100644 index 000000000..d69d5f663 --- /dev/null +++ b/internal/api/handlers/admin_dashboard_layout.go @@ -0,0 +1,214 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "time" + + "github.com/jackc/pgx/v5" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" +) + +// maxDashboardLayoutBytes bounds the PUT body. The layout is a short list of +// widget ids and spans; 16 KiB leaves generous headroom while keeping the blob +// small enough that last-write-wins per admin account stays cheap. +const maxDashboardLayoutBytes = 16 << 10 + +// adminDashboardLayoutResponse is the GET body. Both fields are null when the +// admin has never saved a layout, which the web client reads as "keep the +// local/default layout" rather than as an error. +type adminDashboardLayoutResponse struct { + Layout json.RawMessage `json:"layout"` + UpdatedAt *time.Time `json:"updated_at"` +} + +type adminDashboardLayoutRequest struct { + Layout json.RawMessage `json:"layout"` +} + +// Sentinel validation failures. Their text is the message the client sees, so +// it stays lowercase (staticcheck ST1005) and reads as a sentence fragment. +var ( + errDashboardLayoutInvalidJSON = errors.New("request body must be valid JSON") + errDashboardLayoutMissing = errors.New("layout is required") + errDashboardLayoutNotObject = errors.New("layout must be a JSON object") +) + +// parseDashboardLayoutPayload validates a PUT body and returns the document to +// store. The server treats the layout as opaque past requiring a JSON object: +// widget ids and spans are the web client's vocabulary, and it already +// sanitizes them on load, so validating them here would only add a second +// place to update whenever a widget is added. +func parseDashboardLayoutPayload(body []byte) (json.RawMessage, error) { + var req adminDashboardLayoutRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, errDashboardLayoutInvalidJSON + } + // Unmarshal already checked the syntax of the whole document, so the first + // non-space byte is enough to tell an object from any other JSON value. + raw := json.RawMessage(bytes.TrimSpace(req.Layout)) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return nil, errDashboardLayoutMissing + } + if raw[0] != '{' { + return nil, errDashboardLayoutNotObject + } + return raw, nil +} + +// adminDashboardCapabilitiesResponse advertises the admin dashboard surface a +// server supports. Every field is additive and true on a server that has this +// endpoint at all; they exist so a client can tell "this deployment is older +// than my build" from "this deployment is broken" — a server predating the +// dashboard answers 404 on the aggregates and stores no layout, which is +// otherwise indistinguishable from a transport failure. +// +// Per the v1 rules, new functionality is feature-detected rather than inferred +// from a version. This follows the existing per-subsystem convention +// (/admin/sessions/capabilities, /events/capability). +type adminDashboardCapabilitiesResponse struct { + // ServerLayouts reports that GET/PUT/DELETE /admin/dashboard/layout store + // the widget arrangement per admin account server-side. + ServerLayouts bool `json:"server_layouts"` + // Timeseries reports that GET /admin/stats/timeseries serves sampled + // concurrent-stream and egress history. + Timeseries bool `json:"timeseries"` + // PlaybackActivity reports that GET /admin/stats/playback-activity serves + // the rolling playback activity aggregate. + PlaybackActivity bool `json:"playback_activity"` + // TopActivity reports that GET /admin/stats/top-activity serves the + // most-watched-titles and most-active-profiles leaderboards. + TopActivity bool `json:"top_activity"` + // Health reports that GET /admin/server/status carries the additive + // `health` object the dashboard health strip reads. + Health bool `json:"health"` + // LogLevelList reports that GET /admin/logs/app accepts a multi-level + // filter rather than a single level. + LogLevelList bool `json:"log_level_list"` + // WatchProviders reports that GET /admin/stats carries the `watch_providers` + // per-provider breakdown that replaced the Trakt-only + // `watch_provider_activity` object. + WatchProviders bool `json:"watch_providers"` + // DownloadsStats reports that GET /admin/stats/downloads serves the + // offline-download aggregate and that timeseries points carry the + // additive `download_egress_kbps` split. + DownloadsStats bool `json:"downloads_stats"` +} + +// HandleGetDashboardCapabilities handles GET /admin/dashboard/capabilities. +func (h *AdminHandler) HandleGetDashboardCapabilities(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, adminDashboardCapabilitiesResponse{ + ServerLayouts: true, + Timeseries: true, + PlaybackActivity: true, + TopActivity: true, + Health: true, + LogLevelList: true, + WatchProviders: true, + DownloadsStats: true, + }) +} + +// HandleGetDashboardLayout handles GET /admin/dashboard/layout. +func (h *AdminHandler) HandleGetDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + var ( + layout json.RawMessage + updatedAt time.Time + ) + err := h.pool.QueryRow(r.Context(), + `SELECT layout, updated_at FROM admin_dashboard_layouts WHERE user_id = $1`, + userID, + ).Scan(&layout, &updatedAt) + switch { + case errors.Is(err, pgx.ErrNoRows): + writeJSON(w, http.StatusOK, adminDashboardLayoutResponse{}) + return + case err != nil: + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load dashboard layout") + return + } + + writeJSON(w, http.StatusOK, adminDashboardLayoutResponse{Layout: layout, UpdatedAt: &updatedAt}) +} + +// HandlePutDashboardLayout handles PUT /admin/dashboard/layout. +func (h *AdminHandler) HandlePutDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxDashboardLayoutBytes)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeError(w, http.StatusBadRequest, "bad_request", "Dashboard layout is too large") + return + } + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + + layout, err := parseDashboardLayoutPayload(body) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + // Last write wins. The layout is a per-admin blob, so a race between two of + // the same admin's tabs can only cost the older arrangement; updated_at is + // returned by GET so a compare-and-set could be layered on later. + if _, err := h.pool.Exec(r.Context(), + `INSERT INTO admin_dashboard_layouts (user_id, layout, updated_at) + VALUES ($1, $2, now()) + ON CONFLICT (user_id) DO UPDATE SET layout = EXCLUDED.layout, updated_at = now()`, + userID, []byte(layout), + ); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save dashboard layout") + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// HandleDeleteDashboardLayout handles DELETE /admin/dashboard/layout. Deleting +// the row resets the admin to the default layout; it is idempotent. +func (h *AdminHandler) HandleDeleteDashboardLayout(w http.ResponseWriter, r *http.Request) { + if h.pool == nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + userID := apimw.GetUserID(r.Context()) + if userID == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + + if _, err := h.pool.Exec(r.Context(), + `DELETE FROM admin_dashboard_layouts WHERE user_id = $1`, userID, + ); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to reset dashboard layout") + return + } + + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/handlers/admin_dashboard_layout_test.go b/internal/api/handlers/admin_dashboard_layout_test.go new file mode 100644 index 000000000..231c6e13e --- /dev/null +++ b/internal/api/handlers/admin_dashboard_layout_test.go @@ -0,0 +1,245 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/auth" +) + +func TestParseDashboardLayoutPayload(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + want string + wantErr error + }{ + { + name: "layout object is stored verbatim", + body: `{"layout":{"version":1,"entries":[{"id":"libraries","span":7}]}}`, + want: `{"version":1,"entries":[{"id":"libraries","span":7}]}`, + }, + { + name: "empty object is a valid layout", + body: `{"layout":{}}`, + want: `{}`, + }, + { + name: "unknown widget ids are not the server's business", + body: `{"layout":{"entries":[{"id":"not-a-widget","span":99}]}}`, + want: `{"entries":[{"id":"not-a-widget","span":99}]}`, + }, + { + name: "missing layout", + body: `{}`, + wantErr: errDashboardLayoutMissing, + }, + { + name: "explicit null layout", + body: `{"layout":null}`, + wantErr: errDashboardLayoutMissing, + }, + { + name: "array layout", + body: `{"layout":[{"id":"libraries","span":7}]}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "string layout", + body: `{"layout":"libraries"}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "number layout", + body: `{"layout":7}`, + wantErr: errDashboardLayoutNotObject, + }, + { + name: "malformed json", + body: `{"layout":`, + wantErr: errDashboardLayoutInvalidJSON, + }, + { + name: "empty body", + body: ``, + wantErr: errDashboardLayoutInvalidJSON, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseDashboardLayoutPayload([]byte(tt.body)) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if got != nil { + t.Fatalf("layout = %s, want nil on error", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != tt.want { + t.Fatalf("layout = %s, want %s", got, tt.want) + } + }) + } +} + +// newDashboardLayoutRequest builds an authenticated admin request. The routes +// are admin-gated by the surrounding route group, so the handler only needs a +// user id in the claims. +func newDashboardLayoutRequest(method, body string, userID int) *http.Request { + var req *http.Request + if body == "" { + req = httptest.NewRequest(method, "/admin/dashboard/layout", nil) + } else { + req = httptest.NewRequest(method, "/admin/dashboard/layout", strings.NewReader(body)) + } + if userID == 0 { + return req + } + return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: userID})) +} + +func TestDashboardLayoutHandlersRequireDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + for _, tc := range []struct { + name string + method string + body string + serve func(http.ResponseWriter, *http.Request) + }{ + {"get", http.MethodGet, "", handler.HandleGetDashboardLayout}, + {"put", http.MethodPut, `{"layout":{}}`, handler.HandlePutDashboardLayout}, + {"delete", http.MethodDelete, "", handler.HandleDeleteDashboardLayout}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + tc.serve(rec, newDashboardLayoutRequest(tc.method, tc.body, 7)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } + }) + } +} + +// newUnreachableLayoutHandler returns a handler whose pool never connects, so +// every request that reaches SQL fails. Tests using it assert on the checks +// that run *before* the database is touched. +func newUnreachableLayoutHandler(t *testing.T) *AdminHandler { + t.Helper() + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + return &AdminHandler{pool: pool} +} + +func TestDashboardLayoutHandlersRequireAuthenticatedUser(t *testing.T) { + t.Parallel() + + handler := newUnreachableLayoutHandler(t) + for _, tc := range []struct { + name string + method string + body string + serve func(http.ResponseWriter, *http.Request) + }{ + {"get", http.MethodGet, "", handler.HandleGetDashboardLayout}, + {"put", http.MethodPut, `{"layout":{}}`, handler.HandlePutDashboardLayout}, + {"delete", http.MethodDelete, "", handler.HandleDeleteDashboardLayout}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + tc.serve(rec, newDashboardLayoutRequest(tc.method, tc.body, 0)) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401, body = %s", rec.Code, rec.Body.String()) + } + }) + } +} + +func TestPutDashboardLayoutRejectsInvalidBodies(t *testing.T) { + t.Parallel() + + handler := newUnreachableLayoutHandler(t) + tests := []struct { + name string + body string + wantMessage string + }{ + {"not an object", `{"layout":[]}`, errDashboardLayoutNotObject.Error()}, + {"missing layout", `{}`, errDashboardLayoutMissing.Error()}, + {"malformed", `nope`, errDashboardLayoutInvalidJSON.Error()}, + { + name: "over the size limit", + body: `{"layout":{"pad":"` + strings.Repeat("x", maxDashboardLayoutBytes) + `"}}`, + wantMessage: "Dashboard layout is too large", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + handler.HandlePutDashboardLayout(rec, newDashboardLayoutRequest(http.MethodPut, tt.body, 7)) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + var resp errorResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode error body: %v", err) + } + if resp.Message != tt.wantMessage { + t.Fatalf("message = %q, want %q", resp.Message, tt.wantMessage) + } + }) + } +} + +// TestDashboardCapabilitiesAdvertisesEverySurface pins the feature-detection +// contract for the dashboard: a client reads this to tell an older deployment +// from a failing request, so a surface may only be dropped from the response +// deliberately. +func TestDashboardCapabilitiesAdvertisesEverySurface(t *testing.T) { + t.Parallel() + + rr := httptest.NewRecorder() + (&AdminHandler{}).HandleGetDashboardCapabilities(rr, httptest.NewRequest(http.MethodGet, "/admin/dashboard/capabilities", nil)) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rr.Code, rr.Body.String()) + } + var resp adminDashboardCapabilitiesResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if !resp.ServerLayouts || !resp.Timeseries || !resp.PlaybackActivity || + !resp.TopActivity || !resp.Health || !resp.LogLevelList || !resp.DownloadsStats { + t.Fatalf("capabilities must advertise every dashboard surface: %+v", resp) + } +} diff --git a/internal/api/handlers/admin_logs.go b/internal/api/handlers/admin_logs.go index 50af411f9..27bdc7fbf 100644 --- a/internal/api/handlers/admin_logs.go +++ b/internal/api/handlers/admin_logs.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log/slog" "net/http" + "slices" "strconv" "strings" "time" @@ -58,7 +59,10 @@ func (h *AdminLogsHandler) HandleListAuditLogs(w http.ResponseWriter, r *http.Re func parseOperationalLogOptionsFromRequest(r *http.Request) (opslog.ListOptions, error) { opts := opslog.ListOptions{ - Level: strings.TrimSpace(r.URL.Query().Get("level")), + // `level` accepts a comma-separated list so a single request can ask + // for, say, errors and warnings together — what the dashboard's + // recent-errors widget needs. + Levels: opslog.NormalizeLevels(strings.Split(r.URL.Query().Get("level"), ",")), Component: strings.TrimSpace(r.URL.Query().Get("component")), NodeID: strings.TrimSpace(r.URL.Query().Get("node_id")), RequestID: strings.TrimSpace(r.URL.Query().Get("request_id")), @@ -152,6 +156,32 @@ func parseOptionalIntQuery(r *http.Request, key string) (*int, error) { return &value, nil } +// parseClampedIntQuery reads an optional integer query parameter and clamps it +// into [minValue, maxValue]. An absent or empty parameter yields the clamped +// fallback; a non-numeric one is a bad request rather than a silent default, +// so a client typo surfaces instead of quietly returning the wrong window. +func parseClampedIntQuery(r *http.Request, key string, fallback, minValue, maxValue int) (int, error) { + raw := strings.TrimSpace(r.URL.Query().Get(key)) + if raw == "" { + return clampQueryInt(fallback, minValue, maxValue), nil + } + value, err := strconv.Atoi(raw) + if err != nil { + return 0, invalidQueryError(key) + } + return clampQueryInt(value, minValue, maxValue), nil +} + +func clampQueryInt(value, minValue, maxValue int) int { + if value < minValue { + return minValue + } + if value > maxValue { + return maxValue + } + return value +} + func invalidQueryError(key string) error { return &requestParseError{message: "Invalid " + key} } @@ -428,7 +458,11 @@ func matchesOperationalLog(opts opslog.ListOptions, entry opslog.EntryRow) bool if opts.To != nil && entry.Timestamp.After(*opts.To) { return false } - if opts.Level != "" && entry.Level != strings.ToLower(opts.Level) { + if levels := opslog.NormalizeLevels(opts.Levels); len(levels) > 0 { + if !slices.Contains(levels, strings.ToLower(entry.Level)) { + return false + } + } else if opts.Level != "" && entry.Level != strings.ToLower(opts.Level) { return false } if opts.Component != "" && entry.Component != opts.Component { diff --git a/internal/api/handlers/admin_restart_keys_test.go b/internal/api/handlers/admin_restart_keys_test.go new file mode 100644 index 000000000..472fbdd30 --- /dev/null +++ b/internal/api/handlers/admin_restart_keys_test.go @@ -0,0 +1,61 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "sort" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" +) + +// The admin UI badges restart-required fields from this endpoint, so it has to +// report the compiled registry verbatim — both exact keys and whole namespaces. +func TestHandleGetRestartKeys(t *testing.T) { + handler := &AdminHandler{} + rec := httptest.NewRecorder() + + handler.HandleGetRestartKeys(rec, httptest.NewRequest(http.MethodGet, "/admin/settings/restart-keys", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + var body struct { + Keys []string `json:"keys"` + Prefixes []string `json:"prefixes"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + + if !slices.Equal(body.Keys, config.RestartRequiredKeys()) { + t.Errorf("keys = %v, want %v", body.Keys, config.RestartRequiredKeys()) + } + if !slices.Equal(body.Prefixes, config.RestartRequiredPrefixes()) { + t.Errorf("prefixes = %v, want %v", body.Prefixes, config.RestartRequiredPrefixes()) + } + if !sort.StringsAreSorted(body.Keys) { + t.Errorf("keys are not sorted: %v", body.Keys) + } + + // Spot-check both matching modes so a registry refactor that drops one of + // them fails here rather than silently un-badging the UI. + if !slices.Contains(body.Keys, "auth.jwt_secret") { + t.Errorf("keys missing auth.jwt_secret: %v", body.Keys) + } + if !slices.Contains(body.Prefixes, "database.") { + t.Errorf("prefixes missing database.: %v", body.Prefixes) + } + for _, key := range body.Keys { + if !config.RestartRequired(key) { + t.Errorf("reported key %q is not restart-required", key) + } + } + for _, prefix := range body.Prefixes { + if !config.RestartRequired(prefix + "example") { + t.Errorf("reported prefix %q does not mark its namespace restart-required", prefix) + } + } +} diff --git a/internal/api/handlers/admin_server_status.go b/internal/api/handlers/admin_server_status.go index eb436bd64..855683a4c 100644 --- a/internal/api/handlers/admin_server_status.go +++ b/internal/api/handlers/admin_server_status.go @@ -1,50 +1,197 @@ package handlers import ( + "context" + "log/slog" + "math" "net/http" + "slices" "time" "github.com/Silo-Server/silo-server/internal/jellycompat" ) +const ( + // adminHealthProbeTimeout bounds each dependency probe. A wedged Postgres + // or Redis must not hold the status route open: the dashboard would rather + // be told "not ok" promptly than hang on its health strip. + adminHealthProbeTimeout = 2 * time.Second + + adminLogLevelCountsCacheKey = "log-level-counts-24h" + adminLogLevelCountsCacheTTL = 30 * time.Second +) + type adminServerStatusResponse struct { StartedAt time.Time `json:"started_at"` RestartRequired bool `json:"restart_required"` RestartRequiredAt *time.Time `json:"restart_required_at,omitempty"` RestartRequiredReason string `json:"restart_required_reason,omitempty"` - RestartRequested bool `json:"restart_requested"` - RestartRequestedAt *time.Time `json:"restart_requested_at,omitempty"` + // RestartRequiredReasons accumulates every distinct reason marked since + // boot ("setting:" entries for settings saves), so a client can scope + // a pending restart to the subsystem it belongs to. The singular field + // above only remembers the last save. + RestartRequiredReasons []string `json:"restart_required_reasons,omitempty"` + // RestartMarkCount increments on every restart-required save. The boolean + // above latches for the process lifetime, so this is the client's only + // signal that a NEW requirement arrived after one was dismissed. + RestartMarkCount int `json:"restart_mark_count"` + RestartRequested bool `json:"restart_requested"` + RestartRequestedAt *time.Time `json:"restart_requested_at,omitempty"` + Health adminServerHealth `json:"health"` +} + +// adminServerHealth backs the dashboard health strip. Version, uptime and node +// counts are not repeated here: the client already has them from +// /admin/system/build, started_at above, and /admin/nodes. +type adminServerHealth struct { + Postgres adminHealthComponent `json:"postgres"` + Redis adminHealthComponent `json:"redis"` + Errors24h int64 `json:"errors_24h"` + Warnings24h int64 `json:"warnings_24h"` } +// adminHealthComponent reports one backing service. `configured` false means +// the deployment runs without it (a supported single-node shape for Redis), in +// which case `ok` is absent rather than false — "not present" and "present but +// broken" must not look the same on the strip. +type adminHealthComponent struct { + Configured bool `json:"configured"` + OK *bool `json:"ok,omitempty"` + LatencyMS *float64 `json:"latency_ms,omitempty"` +} + +// adminLogLevelCounts is the cached error/warning tally. It is an array rather +// than a struct so it satisfies cache.TTLCache's comparable constraint without +// a pointer indirection. +type adminLogLevelCounts [2]int64 + // HandleGetServerStatus handles GET /admin/server/status. func (h *AdminHandler) HandleGetServerStatus(w http.ResponseWriter, r *http.Request) { snapshot := h.RestartStatus.Snapshot() resp := adminServerStatusResponse{ - StartedAt: snapshot.StartedAt, - RestartRequired: snapshot.RestartRequired, - RestartRequiredAt: snapshot.RestartRequiredAt, - RestartRequiredReason: snapshot.RestartRequiredReason, - RestartRequested: snapshot.RestartRequested, - RestartRequestedAt: snapshot.RestartRequestedAt, + StartedAt: snapshot.StartedAt, + RestartRequired: snapshot.RestartRequired, + RestartRequiredAt: snapshot.RestartRequiredAt, + RestartRequiredReason: snapshot.RestartRequiredReason, + RestartRequiredReasons: snapshot.RestartReasons, + RestartMarkCount: snapshot.RestartMarkCount, + RestartRequested: snapshot.RestartRequested, + RestartRequestedAt: snapshot.RestartRequestedAt, } + // Settings live in Postgres, so this lookup fails in exactly the outage the + // health object below exists to report. A failure therefore skips the + // jellycompat restart derivation instead of aborting the response — a 500 + // here would hide postgres.ok:false from the one page built to show it — + // and the lookup is bounded like the probes: a wedged pool must not hold + // this optional derivation, and with it the whole response, to the request + // deadline. if h.SettingsRepo != nil { - settings, err := h.SettingsRepo.GetAll(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load settings") - return - } - if jellycompat.WebComponentStatusForConfig(h.Config, settings).RestartRequired { + settingsCtx, cancel := context.WithTimeout(r.Context(), adminHealthProbeTimeout) + settings, err := h.SettingsRepo.GetAll(settingsCtx) + cancel() + if err == nil && jellycompat.WebComponentStatusForConfig(h.Config, settings).RestartRequired { resp.RestartRequired = true if resp.RestartRequiredReason == "" { resp.RestartRequiredReason = "jellyfin_compat" } + // This requirement is derived here rather than marked on the + // tracker, so the accumulated list has to gain it too — a client + // scoping restarts by reason would otherwise never see it. + if !slices.Contains(resp.RestartRequiredReasons, "jellyfin_compat") { + resp.RestartRequiredReasons = append(resp.RestartRequiredReasons, "jellyfin_compat") + } } } + resp.Health = h.collectHealth(r.Context()) + writeJSON(w, http.StatusOK, resp) } +// collectHealth probes the backing services and reads the recent log tallies. +// Every failure here is reported in the body, never as a status code: an +// unreachable dependency is exactly what an admin opened this page to see. +func (h *AdminHandler) collectHealth(ctx context.Context) adminServerHealth { + health := adminServerHealth{ + Postgres: h.probePostgres(ctx), + Redis: h.probeRedis(ctx), + } + counts := h.logLevelCounts24h(ctx) + health.Errors24h = counts[0] + health.Warnings24h = counts[1] + return health +} + +func (h *AdminHandler) probePostgres(ctx context.Context) adminHealthComponent { + if h == nil || h.pool == nil { + return adminHealthComponent{} + } + probeCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + start := time.Now() + err := h.pool.Ping(probeCtx) + return newHealthComponent(err == nil, time.Since(start)) +} + +func (h *AdminHandler) probeRedis(ctx context.Context) adminHealthComponent { + if h == nil || h.RedisClient == nil { + return adminHealthComponent{} + } + probeCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + start := time.Now() + err := h.RedisClient.Ping(probeCtx).Err() + return newHealthComponent(err == nil, time.Since(start)) +} + +func newHealthComponent(ok bool, latency time.Duration) adminHealthComponent { + // Sub-millisecond pings are the normal case for a local Postgres, so the + // latency keeps two decimals instead of rounding an entire healthy install + // down to zero. + ms := math.Round(float64(latency.Microseconds())/10) / 100 + return adminHealthComponent{Configured: true, OK: &ok, LatencyMS: &ms} +} + +// logLevelCounts24h returns [errors, warnings] logged in the last 24 hours. +// A server with operational logging disabled, or one whose log partitions have +// not been created yet, reports zeros and a warn log rather than failing the +// whole status route over a secondary number. +func (h *AdminHandler) logLevelCounts24h(ctx context.Context) adminLogLevelCounts { + if h == nil || h.pool == nil { + return adminLogLevelCounts{} + } + if h.logLevelCounts != nil { + if counts, ok := h.logLevelCounts.Get(adminLogLevelCountsCacheKey); ok { + return counts + } + } + + queryCtx, cancel := context.WithTimeout(ctx, adminHealthProbeTimeout) + defer cancel() + + var counts adminLogLevelCounts + err := h.pool.QueryRow(queryCtx, ` + SELECT + COUNT(*) FILTER (WHERE level = 'error')::bigint, + COUNT(*) FILTER (WHERE level = 'warn')::bigint + FROM operational_logs + WHERE timestamp >= now() - interval '24 hours' + `).Scan(&counts[0], &counts[1]) + if err != nil { + slog.WarnContext(ctx, "failed to count recent operational logs for server status", + "component", "api", "error", err) + return adminLogLevelCounts{} + } + + if h.logLevelCounts != nil { + h.logLevelCounts.Set(adminLogLevelCountsCacheKey, counts, adminLogLevelCountsCacheTTL) + } + return counts +} + func (h *AdminHandler) markServerRestartRequired(reason string) { if h == nil { return diff --git a/internal/api/handlers/admin_server_status_test.go b/internal/api/handlers/admin_server_status_test.go index 7c0883a0e..11bb17d6c 100644 --- a/internal/api/handlers/admin_server_status_test.go +++ b/internal/api/handlers/admin_server_status_test.go @@ -1,11 +1,17 @@ package handlers import ( + "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "testing" + "time" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" "github.com/Silo-Server/silo-server/internal/config" ) @@ -85,3 +91,143 @@ func TestAdminServerStatusDoesNotPromoteLiveJellyfinIdentitySettings(t *testing. t.Fatal("RestartRequired = true, want false for live Jellyfin identity settings") } } + +// A server with neither a pool nor a Redis client must still answer 200 with a +// well-formed health object: the dashboard health strip is the one place an +// admin can see that a dependency is missing. +func TestAdminServerStatusHealthWithoutDependencies(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{RestartStatus: NewServerRestartStatusTracker()} + rec := httptest.NewRecorder() + handler.HandleGetServerStatus(rec, httptest.NewRequest(http.MethodGet, "/admin/server/status", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + + var body struct { + Health struct { + Postgres map[string]any `json:"postgres"` + Redis map[string]any `json:"redis"` + Errors int64 `json:"errors_24h"` + Warnings int64 `json:"warnings_24h"` + } `json:"health"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + + for name, component := range map[string]map[string]any{ + "postgres": body.Health.Postgres, + "redis": body.Health.Redis, + } { + if component == nil { + t.Fatalf("%s health missing from the response", name) + } + if configured, _ := component["configured"].(bool); configured { + t.Fatalf("%s configured = true, want false", name) + } + if _, present := component["ok"]; present { + t.Fatalf("%s reports ok while unconfigured; absent and broken must differ", name) + } + if _, present := component["latency_ms"]; present { + t.Fatalf("%s reports a latency it never measured", name) + } + } + if body.Health.Errors != 0 || body.Health.Warnings != 0 { + t.Fatalf("log counts = %d/%d, want 0/0", body.Health.Errors, body.Health.Warnings) + } +} + +// An unreachable Postgres is reported as ok:false rather than failing the +// route, and the log tallies degrade to zero instead of 500ing. +func TestAdminServerStatusHealthWithUnreachablePostgres(t *testing.T) { + t.Parallel() + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + + handler := &AdminHandler{pool: pool, RestartStatus: NewServerRestartStatusTracker()} + rec := httptest.NewRecorder() + handler.HandleGetServerStatus(rec, httptest.NewRequest(http.MethodGet, "/admin/server/status", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + var body struct { + Health adminServerHealth `json:"health"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if !body.Health.Postgres.Configured { + t.Fatal("postgres configured = false, want true when a pool exists") + } + if body.Health.Postgres.OK == nil || *body.Health.Postgres.OK { + t.Fatalf("postgres ok = %v, want false", body.Health.Postgres.OK) + } + if body.Health.Errors24h != 0 || body.Health.Warnings24h != 0 { + t.Fatalf("log counts = %d/%d, want 0/0 when the query fails", body.Health.Errors24h, body.Health.Warnings24h) + } +} + +// failingSettingsStore models the settings table during a Postgres outage. +type failingSettingsStore struct { + fakeServerSettingsStore +} + +func (f *failingSettingsStore) GetAll(context.Context) (map[string]string, error) { + return nil, fmt.Errorf("settings storage is down") +} + +// A settings lookup that fails — Postgres being down — must not 500 the status +// endpoint: the health object in this response is where that outage is +// supposed to become visible. +func TestAdminServerStatusAnswersWhenSettingsStorageIsDown(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{ + RestartStatus: NewServerRestartStatusTracker(), + SettingsRepo: &failingSettingsStore{}, + } + rec := httptest.NewRecorder() + handler.HandleGetServerStatus(rec, httptest.NewRequest(http.MethodGet, "/admin/server/status", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + var body struct { + Health adminServerHealth `json:"health"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if body.Health.Postgres.Configured { + t.Fatal("postgres configured = true, want false with no pool") + } +} + +func TestLogLevelCountsAreServedFromCache(t *testing.T) { + t.Parallel() + + counts := cache.NewTTLCache[adminLogLevelCounts]() + t.Cleanup(counts.Close) + + pool, err := pgxpool.New(context.Background(), "postgres://silo:silo@127.0.0.1:1/silo?connect_timeout=1") + if err != nil { + t.Fatalf("create unreachable pool: %v", err) + } + t.Cleanup(pool.Close) + + handler := &AdminHandler{pool: pool, logLevelCounts: counts} + counts.Set(adminLogLevelCountsCacheKey, adminLogLevelCounts{4, 12}, time.Minute) + + // The pool cannot connect, so a cache miss would return zeros. + if got := handler.logLevelCounts24h(context.Background()); got != (adminLogLevelCounts{4, 12}) { + t.Fatalf("counts = %v, want [4 12] from cache", got) + } +} diff --git a/internal/api/handlers/admin_settings_checks_test.go b/internal/api/handlers/admin_settings_checks_test.go index 9bf471472..2772e80c4 100644 --- a/internal/api/handlers/admin_settings_checks_test.go +++ b/internal/api/handlers/admin_settings_checks_test.go @@ -1741,3 +1741,80 @@ func withChiParam(r *http.Request, key, value string) *http.Request { routeCtx.URLParams.Add(key, value) return r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, routeCtx)) } + +func TestAdminGetSettingReportsRestartRequired(t *testing.T) { + const restartKey = "scanner.max_concurrent_libraries" + const liveKey = "server.log_level" + + handler := &AdminHandler{ + SettingsRepo: &fakeServerSettingsStore{values: map[string]string{ + restartKey: "4", + liveKey: "debug", + }}, + BootstrapSensitiveValues: map[string]string{ + "playback.ffmpeg_path": "/opt/ffmpeg", + }, + } + + for _, tc := range []struct { + name string + key string + wantValue string + wantRestart bool + wantRestartSeen bool + }{ + { + name: "stored restart-required key", + key: restartKey, + wantValue: "4", + wantRestart: true, + wantRestartSeen: true, + }, + { + name: "stored hot-reloading key omits the flag", + key: liveKey, + wantValue: "debug", + }, + { + name: "bootstrap value", + key: "playback.ffmpeg_path", + wantValue: "/opt/ffmpeg", + wantRestart: true, + wantRestartSeen: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if config.RestartRequired(tc.key) != tc.wantRestart { + t.Fatalf("test fixture drifted: config.RestartRequired(%q) = %v", tc.key, !tc.wantRestart) + } + + req := httptest.NewRequest(http.MethodGet, "/admin/settings/"+tc.key, nil) + req = withChiParam(req, "key", tc.key) + rec := httptest.NewRecorder() + + handler.HandleGetSetting(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var payload struct { + Key string `json:"key"` + Value string `json:"value"` + RestartRequired bool `json:"restart_required"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Key != tc.key || payload.Value != tc.wantValue { + t.Fatalf("payload = %+v, want key %q value %q", payload, tc.key, tc.wantValue) + } + if payload.RestartRequired != tc.wantRestart { + t.Fatalf("restart_required = %v, want %v", payload.RestartRequired, tc.wantRestart) + } + // omitempty must keep the flag off the wire for live keys. + if seen := strings.Contains(rec.Body.String(), "restart_required"); seen != tc.wantRestartSeen { + t.Fatalf("restart_required present = %v, want %v; body=%s", seen, tc.wantRestartSeen, rec.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/admin_stats.go b/internal/api/handlers/admin_stats.go index 535dc9bae..6d84b0217 100644 --- a/internal/api/handlers/admin_stats.go +++ b/internal/api/handlers/admin_stats.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "log/slog" + "sort" "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/watchsync" ) const ( @@ -18,33 +20,58 @@ const ( // AdminStats represents system statistics for the admin dashboard. type AdminStats struct { - TotalItems int `json:"total_items"` - TotalFiles int `json:"total_files"` - TotalUsers int `json:"total_users"` - TotalMovies int `json:"total_movies"` - TotalMovieFiles int `json:"total_movie_files"` - TotalShows int `json:"total_shows"` - TotalShowFiles int `json:"total_show_files"` - ActiveStreams int `json:"active_streams"` - TotalStorageBytes int64 `json:"total_storage_bytes"` - WatchProviderActivity WatchProviderActivity `json:"watch_provider_activity"` + TotalItems int `json:"total_items"` + TotalFiles int `json:"total_files"` + TotalUsers int `json:"total_users"` + TotalMovies int `json:"total_movies"` + TotalMovieFiles int `json:"total_movie_files"` + TotalShows int `json:"total_shows"` + TotalShowFiles int `json:"total_show_files"` + ActiveStreams int `json:"active_streams"` + TotalStorageBytes int64 `json:"total_storage_bytes"` + WatchProviders []WatchProviderStats `json:"watch_providers"` } -type WatchProviderActivity struct { - TraktConnectedProfiles int64 `json:"trakt_connected_profiles"` - TraktEnabledProfiles int64 `json:"trakt_enabled_profiles"` - TraktExportEnabled int64 `json:"trakt_export_enabled"` - TraktScrobbleEnabled int64 `json:"trakt_scrobble_enabled"` - LastSyncCompletedAt *time.Time `json:"last_sync_completed_at,omitempty"` - SyncRuns24h int64 `json:"sync_runs_24h"` - SyncErrors24h int64 `json:"sync_errors_24h"` - ImportedWatched24h int64 `json:"imported_watched_24h"` - ImportedProgress24h int64 `json:"imported_progress_24h"` - ExportedWatched24h int64 `json:"exported_watched_24h"` - PendingExports int64 `json:"pending_exports"` - FailedExports int64 `json:"failed_exports"` - OpenScrobbles int64 `json:"open_scrobbles"` - Scrobbles24h int64 `json:"scrobbles_24h"` +// WatchProviderStats is one registered watch provider's connection and 24-hour +// sync activity. The dashboard renders one row per entry, so every provider the +// watchsync registry knows about appears — including providers contributed by a +// plugin at runtime — with zeros when it has never synced. +// +// An entry whose provider is not (or is no longer) registered still appears +// when the watch-provider tables hold rows for it: uninstalling a plugin must +// not silently drop the history an admin is looking at. Those entries carry +// Registered=false and fall back to the provider key as the display name. +type WatchProviderStats struct { + Provider string `json:"provider"` + DisplayName string `json:"display_name"` + // Registered is false for a provider that only exists in stored rows — + // its plugin is uninstalled or disabled. + Registered bool `json:"registered"` + // Scrobbling and Exporting mirror the provider's declared capabilities so + // the widget can hide counters a provider can never produce. + Scrobbling bool `json:"scrobbling"` + Exporting bool `json:"exporting"` + ConnectedProfiles int64 `json:"connected_profiles"` + EnabledProfiles int64 `json:"enabled_profiles"` + ExportEnabledProfiles int64 `json:"export_enabled_profiles"` + ScrobbleEnabledProfiles int64 `json:"scrobble_enabled_profiles"` + LastSyncCompletedAt *time.Time `json:"last_sync_completed_at,omitempty"` + SyncRuns24h int64 `json:"sync_runs_24h"` + SyncErrors24h int64 `json:"sync_errors_24h"` + ImportedWatched24h int64 `json:"imported_watched_24h"` + ImportedProgress24h int64 `json:"imported_progress_24h"` + ExportedWatched24h int64 `json:"exported_watched_24h"` + PendingExports int64 `json:"pending_exports"` + FailedExports int64 `json:"failed_exports"` + OpenScrobbles int64 `json:"open_scrobbles"` + Scrobbles24h int64 `json:"scrobbles_24h"` +} + +// WatchProviderLister is the narrow view of the watchsync registry the admin +// stats need: which providers exist and what they can do. A nil lister (no +// database, so no registry) degrades to "activity rows only". +type WatchProviderLister interface { + List() []watchsync.ProviderSummary } // AdminStatsSource returns cached or freshly queried admin stats. @@ -55,21 +82,28 @@ type AdminStatsSource interface { // AdminStatsProvider serves exact admin stats with a short in-process TTL and // optional cross-node invalidation via the shared event bus. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value and AdminStats now carries the per-provider slice. Callers only read +// and serialize the snapshot, as with the other admin aggregates. type AdminStatsProvider struct { - pool *pgxpool.Pool - cache *cache.TTLCache[AdminStats] - ttl time.Duration + pool *pgxpool.Pool + providers WatchProviderLister + cache *cache.TTLCache[*AdminStats] + ttl time.Duration } var _ AdminStatsSource = (*AdminStatsProvider)(nil) // NewAdminStatsProvider creates a cached provider and subscribes it to the -// shared invalidation channels when an event bus is configured. -func NewAdminStatsProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminStatsProvider, error) { +// shared invalidation channels when an event bus is configured. providers is +// the watchsync registry (or any narrow view of it) and may be nil. +func NewAdminStatsProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus, providers WatchProviderLister) (*AdminStatsProvider, error) { provider := &AdminStatsProvider{ - pool: pool, - cache: cache.NewTTLCache[AdminStats](), - ttl: adminStatsCacheTTL, + pool: pool, + providers: providers, + cache: cache.NewTTLCache[*AdminStats](), + ttl: adminStatsCacheTTL, } if bus == nil || ctx == nil { @@ -95,15 +129,15 @@ func (p *AdminStatsProvider) Get(ctx context.Context) (AdminStats, error) { if p == nil || p.pool == nil { return AdminStats{}, fmt.Errorf("admin stats provider is not configured") } - if stats, ok := p.cache.Get(adminStatsCacheKey); ok { - return stats, nil + if stats, ok := p.cache.Get(adminStatsCacheKey); ok && stats != nil { + return *stats, nil } - stats, err := queryAdminStats(ctx, p.pool) + stats, err := queryAdminStats(ctx, p.pool, p.providers) if err != nil { return AdminStats{}, err } - p.cache.Set(adminStatsCacheKey, stats, p.ttl) + p.cache.Set(adminStatsCacheKey, &stats, p.ttl) return stats, nil } @@ -123,7 +157,7 @@ func (p *AdminStatsProvider) Close() { p.cache.Close() } -func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error) { +func queryAdminStats(ctx context.Context, pool *pgxpool.Pool, providers WatchProviderLister) (AdminStats, error) { if pool == nil { return AdminStats{}, fmt.Errorf("database not configured") } @@ -220,139 +254,201 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error activity, err := queryWatchProviderActivity(ctx, pool) if err != nil { slog.WarnContext(ctx, "failed to query watch provider admin stats", "component", "api", "error", err) - activity = WatchProviderActivity{} + activity = nil } return AdminStats{ - TotalUsers: int(totalUsers), - TotalItems: int(totalItems), - TotalFiles: int(totalFiles), - TotalMovies: int(totalMovies), - TotalMovieFiles: int(totalMovieFiles), - TotalShows: int(totalShows), - TotalShowFiles: int(totalShowFiles), - ActiveStreams: int(activeStreams), - TotalStorageBytes: totalStorage, - WatchProviderActivity: activity, + TotalUsers: int(totalUsers), + TotalItems: int(totalItems), + TotalFiles: int(totalFiles), + TotalMovies: int(totalMovies), + TotalMovieFiles: int(totalMovieFiles), + TotalShows: int(totalShows), + TotalShowFiles: int(totalShowFiles), + ActiveStreams: int(activeStreams), + TotalStorageBytes: totalStorage, + WatchProviders: mergeWatchProviderStats(listWatchProviders(providers), activity), }, nil } -func queryWatchProviderActivity(ctx context.Context, pool *pgxpool.Pool) (WatchProviderActivity, error) { +// listWatchProviders tolerates both a nil interface and a typed-nil registry. +func listWatchProviders(providers WatchProviderLister) []watchsync.ProviderSummary { + if providers == nil { + return nil + } + return providers.List() +} + +// mergeWatchProviderStats overlays the per-provider activity rows onto the set +// of registered providers. Registered providers always appear (with zeros when +// they have no rows); a row for an unregistered provider is kept so history +// from an uninstalled plugin stays visible. Ordering is by provider key so the +// dashboard rows do not reshuffle between polls. +func mergeWatchProviderStats(summaries []watchsync.ProviderSummary, activity []WatchProviderStats) []WatchProviderStats { + byProvider := make(map[string]WatchProviderStats, len(summaries)+len(activity)) + for _, row := range activity { + if row.Provider == "" { + continue + } + row.DisplayName = row.Provider + byProvider[row.Provider] = row + } + for _, summary := range summaries { + if summary.Key == "" { + continue + } + stats := byProvider[summary.Key] + stats.Provider = summary.Key + stats.DisplayName = summary.DisplayName + if stats.DisplayName == "" { + stats.DisplayName = summary.Key + } + stats.Registered = true + stats.Scrobbling = summary.Capabilities.ScrobblePlayback + stats.Exporting = summary.Capabilities.ExportWatched + byProvider[summary.Key] = stats + } + + merged := make([]WatchProviderStats, 0, len(byProvider)) + for _, stats := range byProvider { + merged = append(merged, stats) + } + sort.Slice(merged, func(i, j int) bool { + return merged[i].Provider < merged[j].Provider + }) + return merged +} + +func queryWatchProviderActivity(ctx context.Context, pool *pgxpool.Pool) ([]WatchProviderStats, error) { ready, err := watchProviderStatsTablesReady(ctx, pool) if err != nil { - return WatchProviderActivity{}, err + return nil, err } if !ready { - return WatchProviderActivity{}, nil + return nil, nil } - var activity WatchProviderActivity - row := pool.QueryRow(ctx, ` + // One pass per table, grouped by provider, then joined onto the union of + // every provider key those tables mention. Grouping keeps this the same + // four scans the single-provider version cost, whatever the provider count. + rows, err := pool.Query(ctx, ` WITH watch_provider_connection_stats AS ( SELECT - COUNT(*) FILTER (WHERE provider = 'trakt')::bigint AS trakt_connected_profiles, + provider, + COUNT(*)::bigint AS connected_profiles, COUNT(*) FILTER ( - WHERE provider = 'trakt' - AND ( - import_watched_enabled - OR import_progress_enabled - OR export_watched_enabled - OR scrobble_enabled - ) - )::bigint AS trakt_enabled_profiles, - COUNT(*) FILTER (WHERE provider = 'trakt' AND export_watched_enabled)::bigint AS trakt_export_enabled, - COUNT(*) FILTER (WHERE provider = 'trakt' AND scrobble_enabled)::bigint AS trakt_scrobble_enabled + WHERE import_watched_enabled + OR import_progress_enabled + OR export_watched_enabled + OR scrobble_enabled + )::bigint AS enabled_profiles, + COUNT(*) FILTER (WHERE export_watched_enabled)::bigint AS export_enabled_profiles, + COUNT(*) FILTER (WHERE scrobble_enabled)::bigint AS scrobble_enabled_profiles FROM watch_provider_connections + GROUP BY provider ), watch_provider_sync_stats AS ( SELECT - MAX(completed_at) FILTER (WHERE provider = 'trakt') AS last_sync_completed_at, - COUNT(*) FILTER ( - WHERE provider = 'trakt' - AND started_at >= now() - interval '24 hours' - )::bigint AS sync_runs_24h, + provider, + MAX(completed_at) AS last_sync_completed_at, + COUNT(*) FILTER (WHERE started_at >= now() - interval '24 hours')::bigint AS sync_runs_24h, COUNT(*) FILTER ( - WHERE provider = 'trakt' - AND status = 'failed' + WHERE status = 'failed' AND started_at >= now() - interval '24 hours' )::bigint AS sync_errors_24h, COALESCE(SUM(inbound_watched_imported) FILTER ( - WHERE provider = 'trakt' - AND started_at >= now() - interval '24 hours' + WHERE started_at >= now() - interval '24 hours' ), 0)::bigint AS imported_watched_24h, COALESCE(SUM(inbound_progress_imported) FILTER ( - WHERE provider = 'trakt' - AND started_at >= now() - interval '24 hours' + WHERE started_at >= now() - interval '24 hours' ), 0)::bigint AS imported_progress_24h, COALESCE(SUM(outbound_sent) FILTER ( - WHERE provider = 'trakt' - AND started_at >= now() - interval '24 hours' + WHERE started_at >= now() - interval '24 hours' ), 0)::bigint AS exported_watched_24h FROM watch_provider_sync_runs + GROUP BY provider ), watch_provider_export_stats AS ( SELECT - COUNT(*) FILTER ( - WHERE c.provider = 'trakt' - AND e.status = 'pending' - )::bigint AS pending_exports, - COUNT(*) FILTER ( - WHERE c.provider = 'trakt' - AND e.status = 'failed' - )::bigint AS failed_exports + c.provider, + COUNT(*) FILTER (WHERE e.status = 'pending')::bigint AS pending_exports, + COUNT(*) FILTER (WHERE e.status = 'failed')::bigint AS failed_exports FROM watch_provider_history_exports e JOIN watch_provider_connections c ON c.id = e.connection_id + GROUP BY c.provider ), watch_provider_scrobble_stats AS ( SELECT - COUNT(*) FILTER ( - WHERE c.provider = 'trakt' - AND s.stop_sent_at IS NULL - )::bigint AS open_scrobbles, - COUNT(*) FILTER ( - WHERE c.provider = 'trakt' - AND s.updated_at >= now() - interval '24 hours' - )::bigint AS scrobbles_24h + c.provider, + COUNT(*) FILTER (WHERE s.stop_sent_at IS NULL)::bigint AS open_scrobbles, + COUNT(*) FILTER (WHERE s.updated_at >= now() - interval '24 hours')::bigint AS scrobbles_24h FROM watch_provider_scrobble_sessions s JOIN watch_provider_connections c ON c.id = s.connection_id + GROUP BY c.provider + ), + watch_provider_keys AS ( + SELECT provider FROM watch_provider_connection_stats + UNION + SELECT provider FROM watch_provider_sync_stats + UNION + SELECT provider FROM watch_provider_export_stats + UNION + SELECT provider FROM watch_provider_scrobble_stats ) SELECT - watch_provider_connection_stats.trakt_connected_profiles, - watch_provider_connection_stats.trakt_enabled_profiles, - watch_provider_connection_stats.trakt_export_enabled, - watch_provider_connection_stats.trakt_scrobble_enabled, - watch_provider_sync_stats.last_sync_completed_at, - watch_provider_sync_stats.sync_runs_24h, - watch_provider_sync_stats.sync_errors_24h, - watch_provider_sync_stats.imported_watched_24h, - watch_provider_sync_stats.imported_progress_24h, - watch_provider_sync_stats.exported_watched_24h, - watch_provider_export_stats.pending_exports, - watch_provider_export_stats.failed_exports, - watch_provider_scrobble_stats.open_scrobbles, - watch_provider_scrobble_stats.scrobbles_24h - FROM watch_provider_connection_stats - CROSS JOIN watch_provider_sync_stats - CROSS JOIN watch_provider_export_stats - CROSS JOIN watch_provider_scrobble_stats + k.provider, + COALESCE(c.connected_profiles, 0), + COALESCE(c.enabled_profiles, 0), + COALESCE(c.export_enabled_profiles, 0), + COALESCE(c.scrobble_enabled_profiles, 0), + s.last_sync_completed_at, + COALESCE(s.sync_runs_24h, 0), + COALESCE(s.sync_errors_24h, 0), + COALESCE(s.imported_watched_24h, 0), + COALESCE(s.imported_progress_24h, 0), + COALESCE(s.exported_watched_24h, 0), + COALESCE(e.pending_exports, 0), + COALESCE(e.failed_exports, 0), + COALESCE(sc.open_scrobbles, 0), + COALESCE(sc.scrobbles_24h, 0) + FROM watch_provider_keys k + LEFT JOIN watch_provider_connection_stats c ON c.provider = k.provider + LEFT JOIN watch_provider_sync_stats s ON s.provider = k.provider + LEFT JOIN watch_provider_export_stats e ON e.provider = k.provider + LEFT JOIN watch_provider_scrobble_stats sc ON sc.provider = k.provider + ORDER BY k.provider `) - if err := row.Scan( - &activity.TraktConnectedProfiles, - &activity.TraktEnabledProfiles, - &activity.TraktExportEnabled, - &activity.TraktScrobbleEnabled, - &activity.LastSyncCompletedAt, - &activity.SyncRuns24h, - &activity.SyncErrors24h, - &activity.ImportedWatched24h, - &activity.ImportedProgress24h, - &activity.ExportedWatched24h, - &activity.PendingExports, - &activity.FailedExports, - &activity.OpenScrobbles, - &activity.Scrobbles24h, - ); err != nil { - return WatchProviderActivity{}, fmt.Errorf("querying watch provider activity stats: %w", err) + if err != nil { + return nil, fmt.Errorf("querying watch provider activity stats: %w", err) + } + defer rows.Close() + + var activity []WatchProviderStats + for rows.Next() { + var stats WatchProviderStats + if err := rows.Scan( + &stats.Provider, + &stats.ConnectedProfiles, + &stats.EnabledProfiles, + &stats.ExportEnabledProfiles, + &stats.ScrobbleEnabledProfiles, + &stats.LastSyncCompletedAt, + &stats.SyncRuns24h, + &stats.SyncErrors24h, + &stats.ImportedWatched24h, + &stats.ImportedProgress24h, + &stats.ExportedWatched24h, + &stats.PendingExports, + &stats.FailedExports, + &stats.OpenScrobbles, + &stats.Scrobbles24h, + ); err != nil { + return nil, fmt.Errorf("scanning watch provider activity stats: %w", err) + } + activity = append(activity, stats) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("querying watch provider activity stats: %w", err) } return activity, nil diff --git a/internal/api/handlers/admin_stats_downloads.go b/internal/api/handlers/admin_stats_downloads.go new file mode 100644 index 000000000..40a06e384 --- /dev/null +++ b/internal/api/handlers/admin_stats_downloads.go @@ -0,0 +1,258 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" +) + +const ( + // adminDownloadsStatsCacheTTL sits at the dashboard's 60s refresh cadence: + // download state changes far slower than playback, and the widget's numbers + // are read as "roughly now", not as a live counter. + adminDownloadsStatsCacheTTL = time.Minute + adminDownloadsStatsCachePrefix = "downloads-limit=" + + adminDownloadsStatsDefaultLimit = 10 + adminDownloadsStatsMinLimit = 1 + adminDownloadsStatsMaxLimit = 25 +) + +// downloadsActiveFilter defines "active": a managed device entry (the +// device-aware lifecycle in internal/downloads) that has not ended in +// failure, cancellation, or revocation. Ephemeral web rows (NULL device_id) +// are one-shot transfers that get pruned, not media somebody keeps on a +// device, so they stay out of the active numbers — they still count in the +// 24-hour started/completed totals below. prefix qualifies the columns for a +// query that aliases the downloads table. +func downloadsActiveFilter(prefix string) string { + return prefix + `device_id IS NOT NULL AND ` + prefix + `status IN ('queued', 'preparing', 'ready', 'downloading', 'completed')` +} + +// AdminDownloadsUser is one row of the per-user active-downloads list. +type AdminDownloadsUser struct { + UserID int `json:"user_id"` + Username string `json:"username"` + // Downloads is this account's active managed device entries. + Downloads int64 `json:"downloads"` + // TotalBytes sums the file sizes of this account's completed device + // entries — bytes actually sitting on devices, not bytes still queued. + TotalBytes int64 `json:"total_bytes"` +} + +// AdminDownloadsStats is the GET /admin/stats/downloads body. Every count is +// zero and TopUsers is an empty array on a deployment where nobody downloads +// (or where the downloads feature is disabled); the widget reads that as its +// empty state, never as an error. +type AdminDownloadsStats struct { + // UsersWithDownloads is the distinct accounts (users rows, not household + // profiles) with at least one active managed download. + UsersWithDownloads int64 `json:"users_with_downloads"` + // ActiveDownloads counts active managed device entries (items, where a + // series batch contributes one entry per episode). + ActiveDownloads int64 `json:"active_downloads"` + // TotalBytes sums the file sizes of completed device entries: the bytes + // currently sitting on devices, as far as the server can know without the + // device reporting back. + TotalBytes int64 `json:"total_bytes"` + // DownloadsStarted24h counts rows created in the last 24 hours across both + // lifecycles (managed device entries and one-shot web downloads). + DownloadsStarted24h int64 `json:"downloads_started_24h"` + // DownloadsCompleted24h counts rows that reached completed in the last 24 + // hours across both lifecycles. + DownloadsCompleted24h int64 `json:"downloads_completed_24h"` + // Limit is the clamped top-list size the response was built with. + Limit int `json:"limit"` + // TopUsers ranks accounts by active managed downloads. + TopUsers []AdminDownloadsUser `json:"top_users"` +} + +// AdminDownloadsStatsSource returns cached or freshly queried download stats. +type AdminDownloadsStatsSource interface { + Get(ctx context.Context, limit int) (*AdminDownloadsStats, error) + Invalidate() +} + +// AdminDownloadsStatsProvider serves the downloads aggregate with a short +// in-process TTL and optional cross-node invalidation via the shared event +// bus, mirroring AdminStatsProvider. Downloads publish no bus events of their +// own today, so the TTL (and the widget's ?refresh=1) is what bounds +// staleness; the admin channel subscription exists so a future downloads +// event needs no provider change. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries a slice. +type AdminDownloadsStatsProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminDownloadsStats] + ttl time.Duration +} + +var _ AdminDownloadsStatsSource = (*AdminDownloadsStatsProvider)(nil) + +// NewAdminDownloadsStatsProvider creates a cached provider and subscribes it +// to the admin invalidation channel when an event bus is configured. +func NewAdminDownloadsStatsProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminDownloadsStatsProvider, error) { + provider := &AdminDownloadsStatsProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminDownloadsStats](), + ttl: adminDownloadsStatsCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + if err := bus.Subscribe(ctx, cache.ChannelAdmin, func(cache.Event) { + provider.Invalidate() + }); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin downloads stats provider to %s: %w", cache.ChannelAdmin, err) + } + + return provider, nil +} + +// Get returns the cached stats when available, otherwise queries Postgres. +func (p *AdminDownloadsStatsProvider) Get(ctx context.Context, limit int) (*AdminDownloadsStats, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin downloads stats provider is not configured") + } + limit = clampQueryInt(limit, adminDownloadsStatsMinLimit, adminDownloadsStatsMaxLimit) + key := adminDownloadsStatsCachePrefix + strconv.Itoa(limit) + if stats, ok := p.cache.Get(key); ok { + return stats, nil + } + + stats, err := queryAdminDownloadsStats(ctx, p.pool, limit) + if err != nil { + return nil, err + } + p.cache.Set(key, stats, p.ttl) + return stats, nil +} + +// Invalidate drops every cached variant. +func (p *AdminDownloadsStatsProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminDownloadsStatsCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminDownloadsStatsProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetDownloadsStats handles GET /admin/stats/downloads. +func (h *AdminHandler) HandleGetDownloadsStats(w http.ResponseWriter, r *http.Request) { + limit, err := parseClampedIntQuery( + r, "limit", + adminDownloadsStatsDefaultLimit, + adminDownloadsStatsMinLimit, + adminDownloadsStatsMaxLimit, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var stats *AdminDownloadsStats + switch { + case h.DownloadsStatsSource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.DownloadsStatsSource.Invalidate() + } + stats, err = h.DownloadsStatsSource.Get(r.Context(), limit) + case h.pool != nil: + stats, err = queryAdminDownloadsStats(r.Context(), h.pool, limit) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get downloads stats") + return + } + + writeJSON(w, http.StatusOK, stats) +} + +// queryAdminDownloadsStats reads the aggregate straight from the downloads +// table (migrations 042 + 20260619020213). The table exists on every deployment +// regardless of whether the downloads service is wired, so a server with the +// feature off answers zeros rather than an error. +func queryAdminDownloadsStats(ctx context.Context, pool *pgxpool.Pool, limit int) (*AdminDownloadsStats, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + stats := &AdminDownloadsStats{ + Limit: limit, + TopUsers: []AdminDownloadsUser{}, + } + + // One pass over the table for every scalar. GREATEST guards file sizes + // recorded before a transfer finished sizing (the column defaults to 0 and + // is never negative in practice, but a SUM must not depend on that). + if err := pool.QueryRow(ctx, ` + SELECT + COUNT(DISTINCT user_id) FILTER (WHERE `+downloadsActiveFilter("")+`), + COUNT(*) FILTER (WHERE `+downloadsActiveFilter("")+`), + COALESCE(SUM(GREATEST(file_size, 0)) FILTER (WHERE device_id IS NOT NULL AND status = 'completed'), 0)::bigint, + COUNT(*) FILTER (WHERE created_at >= now() - interval '24 hours'), + COUNT(*) FILTER (WHERE status = 'completed' AND completed_at >= now() - interval '24 hours') + FROM downloads + `).Scan( + &stats.UsersWithDownloads, + &stats.ActiveDownloads, + &stats.TotalBytes, + &stats.DownloadsStarted24h, + &stats.DownloadsCompleted24h, + ); err != nil { + return nil, fmt.Errorf("querying downloads stats: %w", err) + } + + // The top list ranks accounts, not profiles: a device entry belongs to a + // profile, but quota and the download policy hang off the account, so the + // admin-facing ranking follows the same line. + rows, err := pool.Query(ctx, ` + SELECT d.user_id, + COALESCE(u.username, '') AS username, + COUNT(*)::bigint AS downloads, + COALESCE(SUM(GREATEST(d.file_size, 0)) FILTER (WHERE d.status = 'completed'), 0)::bigint AS total_bytes + FROM downloads d + LEFT JOIN users u ON u.id = d.user_id + WHERE `+downloadsActiveFilter("d.")+` + GROUP BY d.user_id, u.username + ORDER BY downloads DESC, total_bytes DESC, d.user_id + LIMIT $1 + `, limit) + if err != nil { + return nil, fmt.Errorf("querying top download users: %w", err) + } + defer rows.Close() + + for rows.Next() { + var user AdminDownloadsUser + if err := rows.Scan(&user.UserID, &user.Username, &user.Downloads, &user.TotalBytes); err != nil { + return nil, fmt.Errorf("scanning top download user: %w", err) + } + stats.TopUsers = append(stats.TopUsers, user) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating top download users: %w", err) + } + + return stats, nil +} diff --git a/internal/api/handlers/admin_stats_downloads_test.go b/internal/api/handlers/admin_stats_downloads_test.go new file mode 100644 index 000000000..69912ad13 --- /dev/null +++ b/internal/api/handlers/admin_stats_downloads_test.go @@ -0,0 +1,182 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type stubDownloadsStatsSource struct { + stats *AdminDownloadsStats + err error + gotLimit int + invalidated int + callCount int +} + +func (s *stubDownloadsStatsSource) Get(_ context.Context, limit int) (*AdminDownloadsStats, error) { + s.callCount++ + s.gotLimit = limit + return s.stats, s.err +} + +func (s *stubDownloadsStatsSource) Invalidate() { s.invalidated++ } + +func TestHandleGetDownloadsStatsClampsLimit(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantLimit int + }{ + {name: "default", query: "", wantLimit: adminDownloadsStatsDefaultLimit}, + {name: "explicit value passes through", query: "?limit=5", wantLimit: 5}, + {name: "zero clamps up", query: "?limit=0", wantLimit: adminDownloadsStatsMinLimit}, + {name: "oversized clamps down", query: "?limit=999", wantLimit: adminDownloadsStatsMaxLimit}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubDownloadsStatsSource{stats: &AdminDownloadsStats{}} + handler := &AdminHandler{DownloadsStatsSource: source} + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotLimit != tt.wantLimit { + t.Fatalf("limit = %d, want %d", source.gotLimit, tt.wantLimit) + } + }) + } +} + +func TestHandleGetDownloadsStatsRejectsNonNumericLimit(t *testing.T) { + t.Parallel() + + source := &stubDownloadsStatsSource{stats: &AdminDownloadsStats{}} + handler := &AdminHandler{DownloadsStatsSource: source} + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads?limit=all", nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } +} + +func TestHandleGetDownloadsStatsRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubDownloadsStatsSource{stats: &AdminDownloadsStats{}} + handler := &AdminHandler{DownloadsStatsSource: source} + + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads?refresh=true", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if source.invalidated != 1 { + t.Fatalf("invalidated = %d, want 1", source.invalidated) + } +} + +func TestHandleGetDownloadsStatsSourceFailureIs500(t *testing.T) { + t.Parallel() + + source := &stubDownloadsStatsSource{err: errors.New("boom")} + handler := &AdminHandler{DownloadsStatsSource: source} + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +func TestHandleGetDownloadsStatsWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +// A deployment where nobody downloads must answer zeros with an empty array, +// not nulls: the widget maps over top_users directly and reads all-zero +// headline numbers as its empty state. +func TestAdminDownloadsStatsEmptyListSerializesAsArray(t *testing.T) { + t.Parallel() + + source := &stubDownloadsStatsSource{stats: &AdminDownloadsStats{ + Limit: 10, + TopUsers: []AdminDownloadsUser{}, + }} + handler := &AdminHandler{DownloadsStatsSource: source} + rec := httptest.NewRecorder() + handler.HandleGetDownloadsStats(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/downloads", nil)) + + var body struct { + TopUsers []AdminDownloadsUser `json:"top_users"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.TopUsers == nil { + t.Fatalf("top_users decoded as null: %s", rec.Body.String()) + } +} + +func TestAdminDownloadsStatsProviderInvalidateClearsEveryVariant(t *testing.T) { + t.Parallel() + + provider, err := NewAdminDownloadsStatsProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + keys := []string{ + adminDownloadsStatsCachePrefix + "10", + adminDownloadsStatsCachePrefix + "25", + } + for _, key := range keys { + provider.cache.Set(key, &AdminDownloadsStats{}, time.Minute) + } + + provider.Invalidate() + + for _, key := range keys { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminDownloadsStatsProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminDownloadsStatsProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 10); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/admin_stats_playback.go b/internal/api/handlers/admin_stats_playback.go new file mode 100644 index 000000000..bd3600459 --- /dev/null +++ b/internal/api/handlers/admin_stats_playback.go @@ -0,0 +1,413 @@ +package handlers + +import ( + "context" + "fmt" + "math" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/playback" +) + +const ( + adminPlaybackActivityCacheTTL = 60 * time.Second + adminPlaybackActivityCachePrefix = "hours=" + + adminPlaybackActivityDefaultHours = 24 + adminPlaybackActivityMinHours = 1 + adminPlaybackActivityMaxHours = 744 + + // adminPlaybackActivityHourlyMaxHours is the widest window still bucketed by + // hour. Past two days an hourly chart is more columns than a widget has + // pixels, so the buckets become days. + adminPlaybackActivityHourlyMaxHours = 48 + + adminPlaybackActivityHourSeconds = 3600 + adminPlaybackActivityDaySeconds = 86400 +) + +// playbackActivityBucketSeconds picks the bucket width for a window: hourly up +// to two days, daily beyond it. The response reports the choice in +// bucket_seconds so the client zero-fills the same shape the server grouped by. +func playbackActivityBucketSeconds(hours int) int { + if hours <= adminPlaybackActivityHourlyMaxHours { + return adminPlaybackActivityHourSeconds + } + return adminPlaybackActivityDaySeconds +} + +// date_trunc field names for the two bucket widths. +const ( + playbackActivityTruncDay = "day" + playbackActivityTruncHour = "hour" +) + +// playbackActivityTruncField is the date_trunc field matching a bucket width. +func playbackActivityTruncField(bucketSeconds int) string { + if bucketSeconds >= adminPlaybackActivityDaySeconds { + return playbackActivityTruncDay + } + return playbackActivityTruncHour +} + +// AdminPlaybackActivityBucket is one bucket of playback starts, split by the +// play method that was resolved for the session. Only buckets with at least one +// session are present; the dashboard zero-fills the rest of the window so a +// quiet server draws an empty column rather than a shorter chart. +// +// The field stays `hour` for every bucket width: it is the bucket's start +// instant, and renaming it would break every client for no new information — +// bucket_seconds already says how wide the bucket is. +type AdminPlaybackActivityBucket struct { + Hour time.Time `json:"hour"` + Direct int64 `json:"direct"` + Remux int64 `json:"remux"` + Transcode int64 `json:"transcode"` +} + +// AdminPlaybackReliability summarizes how playback went over the window. +// +// Time-to-first-frame and failed-start counts are deliberately absent: nothing +// records a playback *start* event today (playback_history_admin only gains a +// row when a session finalizes), so both would have to be guessed from log +// parsing. They need client telemetry first — see docs/admin-api.md. +type AdminPlaybackReliability struct { + SessionsStarted int64 `json:"sessions_started"` + TranscodeStarts int64 `json:"transcode_starts"` + FinalizedSessions int64 `json:"finalized_sessions"` + CompletedSessions int64 `json:"completed_sessions"` + CompletionRate float64 `json:"completion_rate"` + UniqueProfiles int64 `json:"unique_profiles"` +} + +// AdminPlaybackActivity is the GET /admin/stats/playback-activity body. +// +// Buckets are hourly or daily depending on the window (BucketSeconds says +// which). Reliability is computed over the whole requested window, while +// ProfilesActive24h is a fixed rolling-24h tile that ignores the window — it +// answers "who watched today", which does not become "who watched this month" +// because a chart next to it got wider. +type AdminPlaybackActivity struct { + Hours int `json:"hours"` + BucketSeconds int `json:"bucket_seconds"` + // From/To are the window on the database clock, which is the clock the + // bucket filter ran against. The dashboard anchors its bucket grid on To + // rather than the browser clock: a few seconds of client/server skew + // around an hour or day boundary would otherwise discard the newest + // bucket and show a stale one. + From time.Time `json:"from"` + To time.Time `json:"to"` + Buckets []AdminPlaybackActivityBucket `json:"buckets"` + Reliability AdminPlaybackReliability `json:"reliability"` + ProfilesActive24h int64 `json:"profiles_active_24h"` +} + +// AdminPlaybackActivitySource returns cached or freshly queried playback +// activity for a window of hours. +type AdminPlaybackActivitySource interface { + Get(ctx context.Context, hours int) (*AdminPlaybackActivity, error) + Invalidate() +} + +// AdminPlaybackActivityProvider serves playback activity with a short in-process +// TTL and optional cross-node invalidation via the shared event bus, mirroring +// AdminStatsProvider. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries a slice. +type AdminPlaybackActivityProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminPlaybackActivity] + ttl time.Duration +} + +var _ AdminPlaybackActivitySource = (*AdminPlaybackActivityProvider)(nil) + +// NewAdminPlaybackActivityProvider creates a cached provider and subscribes it +// to the playback/admin invalidation channels when an event bus is configured. +func NewAdminPlaybackActivityProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminPlaybackActivityProvider, error) { + provider := &AdminPlaybackActivityProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminPlaybackActivity](), + ttl: adminPlaybackActivityCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin playback activity provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached window when available, otherwise queries Postgres. +func (p *AdminPlaybackActivityProvider) Get(ctx context.Context, hours int) (*AdminPlaybackActivity, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin playback activity provider is not configured") + } + hours = clampQueryInt(hours, adminPlaybackActivityMinHours, adminPlaybackActivityMaxHours) + key := adminPlaybackActivityCachePrefix + strconv.Itoa(hours) + if activity, ok := p.cache.Get(key); ok { + return activity, nil + } + + activity, err := queryAdminPlaybackActivity(ctx, p.pool, hours) + if err != nil { + return nil, err + } + p.cache.Set(key, activity, p.ttl) + return activity, nil +} + +// Invalidate drops every cached window. +func (p *AdminPlaybackActivityProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminPlaybackActivityCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminPlaybackActivityProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetPlaybackActivity handles GET /admin/stats/playback-activity. +func (h *AdminHandler) HandleGetPlaybackActivity(w http.ResponseWriter, r *http.Request) { + hours, err := parseClampedIntQuery( + r, "hours", + adminPlaybackActivityDefaultHours, + adminPlaybackActivityMinHours, + adminPlaybackActivityMaxHours, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var activity *AdminPlaybackActivity + switch { + case h.PlaybackActivitySource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.PlaybackActivitySource.Invalidate() + } + activity, err = h.PlaybackActivitySource.Get(r.Context(), hours) + case h.pool != nil: + activity, err = queryAdminPlaybackActivity(r.Context(), h.pool, hours) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get playback activity") + return + } + + writeJSON(w, http.StatusOK, activity) +} + +// playbackActivityRow is one (bucket, play_method) group from the union of +// finalized history and live sessions. +type playbackActivityRow struct { + Hour time.Time + PlayMethod string + Count int64 +} + +// assemblePlaybackBuckets folds the grouped rows into one bucket per instant. +// Play methods the resolver does not produce (empty or unknown strings on old +// rows) are dropped from the stacked series but still counted in +// reliability.sessions_started, which is measured by SQL over the same union. +func assemblePlaybackBuckets(rows []playbackActivityRow) []AdminPlaybackActivityBucket { + buckets := make([]AdminPlaybackActivityBucket, 0, len(rows)) + index := make(map[time.Time]int, len(rows)) + + for _, row := range rows { + hour := row.Hour.UTC() + pos, ok := index[hour] + if !ok { + buckets = append(buckets, AdminPlaybackActivityBucket{Hour: hour}) + pos = len(buckets) - 1 + index[hour] = pos + } + switch playback.PlayMethod(row.PlayMethod) { + case playback.PlayDirect: + buckets[pos].Direct += row.Count + case playback.PlayRemux: + buckets[pos].Remux += row.Count + case playback.PlayTranscode: + buckets[pos].Transcode += row.Count + } + } + + return buckets +} + +// completionRate is completed over finalized sessions. Live sessions are +// excluded from both sides by the caller: a session that is still playing has +// not failed to complete, so counting it as a miss would drag the rate down +// whenever someone is watching. +func completionRate(completed, finalized int64) float64 { + if finalized <= 0 { + return 0 + } + return math.Round(float64(completed)/float64(finalized)*10000) / 10000 +} + +// adminPlaybackSessionsCTE is the union both activity queries aggregate over. +// +// playback_history_admin only gains a row when a session finalizes, so the +// current hour would be under-counted without the live sessions. A finalizing +// session briefly exists on both sides — history is written before the sync +// row is deleted, and the deletion can fail until stale-session cleanup — so +// live rows whose session already reached history are excluded rather than +// counted twice. playback_sessions_sync.started_at is nullable for sessions +// reconstructed after a restart, hence the COALESCE onto updated_at. +const adminPlaybackSessionsCTE = ` + WITH history AS ( + SELECT session_id, started_at, play_method, completed, user_id, profile_id, FALSE AS live + FROM playback_history_admin + WHERE started_at >= now() - make_interval(hours => $1) + ), + sessions AS ( + SELECT started_at, play_method, completed, user_id, profile_id, live FROM history + UNION ALL + SELECT COALESCE(s.started_at, s.updated_at) AS started_at, s.play_method, FALSE, s.user_id, s.profile_id, TRUE + FROM playback_sessions_sync s + WHERE COALESCE(s.started_at, s.updated_at) >= now() - make_interval(hours => $1) + AND NOT EXISTS (SELECT 1 FROM history h WHERE h.session_id = s.session_id) + )` + +func queryAdminPlaybackActivity(ctx context.Context, pool *pgxpool.Pool, hours int) (*AdminPlaybackActivity, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + bucketSeconds := playbackActivityBucketSeconds(hours) + + // Both statements run inside one repeatable-read transaction so they see a + // single snapshot and a single clock: read committed would let a session + // that starts or finalizes between them make reliability describe a + // different session set from the buckets, and now() — which is the + // transaction timestamp — would otherwise differ between the bucket filter + // and the reported window, moving the client's grid off the buckets around + // an hour or day boundary. + tx, err := pool.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadOnly, + }) + if err != nil { + return nil, fmt.Errorf("beginning playback activity read: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + // Truncation is pinned to UTC: date_trunc otherwise cuts on the session + // TimeZone's boundaries, and the dashboard zero-fills on epoch-aligned UTC + // buckets — a daily bucket cut in another zone would land in the wrong + // column (or between columns) client-side. + rows, err := tx.Query(ctx, adminPlaybackSessionsCTE+` + SELECT date_trunc($2, started_at, 'UTC') AS bucket, + COALESCE(play_method, '') AS play_method, + COUNT(*)::bigint AS sessions + FROM sessions + WHERE started_at IS NOT NULL + GROUP BY 1, 2 + ORDER BY 1 + `, hours, playbackActivityTruncField(bucketSeconds)) + if err != nil { + return nil, fmt.Errorf("querying playback activity buckets: %w", err) + } + defer rows.Close() + + grouped := make([]playbackActivityRow, 0, 64) + for rows.Next() { + var row playbackActivityRow + if err := rows.Scan(&row.Hour, &row.PlayMethod, &row.Count); err != nil { + return nil, fmt.Errorf("scanning playback activity bucket: %w", err) + } + grouped = append(grouped, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating playback activity buckets: %w", err) + } + + activity := &AdminPlaybackActivity{ + Hours: hours, + BucketSeconds: bucketSeconds, + Buckets: assemblePlaybackBuckets(grouped), + } + + // profiles_active_24h deliberately excludes imported and provider-synced + // history so the tile means "watched on this server", not "appeared in + // someone's Trakt backlog". Manual marks stay in: they are on-server + // actions. + row := tx.QueryRow(ctx, adminPlaybackSessionsCTE+`, + reliability AS ( + SELECT + COUNT(*)::bigint AS sessions_started, + COUNT(*) FILTER (WHERE play_method = 'transcode')::bigint AS transcode_starts, + COUNT(*) FILTER (WHERE NOT live)::bigint AS finalized_sessions, + COUNT(*) FILTER (WHERE NOT live AND completed)::bigint AS completed_sessions, + COUNT(DISTINCT (user_id, profile_id))::bigint AS unique_profiles + FROM sessions + ), + active_profiles AS ( + SELECT COUNT(DISTINCT (user_id, profile_id))::bigint AS profiles_active_24h + FROM user_watch_history + WHERE watched_at >= now() - interval '24 hours' + AND COALESCE(source, 'legacy') IN ('legacy', 'manual', 'playback', 'jellycompat') + ) + SELECT + reliability.sessions_started, + reliability.transcode_starts, + reliability.finalized_sessions, + reliability.completed_sessions, + reliability.unique_profiles, + active_profiles.profiles_active_24h, + now() - make_interval(hours => $1) AS window_from, + now() AS window_to + FROM reliability + CROSS JOIN active_profiles + `, hours) + if err := row.Scan( + &activity.Reliability.SessionsStarted, + &activity.Reliability.TranscodeStarts, + &activity.Reliability.FinalizedSessions, + &activity.Reliability.CompletedSessions, + &activity.Reliability.UniqueProfiles, + &activity.ProfilesActive24h, + &activity.From, + &activity.To, + ); err != nil { + return nil, fmt.Errorf("querying playback reliability: %w", err) + } + activity.From = activity.From.UTC() + activity.To = activity.To.UTC() + activity.Reliability.CompletionRate = completionRate( + activity.Reliability.CompletedSessions, + activity.Reliability.FinalizedSessions, + ) + + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("closing playback activity read: %w", err) + } + return activity, nil +} diff --git a/internal/api/handlers/admin_stats_playback_test.go b/internal/api/handlers/admin_stats_playback_test.go new file mode 100644 index 000000000..c9a9e15a0 --- /dev/null +++ b/internal/api/handlers/admin_stats_playback_test.go @@ -0,0 +1,325 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// stubPlaybackActivitySource records what the handler asked for so the tests +// can assert on parameter handling without a database. +type stubPlaybackActivitySource struct { + activity *AdminPlaybackActivity + err error + gotHours int + invalidated int + callCount int +} + +func (s *stubPlaybackActivitySource) Get(_ context.Context, hours int) (*AdminPlaybackActivity, error) { + s.callCount++ + s.gotHours = hours + return s.activity, s.err +} + +func (s *stubPlaybackActivitySource) Invalidate() { s.invalidated++ } + +func TestAssemblePlaybackBuckets(t *testing.T) { + t.Parallel() + + hourOne := time.Date(2026, 8, 26, 10, 0, 0, 0, time.UTC) + hourTwo := time.Date(2026, 8, 26, 11, 0, 0, 0, time.UTC) + + tests := []struct { + name string + rows []playbackActivityRow + want []AdminPlaybackActivityBucket + }{ + { + name: "no rows yields an empty slice", + rows: nil, + want: []AdminPlaybackActivityBucket{}, + }, + { + name: "methods fold into one bucket per hour", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "direct", Count: 4}, + {Hour: hourOne, PlayMethod: "transcode", Count: 2}, + {Hour: hourTwo, PlayMethod: "remux", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 4, Transcode: 2}, + {Hour: hourTwo, Remux: 1}, + }, + }, + { + name: "history and live rows for the same hour and method add up", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "direct", Count: 3}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 4}, + }, + }, + { + name: "unknown play methods do not create a phantom series", + rows: []playbackActivityRow{ + {Hour: hourOne, PlayMethod: "", Count: 9}, + {Hour: hourOne, PlayMethod: "sorcery", Count: 5}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourOne, Direct: 1}, + }, + }, + { + name: "hours arrive in the query's order and keep it", + rows: []playbackActivityRow{ + {Hour: hourTwo, PlayMethod: "direct", Count: 1}, + {Hour: hourOne, PlayMethod: "direct", Count: 1}, + }, + want: []AdminPlaybackActivityBucket{ + {Hour: hourTwo, Direct: 1}, + {Hour: hourOne, Direct: 1}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := assemblePlaybackBuckets(tt.rows) + if len(got) != len(tt.want) { + t.Fatalf("buckets = %+v, want %+v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("bucket %d = %+v, want %+v", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestCompletionRate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + completed int64 + finalized int64 + want float64 + }{ + {name: "no finalized sessions is zero, not NaN", finalized: 0, completed: 0, want: 0}, + {name: "live-only window stays zero", finalized: 0, completed: 5, want: 0}, + {name: "everything completed", finalized: 4, completed: 4, want: 1}, + {name: "rounded to four decimals", finalized: 38, completed: 27, want: 0.7105}, + {name: "nothing completed", finalized: 9, completed: 0, want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := completionRate(tt.completed, tt.finalized); got != tt.want { + t.Fatalf("completionRate(%d, %d) = %v, want %v", tt.completed, tt.finalized, got, tt.want) + } + }) + } +} + +func TestPlaybackActivityBucketSeconds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hours int + want int + wantTrunc string + }{ + {name: "the minimum window is hourly", hours: adminPlaybackActivityMinHours, want: 3600, wantTrunc: "hour"}, + {name: "a day is hourly", hours: 24, want: 3600, wantTrunc: "hour"}, + {name: "two days is the last hourly window", hours: 48, want: 3600, wantTrunc: "hour"}, + {name: "just past two days becomes daily", hours: 49, want: 86400, wantTrunc: "day"}, + {name: "a week is daily", hours: 168, want: 86400, wantTrunc: "day"}, + {name: "the maximum window is daily", hours: adminPlaybackActivityMaxHours, want: 86400, wantTrunc: "day"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := playbackActivityBucketSeconds(tt.hours) + if got != tt.want { + t.Fatalf("playbackActivityBucketSeconds(%d) = %d, want %d", tt.hours, got, tt.want) + } + if trunc := playbackActivityTruncField(got); trunc != tt.wantTrunc { + t.Fatalf("playbackActivityTruncField(%d) = %q, want %q", got, trunc, tt.wantTrunc) + } + }) + } +} + +func TestHandleGetPlaybackActivityClampsHours(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + want int + }{ + {name: "absent uses the default window", query: "", want: adminPlaybackActivityDefaultHours}, + {name: "empty uses the default window", query: "?hours=", want: adminPlaybackActivityDefaultHours}, + {name: "explicit value passes through", query: "?hours=6", want: 6}, + {name: "a week passes through", query: "?hours=168", want: 168}, + {name: "zero clamps up", query: "?hours=0", want: adminPlaybackActivityMinHours}, + {name: "negative clamps up", query: "?hours=-12", want: adminPlaybackActivityMinHours}, + {name: "a month is the ceiling", query: "?hours=744", want: 744}, + {name: "oversized clamps down", query: "?hours=100000", want: adminPlaybackActivityMaxHours}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotHours != tt.want { + t.Fatalf("hours = %d, want %d", source.gotHours, tt.want) + } + }) + } +} + +func TestHandleGetPlaybackActivityRejectsNonNumericHours(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity?hours=soon", nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } +} + +func TestHandleGetPlaybackActivityRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{}} + handler := &AdminHandler{PlaybackActivitySource: source} + + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + if source.invalidated != 0 { + t.Fatalf("invalidated = %d on a plain read, want 0", source.invalidated) + } + + rec = httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity?refresh=1", nil)) + if source.invalidated != 1 { + t.Fatalf("invalidated = %d after refresh=1, want 1", source.invalidated) + } +} + +func TestHandleGetPlaybackActivityWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestHandleGetPlaybackActivityEmptyWindowSerializesArrays(t *testing.T) { + t.Parallel() + + source := &stubPlaybackActivitySource{activity: &AdminPlaybackActivity{ + Hours: 24, + BucketSeconds: adminPlaybackActivityHourSeconds, + Buckets: []AdminPlaybackActivityBucket{}, + }} + handler := &AdminHandler{PlaybackActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetPlaybackActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/playback-activity", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + Hours int `json:"hours"` + BucketSeconds int `json:"bucket_seconds"` + Buckets []json.RawMessage `json:"buckets"` + Reliability struct { + CompletionRate float64 `json:"completion_rate"` + } `json:"reliability"` + ProfilesActive24h int64 `json:"profiles_active_24h"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Buckets == nil { + t.Fatal("buckets decoded as null; a data-free server must send []") + } + if len(body.Buckets) != 0 || body.ProfilesActive24h != 0 || body.Reliability.CompletionRate != 0 { + t.Fatalf("unexpected body: %+v", body) + } + // The client zero-fills from bucket_seconds, so it has to survive + // serialization even on a server with nothing to report. + if body.BucketSeconds != adminPlaybackActivityHourSeconds { + t.Fatalf("bucket_seconds = %d, want %d", body.BucketSeconds, adminPlaybackActivityHourSeconds) + } +} + +func TestAdminPlaybackActivityProviderInvalidateClearsEveryWindow(t *testing.T) { + t.Parallel() + + provider, err := NewAdminPlaybackActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + provider.cache.Set(adminPlaybackActivityCachePrefix+"1", &AdminPlaybackActivity{Hours: 1}, time.Minute) + provider.cache.Set(adminPlaybackActivityCachePrefix+"24", &AdminPlaybackActivity{Hours: 24}, time.Minute) + + provider.Invalidate() + + for _, key := range []string{adminPlaybackActivityCachePrefix + "1", adminPlaybackActivityCachePrefix + "24"} { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminPlaybackActivityProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminPlaybackActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 24); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/admin_stats_test.go b/internal/api/handlers/admin_stats_test.go new file mode 100644 index 000000000..af0a44bb9 --- /dev/null +++ b/internal/api/handlers/admin_stats_test.go @@ -0,0 +1,150 @@ +package handlers + +import ( + "encoding/json" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +type stubWatchProviderLister struct { + summaries []watchsync.ProviderSummary +} + +func (s stubWatchProviderLister) List() []watchsync.ProviderSummary { return s.summaries } + +func providerStatsByKey(t *testing.T, stats []WatchProviderStats, key string) WatchProviderStats { + t.Helper() + for _, entry := range stats { + if entry.Provider == key { + return entry + } + } + t.Fatalf("provider %q missing from %+v", key, stats) + return WatchProviderStats{} +} + +func TestMergeWatchProviderStatsIncludesRegisteredProvidersWithoutActivity(t *testing.T) { + merged := mergeWatchProviderStats([]watchsync.ProviderSummary{ + { + Key: "trakt", + DisplayName: "Trakt", + Capabilities: watchsync.Capabilities{ExportWatched: true, ScrobblePlayback: true}, + }, + {Key: "mdblist", DisplayName: "MDBList"}, + }, nil) + + if len(merged) != 2 { + t.Fatalf("expected 2 providers, got %d: %+v", len(merged), merged) + } + + mdblist := providerStatsByKey(t, merged, "mdblist") + if !mdblist.Registered { + t.Fatalf("registered provider should be marked registered: %+v", mdblist) + } + if mdblist.DisplayName != "MDBList" { + t.Fatalf("display name = %q, want MDBList", mdblist.DisplayName) + } + if mdblist.ConnectedProfiles != 0 || mdblist.SyncRuns24h != 0 || mdblist.LastSyncCompletedAt != nil { + t.Fatalf("provider without activity should be all zeros: %+v", mdblist) + } + if mdblist.Scrobbling || mdblist.Exporting { + t.Fatalf("capabilities should follow the registry summary: %+v", mdblist) + } + + trakt := providerStatsByKey(t, merged, "trakt") + if !trakt.Scrobbling || !trakt.Exporting { + t.Fatalf("trakt capabilities not carried through: %+v", trakt) + } +} + +func TestMergeWatchProviderStatsOverlaysActivityAndOrdersByKey(t *testing.T) { + synced := time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC) + merged := mergeWatchProviderStats([]watchsync.ProviderSummary{ + {Key: "trakt", DisplayName: "Trakt"}, + {Key: "simkl", DisplayName: "Simkl"}, + }, []WatchProviderStats{ + { + Provider: "trakt", + ConnectedProfiles: 2, + SyncRuns24h: 5, + SyncErrors24h: 1, + ImportedWatched24h: 30, + ExportedWatched24h: 4, + PendingExports: 7, + LastSyncCompletedAt: &synced, + }, + }) + + got := make([]string, 0, len(merged)) + for _, entry := range merged { + got = append(got, entry.Provider) + } + // Deterministic order: sorted by provider key, not by map iteration. + if len(got) != 2 || got[0] != "simkl" || got[1] != "trakt" { + t.Fatalf("provider order = %v, want [simkl trakt]", got) + } + + trakt := providerStatsByKey(t, merged, "trakt") + if trakt.ConnectedProfiles != 2 || trakt.SyncRuns24h != 5 || trakt.PendingExports != 7 { + t.Fatalf("activity not overlaid onto the registered provider: %+v", trakt) + } + if trakt.LastSyncCompletedAt == nil || !trakt.LastSyncCompletedAt.Equal(synced) { + t.Fatalf("last sync time not carried through: %+v", trakt) + } + if trakt.DisplayName != "Trakt" { + t.Fatalf("registry display name should win over the key: %q", trakt.DisplayName) + } +} + +func TestMergeWatchProviderStatsKeepsUnregisteredProviderWithRows(t *testing.T) { + // A provider whose plugin was uninstalled still has rows in the + // watch-provider tables; its history stays visible, flagged as unregistered + // and named by its key. + merged := mergeWatchProviderStats([]watchsync.ProviderSummary{ + {Key: "trakt", DisplayName: "Trakt"}, + }, []WatchProviderStats{ + {Provider: "letterboxd", ConnectedProfiles: 1, SyncRuns24h: 3}, + }) + + legacy := providerStatsByKey(t, merged, "letterboxd") + if legacy.Registered { + t.Fatalf("unregistered provider should not be marked registered: %+v", legacy) + } + if legacy.DisplayName != "letterboxd" { + t.Fatalf("display name = %q, want the provider key", legacy.DisplayName) + } + if legacy.SyncRuns24h != 3 { + t.Fatalf("activity dropped for unregistered provider: %+v", legacy) + } +} + +func TestMergeWatchProviderStatsWithoutRegistrySerializesAsArray(t *testing.T) { + merged := mergeWatchProviderStats(listWatchProviders(nil), nil) + if merged == nil { + t.Fatal("expected an empty slice, not nil") + } + + encoded, err := json.Marshal(AdminStats{WatchProviders: merged}) + if err != nil { + t.Fatalf("marshal admin stats: %v", err) + } + var decoded map[string]json.RawMessage + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal admin stats: %v", err) + } + if string(decoded["watch_providers"]) != "[]" { + t.Fatalf("watch_providers = %s, want []", decoded["watch_providers"]) + } +} + +func TestListWatchProvidersToleratesNilRegistry(t *testing.T) { + var registry *watchsync.Registry + if got := listWatchProviders(registry); got != nil { + t.Fatalf("nil registry should list no providers, got %+v", got) + } + if got := listWatchProviders(stubWatchProviderLister{}); got != nil { + t.Fatalf("empty lister should list no providers, got %+v", got) + } +} diff --git a/internal/api/handlers/admin_stats_timeseries.go b/internal/api/handlers/admin_stats_timeseries.go new file mode 100644 index 000000000..0dabbfa2c --- /dev/null +++ b/internal/api/handlers/admin_stats_timeseries.go @@ -0,0 +1,356 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" +) + +const ( + adminTimeseriesCacheTTL = 30 * time.Second + adminTimeseriesCachePrefix = "hours=" + + adminTimeseriesDefaultHours = 24 + adminTimeseriesMinHours = 1 + // adminTimeseriesMaxHours is the sampler's retention window + // (internal/dashmetrics), so the widest window a client can ask for is the + // widest one the table can answer. + adminTimeseriesMaxHours = 744 + + // adminTimeseriesResolutionSeconds is the sampler's minute cadence + // (internal/dashmetrics) and therefore the finest bucket a read can return. + // The response reports the bucket it actually used, so the client sizes gaps + // from the data instead of hardcoding a second copy of this number. + adminTimeseriesResolutionSeconds = 60 + + // adminTimeseriesMaxPoints is the point budget a response aims to stay + // under. A dashboard widget is a few hundred CSS pixels wide, so a month of + // minutes would spend megabytes drawing sub-pixel detail nobody can see. + adminTimeseriesMaxPoints = 750 +) + +// timeseriesBucketSeconds picks the display bucket for a window, keeping every +// response under adminTimeseriesMaxPoints points. +// +// The thresholds are fixed rather than derived from the budget so that two +// clients asking for neighboring windows land on the same bucket — a bucket +// that drifted with the window would make consecutive reads incomparable, and +// the cache key (which is the window) would no longer imply the resolution. +func timeseriesBucketSeconds(hours int) int { + switch { + case hours <= 2: + return adminTimeseriesResolutionSeconds // 120 points at most + case hours <= 48: + return 300 // 5 minutes, 576 points at most + case hours <= 336: + return 1800 // 30 minutes, 672 points at most + default: + return 7200 // 2 hours, 372 points at most for a 31-day window + } +} + +// AdminTimeseriesPoint is one display bucket of sampled dashboard metrics — +// one sampled minute for short windows, several minutes collapsed for wide +// ones. Stream counts come from the cluster-wide "shared" sample; egress sums +// every source for a minute, so node egress and the egress each API process +// served are both included. +// +// A bucket that spans several minutes reports the peak minute of each column, +// never an average: concurrency and egress are read to answer "how bad did it +// get", and a mean would hide exactly that. +// +// EgressKbps keeps its pre-split meaning — every source's total viewer egress — +// so charts drawn from it alone stay truthful. DownloadEgressKbps is the +// additive file-transfer subset of that total (offline/direct downloads, ebook +// and ABS file fetches, measured by the API processes; node egress cannot be +// split and therefore counts entirely outside the subset). A client shows the +// split as download versus egress − download; the sampler clamps the subset +// under the total per minute, and both columns take their per-bucket MAX over +// the same minutes, so the difference is never negative. Samples written before +// the split report a zero subset. +type AdminTimeseriesPoint struct { + T time.Time `json:"t"` + Streams int64 `json:"streams"` + Direct int64 `json:"direct"` + Remux int64 `json:"remux"` + Transcode int64 `json:"transcode"` + EgressKbps int64 `json:"egress_kbps"` + DownloadEgressKbps int64 `json:"download_egress_kbps"` +} + +// AdminTimeseries is the GET /admin/stats/timeseries body. +// +// Buckets the sampler missed — a restart, a paused process, a server that was +// simply off — are absent from Points rather than zero-filled: a gap and an +// idle bucket are different facts and the chart draws them differently. +// OldestSampleAt is nil until the sampler has written anything, which is how +// the dashboard knows to say it is still collecting data. +// +// ResolutionSeconds is the bucket this window was aggregated into, not a +// constant: it widens with the requested window (timeseriesBucketSeconds). +type AdminTimeseries struct { + ResolutionSeconds int `json:"resolution_seconds"` + From time.Time `json:"from"` + To time.Time `json:"to"` + OldestSampleAt *time.Time `json:"oldest_sample_at"` + Points []AdminTimeseriesPoint `json:"points"` +} + +// AdminTimeseriesSource returns cached or freshly queried samples for a window +// of hours. +type AdminTimeseriesSource interface { + Get(ctx context.Context, hours int) (*AdminTimeseries, error) + Invalidate() +} + +// AdminTimeseriesProvider serves sampled dashboard metrics with a short +// in-process TTL and optional cross-node invalidation via the shared event bus, +// mirroring AdminStatsProvider. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries a slice. +type AdminTimeseriesProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminTimeseries] + ttl time.Duration +} + +var _ AdminTimeseriesSource = (*AdminTimeseriesProvider)(nil) + +// NewAdminTimeseriesProvider creates a cached provider and subscribes it to the +// playback/admin invalidation channels when an event bus is configured. +func NewAdminTimeseriesProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminTimeseriesProvider, error) { + provider := &AdminTimeseriesProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminTimeseries](), + ttl: adminTimeseriesCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin timeseries provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached window when available, otherwise queries Postgres. +// +// The window is clamped before the key is built, so two requests the server +// would answer identically share one entry. The bucket size is a function of +// the clamped window, so the key needs nothing else to stay honest. +func (p *AdminTimeseriesProvider) Get(ctx context.Context, hours int) (*AdminTimeseries, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin timeseries provider is not configured") + } + hours = clampQueryInt(hours, adminTimeseriesMinHours, adminTimeseriesMaxHours) + key := adminTimeseriesCachePrefix + strconv.Itoa(hours) + if series, ok := p.cache.Get(key); ok { + return series, nil + } + + series, err := queryAdminTimeseries(ctx, p.pool, hours) + if err != nil { + return nil, err + } + p.cache.Set(key, series, p.ttl) + return series, nil +} + +// Invalidate drops every cached window. +func (p *AdminTimeseriesProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminTimeseriesCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminTimeseriesProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetTimeseries handles GET /admin/stats/timeseries. +func (h *AdminHandler) HandleGetTimeseries(w http.ResponseWriter, r *http.Request) { + hours, err := parseClampedIntQuery( + r, "hours", + adminTimeseriesDefaultHours, + adminTimeseriesMinHours, + adminTimeseriesMaxHours, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var series *AdminTimeseries + switch { + case h.TimeseriesSource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.TimeseriesSource.Invalidate() + } + series, err = h.TimeseriesSource.Get(r.Context(), hours) + case h.pool != nil: + series, err = queryAdminTimeseries(r.Context(), h.pool, hours) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get dashboard timeseries") + return + } + + writeJSON(w, http.StatusOK, series) +} + +// timeseriesRow is one grouped display bucket as it comes back from Postgres. +// The stream counts are nullable: a bucket can hold process egress rows without +// a shared row when a replica sampled egress but every attempt at the shared +// row lost the race or failed. +type timeseriesRow struct { + Bucket time.Time + Streams *int64 + Direct *int64 + Remux *int64 + Transcode *int64 + EgressKbps int64 + DownloadEgressKbps int64 +} + +// assembleTimeseries turns grouped rows into response points, reading a missing +// shared sample as zero streams rather than dropping the bucket — the egress it +// carries is still real. +func assembleTimeseries(rows []timeseriesRow) []AdminTimeseriesPoint { + points := make([]AdminTimeseriesPoint, 0, len(rows)) + for _, row := range rows { + points = append(points, AdminTimeseriesPoint{ + T: row.Bucket.UTC(), + Streams: nullableCount(row.Streams), + Direct: nullableCount(row.Direct), + Remux: nullableCount(row.Remux), + Transcode: nullableCount(row.Transcode), + EgressKbps: row.EgressKbps, + DownloadEgressKbps: row.DownloadEgressKbps, + }) + } + return points +} + +func nullableCount(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func queryAdminTimeseries(ctx context.Context, pool *pgxpool.Pool, hours int) (*AdminTimeseries, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + bucketSeconds := timeseriesBucketSeconds(hours) + + // Two passes, in one statement. The inner one collapses the sources of a + // single minute: MAX over the shared source rather than SUM, because + // replicas may each have written the same minute's cluster-wide counts and + // those describe one cluster, not several — while egress is the opposite, + // since every source served different bytes. + // + // The outer one collapses those minutes into the display bucket by taking + // the peak minute of each column. Averaging would erase the spikes these + // charts exist to show; a bucket therefore answers "the worst minute in + // here", and its columns may come from different minutes. + rows, err := pool.Query(ctx, ` + WITH minutes AS ( + SELECT bucket, + MAX(streams_total) FILTER (WHERE source = 'shared') AS streams_total, + MAX(streams_direct) FILTER (WHERE source = 'shared') AS streams_direct, + MAX(streams_remux) FILTER (WHERE source = 'shared') AS streams_remux, + MAX(streams_transcode) FILTER (WHERE source = 'shared') AS streams_transcode, + COALESCE(SUM(egress_kbps), 0)::bigint AS egress_kbps, + COALESCE(SUM(download_egress_kbps), 0)::bigint AS download_egress_kbps + FROM dashboard_metric_samples + WHERE bucket >= now() - make_interval(hours => $1) + GROUP BY bucket + ) + SELECT date_bin(make_interval(secs => $2), bucket, 'epoch') AS display_bucket, + MAX(streams_total), + MAX(streams_direct), + MAX(streams_remux), + MAX(streams_transcode), + MAX(egress_kbps)::bigint, + MAX(download_egress_kbps)::bigint + FROM minutes + GROUP BY display_bucket + ORDER BY display_bucket + `, hours, float64(bucketSeconds)) + if err != nil { + return nil, fmt.Errorf("querying dashboard metric samples: %w", err) + } + defer rows.Close() + + grouped := make([]timeseriesRow, 0, hours*3600/bucketSeconds+1) + for rows.Next() { + var row timeseriesRow + if err := rows.Scan( + &row.Bucket, + &row.Streams, + &row.Direct, + &row.Remux, + &row.Transcode, + &row.EgressKbps, + &row.DownloadEgressKbps, + ); err != nil { + return nil, fmt.Errorf("scanning dashboard metric sample: %w", err) + } + grouped = append(grouped, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating dashboard metric samples: %w", err) + } + + series := &AdminTimeseries{ + ResolutionSeconds: bucketSeconds, + Points: assembleTimeseries(grouped), + } + + // The window bounds come from the same clock the samples were bucketed + // with, so a client comparing them against point timestamps never sees the + // API server's clock skew. + var oldest *time.Time + if err := pool.QueryRow(ctx, ` + SELECT MIN(bucket), + now() - make_interval(hours => $1), + now() + FROM dashboard_metric_samples + `, hours).Scan(&oldest, &series.From, &series.To); err != nil { + return nil, fmt.Errorf("querying dashboard metric sample window: %w", err) + } + if oldest != nil { + utc := oldest.UTC() + series.OldestSampleAt = &utc + } + series.From = series.From.UTC() + series.To = series.To.UTC() + + return series, nil +} diff --git a/internal/api/handlers/admin_stats_timeseries_test.go b/internal/api/handlers/admin_stats_timeseries_test.go new file mode 100644 index 000000000..e3d9b381c --- /dev/null +++ b/internal/api/handlers/admin_stats_timeseries_test.go @@ -0,0 +1,308 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// stubTimeseriesSource records what the handler asked for so the tests can +// assert on parameter handling without a database. +type stubTimeseriesSource struct { + series *AdminTimeseries + err error + gotHours int + invalidated int + callCount int +} + +func (s *stubTimeseriesSource) Get(_ context.Context, hours int) (*AdminTimeseries, error) { + s.callCount++ + s.gotHours = hours + return s.series, s.err +} + +func (s *stubTimeseriesSource) Invalidate() { s.invalidated++ } + +func int64Ptr(value int64) *int64 { return &value } + +func TestAssembleTimeseries(t *testing.T) { + t.Parallel() + + minuteOne := time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC) + minuteTwo := time.Date(2026, 8, 26, 12, 1, 0, 0, time.UTC) + + tests := []struct { + name string + rows []timeseriesRow + want []AdminTimeseriesPoint + }{ + { + name: "no samples yields an empty slice", + rows: nil, + want: []AdminTimeseriesPoint{}, + }, + { + name: "a full minute maps straight through", + rows: []timeseriesRow{{ + Bucket: minuteOne, + Streams: int64Ptr(3), + Direct: int64Ptr(1), + Remux: int64Ptr(0), + Transcode: int64Ptr(2), + EgressKbps: 48_211, + DownloadEgressKbps: 6_100, + }}, + want: []AdminTimeseriesPoint{ + {T: minuteOne, Streams: 3, Direct: 1, Remux: 0, Transcode: 2, EgressKbps: 48_211, DownloadEgressKbps: 6_100}, + }, + }, + { + name: "a minute with only process egress reads as zero streams", + rows: []timeseriesRow{{Bucket: minuteOne, EgressKbps: 900}}, + want: []AdminTimeseriesPoint{{T: minuteOne, EgressKbps: 900}}, + }, + { + name: "gaps are left as gaps, not zero-filled", + rows: []timeseriesRow{ + {Bucket: minuteOne, Streams: int64Ptr(1), EgressKbps: 10}, + {Bucket: minuteTwo, Streams: int64Ptr(2), EgressKbps: 20}, + }, + want: []AdminTimeseriesPoint{ + {T: minuteOne, Streams: 1, EgressKbps: 10}, + {T: minuteTwo, Streams: 2, EgressKbps: 20}, + }, + }, + { + name: "buckets are reported in UTC", + rows: []timeseriesRow{{Bucket: minuteOne.In(time.FixedZone("UTC-5", -5*60*60)), Streams: int64Ptr(1)}}, + want: []AdminTimeseriesPoint{{T: minuteOne, Streams: 1}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := assembleTimeseries(tt.rows) + if len(got) != len(tt.want) { + t.Fatalf("points = %+v, want %+v", got, tt.want) + } + for i := range got { + if !got[i].T.Equal(tt.want[i].T) { + t.Fatalf("point %d time = %s, want %s", i, got[i].T, tt.want[i].T) + } + // Equal ignores the location, so the instant matching is not + // enough: the wire format is UTC, and a dropped .UTC() would + // serialize an offset the charts do not expect. + if loc := got[i].T.Location(); loc != time.UTC { + t.Fatalf("point %d location = %s, want UTC", i, loc) + } + if got[i].Streams != tt.want[i].Streams || + got[i].Direct != tt.want[i].Direct || + got[i].Remux != tt.want[i].Remux || + got[i].Transcode != tt.want[i].Transcode || + got[i].EgressKbps != tt.want[i].EgressKbps || + got[i].DownloadEgressKbps != tt.want[i].DownloadEgressKbps { + t.Fatalf("point %d = %+v, want %+v", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestTimeseriesBucketSeconds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hours int + want int + }{ + {name: "the minimum window keeps sampled minutes", hours: adminTimeseriesMinHours, want: 60}, + {name: "two hours is the last minute-resolution window", hours: 2, want: 60}, + {name: "just past two hours steps to five minutes", hours: 3, want: 300}, + {name: "a day is five minutes", hours: 24, want: 300}, + {name: "two days is the last five-minute window", hours: 48, want: 300}, + {name: "just past two days steps to half an hour", hours: 49, want: 1800}, + {name: "a week is half an hour", hours: 168, want: 1800}, + {name: "a fortnight is the last half-hour window", hours: 336, want: 1800}, + {name: "just past a fortnight steps to two hours", hours: 337, want: 7200}, + {name: "the maximum window is two hours", hours: adminTimeseriesMaxHours, want: 7200}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := timeseriesBucketSeconds(tt.hours) + if got != tt.want { + t.Fatalf("timeseriesBucketSeconds(%d) = %d, want %d", tt.hours, got, tt.want) + } + // The point budget is the reason the buckets exist; a threshold + // that drifts past it silently regresses every wide window. + if points := tt.hours * 3600 / got; points > adminTimeseriesMaxPoints { + t.Fatalf("a %d-hour window yields %d points at %ds buckets, over the %d budget", + tt.hours, points, got, adminTimeseriesMaxPoints) + } + }) + } +} + +func TestHandleGetTimeseriesClampsHours(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + want int + }{ + {name: "absent uses the default window", query: "", want: adminTimeseriesDefaultHours}, + {name: "empty uses the default window", query: "?hours=", want: adminTimeseriesDefaultHours}, + {name: "explicit value passes through", query: "?hours=1", want: 1}, + {name: "a week passes through", query: "?hours=168", want: 168}, + {name: "zero clamps up", query: "?hours=0", want: adminTimeseriesMinHours}, + {name: "negative clamps up", query: "?hours=-3", want: adminTimeseriesMinHours}, + {name: "the retention window is the ceiling", query: "?hours=744", want: 744}, + {name: "past retention clamps down", query: "?hours=100000", want: adminTimeseriesMaxHours}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotHours != tt.want { + t.Fatalf("hours = %d, want %d", source.gotHours, tt.want) + } + }) + } +} + +func TestHandleGetTimeseriesRejectsNonNumericHours(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries?hours=lots", nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } +} + +func TestHandleGetTimeseriesRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{}} + handler := &AdminHandler{TimeseriesSource: source} + + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + if source.invalidated != 0 { + t.Fatalf("invalidated = %d on a plain read, want 0", source.invalidated) + } + + rec = httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries?refresh=1", nil)) + if source.invalidated != 1 { + t.Fatalf("invalidated = %d after refresh=1, want 1", source.invalidated) + } +} + +func TestHandleGetTimeseriesWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestHandleGetTimeseriesFreshInstallSerializesCollectingState(t *testing.T) { + t.Parallel() + + source := &stubTimeseriesSource{series: &AdminTimeseries{ + ResolutionSeconds: adminTimeseriesResolutionSeconds, + Points: []AdminTimeseriesPoint{}, + }} + handler := &AdminHandler{TimeseriesSource: source} + rec := httptest.NewRecorder() + handler.HandleGetTimeseries(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/timeseries", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + ResolutionSeconds int `json:"resolution_seconds"` + Points []json.RawMessage `json:"points"` + OldestSampleAt *string `json:"oldest_sample_at"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Points == nil { + t.Fatal("points decoded as null; a server with no samples must send []") + } + if len(body.Points) != 0 { + t.Fatalf("points = %v, want empty", body.Points) + } + if body.OldestSampleAt != nil { + t.Fatalf("oldest_sample_at = %v, want null before the first sample", *body.OldestSampleAt) + } + if body.ResolutionSeconds != adminTimeseriesResolutionSeconds { + t.Fatalf("resolution_seconds = %d, want %d", body.ResolutionSeconds, adminTimeseriesResolutionSeconds) + } +} + +func TestAdminTimeseriesProviderInvalidateClearsEveryWindow(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTimeseriesProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + provider.cache.Set(adminTimeseriesCachePrefix+"1", &AdminTimeseries{}, time.Minute) + provider.cache.Set(adminTimeseriesCachePrefix+"24", &AdminTimeseries{}, time.Minute) + + provider.Invalidate() + + for _, key := range []string{adminTimeseriesCachePrefix + "1", adminTimeseriesCachePrefix + "24"} { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminTimeseriesProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTimeseriesProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 24); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/admin_stats_top.go b/internal/api/handlers/admin_stats_top.go new file mode 100644 index 000000000..baafc7fd3 --- /dev/null +++ b/internal/api/handlers/admin_stats_top.go @@ -0,0 +1,364 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/cache" +) + +const ( + adminTopActivityCacheTTL = 5 * time.Minute + adminTopActivityCachePrefix = "days=" + + adminTopActivityDefaultDays = 7 + adminTopActivityMinDays = 1 + adminTopActivityMaxDays = 30 + + adminTopActivityDefaultLimit = 10 + adminTopActivityMinLimit = 1 + adminTopActivityMaxLimit = 25 +) + +// adminTopActivityWatchSourceFilter keeps only history that originated on this +// server, so the leaderboards describe what people actually played here. +// `manual` (marked-watched) stays in as an on-server action, and `jellycompat` +// is real playback through the Jellyfin surface. This is an allowlist rather +// than a denylist of known providers because plugin watch providers store +// their own arbitrary keys in `source` — a denylist would silently count any +// newly installed provider's imported backlog as local plays. +const adminTopActivityWatchSourceFilter = `COALESCE(h.source, 'legacy') IN ('legacy', 'manual', 'playback', 'jellycompat')` + +// AdminTopTitle is one row of the most-watched-titles list. Episodes are rolled +// up to their series, so media_item_id is a series content id for TV. +type AdminTopTitle struct { + MediaItemID string `json:"media_item_id"` + Title string `json:"title"` + MediaType string `json:"media_type"` + Plays int64 `json:"plays"` + // TotalSeconds is watched time summed from finalized playback sessions, not + // the runtime of the titles played. A title that was only ever marked + // watched has no sessions and reports 0. + TotalSeconds int64 `json:"total_seconds"` +} + +// AdminTopProfile is one row of the most-active-profiles list. +type AdminTopProfile struct { + UserID int `json:"user_id"` + Username string `json:"username"` + ProfileID string `json:"profile_id"` + ProfileName string `json:"profile_name"` + Plays int64 `json:"plays"` + // TotalSeconds is watched time summed from this profile's finalized + // playback sessions, not the runtime of what it played. + TotalSeconds int64 `json:"total_seconds"` +} + +// AdminTopActivity is the GET /admin/stats/top-activity body. +type AdminTopActivity struct { + Days int `json:"days"` + Limit int `json:"limit"` + Titles []AdminTopTitle `json:"titles"` + Profiles []AdminTopProfile `json:"profiles"` +} + +// AdminTopActivitySource returns cached or freshly queried leaderboards. +type AdminTopActivitySource interface { + Get(ctx context.Context, days, limit int) (*AdminTopActivity, error) + Invalidate() +} + +// AdminTopActivityProvider serves the leaderboards with a longer TTL than the +// other dashboard aggregates: a seven-day ranking barely moves within minutes, +// and the query is the most expensive one on the page. +// +// The cached payload is a pointer because cache.TTLCache requires a comparable +// value type and this struct carries slices. +type AdminTopActivityProvider struct { + pool *pgxpool.Pool + cache *cache.TTLCache[*AdminTopActivity] + ttl time.Duration +} + +var _ AdminTopActivitySource = (*AdminTopActivityProvider)(nil) + +// NewAdminTopActivityProvider creates a cached provider and subscribes it to +// the playback/admin invalidation channels when an event bus is configured. +func NewAdminTopActivityProvider(ctx context.Context, pool *pgxpool.Pool, bus cache.EventBus) (*AdminTopActivityProvider, error) { + provider := &AdminTopActivityProvider{ + pool: pool, + cache: cache.NewTTLCache[*AdminTopActivity](), + ttl: adminTopActivityCacheTTL, + } + + if bus == nil || ctx == nil { + return provider, nil + } + + handler := func(cache.Event) { + provider.Invalidate() + } + for _, channel := range []string{cache.ChannelAdmin, cache.ChannelPlayback} { + if err := bus.Subscribe(ctx, channel, handler); err != nil { + provider.Close() + return nil, fmt.Errorf("subscribing admin top activity provider to %s: %w", channel, err) + } + } + + return provider, nil +} + +// Get returns the cached leaderboards when available, otherwise queries Postgres. +func (p *AdminTopActivityProvider) Get(ctx context.Context, days, limit int) (*AdminTopActivity, error) { + if p == nil || p.pool == nil { + return nil, fmt.Errorf("admin top activity provider is not configured") + } + days = clampQueryInt(days, adminTopActivityMinDays, adminTopActivityMaxDays) + limit = clampQueryInt(limit, adminTopActivityMinLimit, adminTopActivityMaxLimit) + key := adminTopActivityCachePrefix + strconv.Itoa(days) + "&limit=" + strconv.Itoa(limit) + if activity, ok := p.cache.Get(key); ok { + return activity, nil + } + + activity, err := queryAdminTopActivity(ctx, p.pool, days, limit) + if err != nil { + return nil, err + } + p.cache.Set(key, activity, p.ttl) + return activity, nil +} + +// Invalidate drops every cached window. +func (p *AdminTopActivityProvider) Invalidate() { + if p == nil || p.cache == nil { + return + } + p.cache.InvalidatePrefix(adminTopActivityCachePrefix) +} + +// Close stops the background TTL sweeper. +func (p *AdminTopActivityProvider) Close() { + if p == nil || p.cache == nil { + return + } + p.cache.Close() +} + +// HandleGetTopActivity handles GET /admin/stats/top-activity. +func (h *AdminHandler) HandleGetTopActivity(w http.ResponseWriter, r *http.Request) { + days, err := parseClampedIntQuery( + r, "days", + adminTopActivityDefaultDays, + adminTopActivityMinDays, + adminTopActivityMaxDays, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + limit, err := parseClampedIntQuery( + r, "limit", + adminTopActivityDefaultLimit, + adminTopActivityMinLimit, + adminTopActivityMaxLimit, + ) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + var activity *AdminTopActivity + switch { + case h.TopActivitySource != nil: + if isTruthyQuery(r.URL.Query().Get("refresh")) { + h.TopActivitySource.Invalidate() + } + activity, err = h.TopActivitySource.Get(r.Context(), days, limit) + case h.pool != nil: + activity, err = queryAdminTopActivity(r.Context(), h.pool, days, limit) + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Database not configured") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get top activity") + return + } + + writeJSON(w, http.StatusOK, activity) +} + +func queryAdminTopActivity(ctx context.Context, pool *pgxpool.Pool, days, limit int) (*AdminTopActivity, error) { + if pool == nil { + return nil, fmt.Errorf("database not configured") + } + + activity := &AdminTopActivity{ + Days: days, + Limit: limit, + Titles: []AdminTopTitle{}, + Profiles: []AdminTopProfile{}, + } + + // Episodes roll up to their series: a season binge should read as one show, + // not twelve one-play entries. + // + // Plays come from user_watch_history (marked-watched counts as a play), + // while total_seconds is watched time summed from finalized playback + // sessions — user_watch_history.duration_seconds is the media's full + // runtime, so summing it would report three hours for a movie someone + // abandoned after a minute. A title that was only ever marked watched has + // no session rows and reports 0 seconds. Sessions are windowed on ended_at + // — the same stop instant watched_at records — so a session that started + // before the cutoff but stopped inside the window counts toward both plays + // and watch time rather than only the former. + // + // watched_seconds records the final absolute position, not elapsed viewing + // time, so a session resumed at the one-hour mark would claim the first + // hour again; each session's contribution is therefore capped at its + // wall-clock length. Still an estimate — recording true elapsed playback + // needs a session start position, which does not exist yet. + // + // The ranking is computed and limited first so the title lookup and the + // watched-seconds aggregate only run over the rows that survive. + titleRows, err := pool.Query(ctx, ` + WITH ranked AS ( + SELECT COALESCE(ep.series_id, h.media_item_id) AS item_id, + bool_or(ep.content_id IS NOT NULL) AS is_series, + COUNT(*)::bigint AS plays + FROM user_watch_history h + LEFT JOIN episodes ep ON ep.content_id = h.media_item_id + WHERE h.watched_at >= now() - make_interval(days => $1) + AND `+adminTopActivityWatchSourceFilter+` + GROUP BY 1 + ORDER BY plays DESC, item_id + LIMIT $2 + ), + watched AS ( + SELECT COALESCE(ep.series_id, p.media_item_id) AS item_id, + SUM(LEAST(GREATEST(p.watched_seconds, 0), + GREATEST(EXTRACT(EPOCH FROM (p.ended_at - p.started_at)), 0))) AS total_seconds + FROM playback_history_admin p + LEFT JOIN episodes ep ON ep.content_id = p.media_item_id + WHERE p.ended_at >= now() - make_interval(days => $1) + AND COALESCE(ep.series_id, p.media_item_id) IN (SELECT item_id FROM ranked) + GROUP BY 1 + ) + SELECT r.item_id, + COALESCE(mi.title, '') AS title, + COALESCE(CASE WHEN r.is_series THEN 'series' ELSE mi.type END, '') AS media_type, + r.plays, + COALESCE(w.total_seconds, 0)::bigint AS total_seconds + FROM ranked r + LEFT JOIN media_items mi ON mi.content_id = r.item_id + LEFT JOIN watched w ON w.item_id = r.item_id + ORDER BY r.plays DESC, total_seconds DESC, r.item_id + `, days, limit) + if err != nil { + return nil, fmt.Errorf("querying top titles: %w", err) + } + defer titleRows.Close() + + for titleRows.Next() { + var title AdminTopTitle + if err := titleRows.Scan( + &title.MediaItemID, + &title.Title, + &title.MediaType, + &title.Plays, + &title.TotalSeconds, + ); err != nil { + return nil, fmt.Errorf("scanning top title: %w", err) + } + activity.Titles = append(activity.Titles, title) + } + if err := titleRows.Err(); err != nil { + return nil, fmt.Errorf("iterating top titles: %w", err) + } + titleRows.Close() + + // Profile display names live in the per-user stores, not in + // user_watch_history, so they are read back from the most recent admin + // playback-history row for that profile. A profile that has only ever been + // marked-watched has no such row and falls back to its id. + // + // The ranking groups on (user_id, profile_id) alone: a profile that was + // renamed mid-window would otherwise split into two rows. The name lookup + // and the watched-seconds aggregate run after the limit, once per surviving + // profile rather than once per history row. As above, total_seconds is + // watched time from finalized playback sessions. + profileRows, err := pool.Query(ctx, ` + WITH ranked AS ( + SELECT h.user_id, + h.profile_id, + COUNT(*)::bigint AS plays + FROM user_watch_history h + WHERE h.watched_at >= now() - make_interval(days => $1) + AND `+adminTopActivityWatchSourceFilter+` + GROUP BY 1, 2 + ORDER BY plays DESC, h.user_id, h.profile_id + LIMIT $2 + ), + watched AS ( + SELECT p.user_id, + p.profile_id, + SUM(LEAST(GREATEST(p.watched_seconds, 0), + GREATEST(EXTRACT(EPOCH FROM (p.ended_at - p.started_at)), 0))) AS total_seconds + FROM playback_history_admin p + WHERE p.ended_at >= now() - make_interval(days => $1) + AND EXISTS ( + SELECT 1 FROM ranked r + WHERE r.user_id = p.user_id AND r.profile_id = p.profile_id + ) + GROUP BY 1, 2 + ) + SELECT r.user_id, + COALESCE(u.username, '') AS username, + r.profile_id, + COALESCE(pn.profile_name, r.profile_id) AS profile_name, + r.plays, + COALESCE(w.total_seconds, 0)::bigint AS total_seconds + FROM ranked r + LEFT JOIN users u ON u.id = r.user_id + LEFT JOIN watched w ON w.user_id = r.user_id AND w.profile_id = r.profile_id + LEFT JOIN LATERAL ( + SELECT p.profile_name + FROM playback_history_admin p + WHERE p.user_id = r.user_id + AND p.profile_id = r.profile_id + AND p.profile_name <> '' + ORDER BY p.ended_at DESC + LIMIT 1 + ) pn ON TRUE + ORDER BY r.plays DESC, total_seconds DESC, r.user_id, r.profile_id + `, days, limit) + if err != nil { + return nil, fmt.Errorf("querying top profiles: %w", err) + } + defer profileRows.Close() + + for profileRows.Next() { + var profile AdminTopProfile + if err := profileRows.Scan( + &profile.UserID, + &profile.Username, + &profile.ProfileID, + &profile.ProfileName, + &profile.Plays, + &profile.TotalSeconds, + ); err != nil { + return nil, fmt.Errorf("scanning top profile: %w", err) + } + activity.Profiles = append(activity.Profiles, profile) + } + if err := profileRows.Err(); err != nil { + return nil, fmt.Errorf("iterating top profiles: %w", err) + } + + return activity, nil +} diff --git a/internal/api/handlers/admin_stats_top_test.go b/internal/api/handlers/admin_stats_top_test.go new file mode 100644 index 000000000..41d3e5653 --- /dev/null +++ b/internal/api/handlers/admin_stats_top_test.go @@ -0,0 +1,213 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type stubTopActivitySource struct { + activity *AdminTopActivity + err error + gotDays int + gotLimit int + invalidated int + callCount int +} + +func (s *stubTopActivitySource) Get(_ context.Context, days, limit int) (*AdminTopActivity, error) { + s.callCount++ + s.gotDays = days + s.gotLimit = limit + return s.activity, s.err +} + +func (s *stubTopActivitySource) Invalidate() { s.invalidated++ } + +func TestHandleGetTopActivityClampsParams(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantDays int + wantLimit int + }{ + { + name: "defaults", + query: "", + wantDays: adminTopActivityDefaultDays, + wantLimit: adminTopActivityDefaultLimit, + }, + { + name: "explicit values pass through", + query: "?days=14&limit=5", + wantDays: 14, + wantLimit: 5, + }, + { + name: "zero clamps up", + query: "?days=0&limit=0", + wantDays: adminTopActivityMinDays, + wantLimit: adminTopActivityMinLimit, + }, + { + name: "oversized clamps down", + query: "?days=999&limit=999", + wantDays: adminTopActivityMaxDays, + wantLimit: adminTopActivityMaxLimit, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity"+tt.query, nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body = %s", rec.Code, rec.Body.String()) + } + if source.gotDays != tt.wantDays || source.gotLimit != tt.wantLimit { + t.Fatalf("days/limit = %d/%d, want %d/%d", source.gotDays, source.gotLimit, tt.wantDays, tt.wantLimit) + } + }) + } +} + +func TestHandleGetTopActivityRejectsNonNumericParams(t *testing.T) { + t.Parallel() + + for _, query := range []string{"?days=week", "?limit=all"} { + t.Run(query, func(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity"+query, nil)) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400, body = %s", rec.Code, rec.Body.String()) + } + if source.callCount != 0 { + t.Fatalf("source was queried %d times for an invalid request", source.callCount) + } + }) + } +} + +func TestHandleGetTopActivityRefreshInvalidates(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{}} + handler := &AdminHandler{TopActivitySource: source} + + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity?refresh=true", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if source.invalidated != 1 { + t.Fatalf("invalidated = %d, want 1", source.invalidated) + } +} + +func TestHandleGetTopActivitySourceFailureIs500(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{err: errors.New("boom")} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +func TestHandleGetTopActivityWithoutDatabase(t *testing.T) { + t.Parallel() + + handler := &AdminHandler{} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } +} + +// A server with no watch history must answer with empty lists rather than +// nulls: the bar-list widgets map over these fields directly. +func TestAdminTopActivityEmptyListsSerializeAsArrays(t *testing.T) { + t.Parallel() + + source := &stubTopActivitySource{activity: &AdminTopActivity{ + Days: 7, + Limit: 10, + Titles: []AdminTopTitle{}, + Profiles: []AdminTopProfile{}, + }} + handler := &AdminHandler{TopActivitySource: source} + rec := httptest.NewRecorder() + handler.HandleGetTopActivity(rec, httptest.NewRequest(http.MethodGet, "/admin/stats/top-activity", nil)) + + var body struct { + Titles []AdminTopTitle `json:"titles"` + Profiles []AdminTopProfile `json:"profiles"` + } + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Titles == nil || body.Profiles == nil { + t.Fatalf("titles/profiles decoded as null: %s", rec.Body.String()) + } +} + +func TestAdminTopActivityProviderInvalidateClearsEveryVariant(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTopActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + keys := []string{ + adminTopActivityCachePrefix + "7&limit=10", + adminTopActivityCachePrefix + "30&limit=25", + } + for _, key := range keys { + provider.cache.Set(key, &AdminTopActivity{}, time.Minute) + } + + provider.Invalidate() + + for _, key := range keys { + if _, ok := provider.cache.Get(key); ok { + t.Fatalf("%s survived Invalidate", key) + } + } +} + +func TestAdminTopActivityProviderWithoutPool(t *testing.T) { + t.Parallel() + + provider, err := NewAdminTopActivityProvider(context.Background(), nil, nil) + if err != nil { + t.Fatalf("new provider: %v", err) + } + t.Cleanup(provider.Close) + + if _, err := provider.Get(context.Background(), 7, 10); err == nil { + t.Fatal("expected an error from a provider with no pool") + } +} diff --git a/internal/api/handlers/catalog.go b/internal/api/handlers/catalog.go index 00812e45c..867176d92 100644 --- a/internal/api/handlers/catalog.go +++ b/internal/api/handlers/catalog.go @@ -119,16 +119,7 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request if groupedByWork { result, entries, err := h.resolveGroupedCatalogByWork(r, req, accessFilter) if err != nil { - if errors.Is(err, catalog.ErrInvalidCatalogRequest) { - writeError(w, http.StatusBadRequest, "bad_request", err.Error()) - return - } - if errors.Is(err, catalog.ErrCatalogSourceNotFound) { - writeError(w, http.StatusNotFound, "not_found", "Catalog source not found") - return - } - slog.ErrorContext(r.Context(), "catalog: resolve grouped by work failed", "component", "api", "err_msg", err.Error()) - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to resolve catalog") + handleCatalogResolveError(w, r, err, true) return } @@ -145,25 +136,39 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request result, err := h.resolver.Resolve(r.Context(), req, accessFilter) if err != nil { - if errors.Is(err, catalog.ErrInvalidCatalogRequest) { - writeError(w, http.StatusBadRequest, "bad_request", err.Error()) - return - } - if errors.Is(err, catalog.ErrCatalogSourceNotFound) { - writeError(w, http.StatusNotFound, "not_found", "Catalog source not found") - return - } - if handleCatalogSearchContextError(w, r, err) { - return - } - slog.ErrorContext(r.Context(), "catalog: resolve failed", "component", "api", "err_msg", err.Error()) - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to resolve catalog") + handleCatalogResolveError(w, r, err, false) return } items := h.catalogItemResponses(r, result.Items, catalogSortMetricField(req, result), playableTargetLibraryIDs(req), accessFilter) h.writeCatalogResponse(w, result, items, groupedByWork) } +// handleCatalogResolveError keeps grouped and ordinary catalog resolution on +// the same error contract. In particular, a grouped search still runs through +// the bounded PostgreSQL path and must return search_timeout rather than hiding +// a deadline behind the generic 500 response. +func handleCatalogResolveError(w http.ResponseWriter, r *http.Request, err error, groupedByWork bool) { + if errors.Is(err, catalog.ErrInvalidCatalogRequest) { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if errors.Is(err, catalog.ErrCatalogSourceNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Catalog source not found") + return + } + if handleCatalogSearchContextError(w, r, err) { + return + } + slog.ErrorContext( + r.Context(), + "catalog: resolve failed", + "component", "api", + "grouped_by_work", groupedByWork, + "err_msg", err.Error(), + ) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to resolve catalog") +} + // handleCatalogSearchContextError translates bounded-search termination without // treating it as an ordinary server fault. Superseded live queries cancel their // request context, so there is no client left to receive a response. A server diff --git a/internal/api/handlers/catalog_diagnostics_test.go b/internal/api/handlers/catalog_diagnostics_test.go index ea8473346..4091505da 100644 --- a/internal/api/handlers/catalog_diagnostics_test.go +++ b/internal/api/handlers/catalog_diagnostics_test.go @@ -153,3 +153,24 @@ func TestHandleCatalogSearchContextError_CanceledRequestWritesNothing(t *testing t.Fatalf("canceled request wrote a response body: %q", rec.Body.String()) } } + +func TestHandleCatalogResolveError_GroupedDeadlineReturnsRetryableTimeout(t *testing.T) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/catalog?source=query&q=slow&group=work", nil) + handleCatalogResolveError( + rec, + req, + errors.Join(errors.New("grouped search failed"), context.DeadlineExceeded), + true, + ) + if rec.Code != http.StatusGatewayTimeout { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusGatewayTimeout, rec.Body.String()) + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode grouped timeout body: %v; body=%s", err, rec.Body.String()) + } + if body["error"] != "search_timeout" { + t.Fatalf("error = %v, want search_timeout; body=%v", body["error"], body) + } +} diff --git a/internal/api/handlers/downloads.go b/internal/api/handlers/downloads.go index 288f12d81..5c4bffd04 100644 --- a/internal/api/handlers/downloads.go +++ b/internal/api/handlers/downloads.go @@ -644,7 +644,10 @@ func (h *DownloadHandler) redirectToProxy(w http.ResponseWriter, r *http.Request releaseReservation() return false, fmt.Errorf("sign proxy download token: %w", err) } - location := strings.TrimRight(plan.ProxyNode.URL, "/") + "/downloads/file/" + url.PathEscape(token) + // The location is what the client downloads from, so it uses the proxy's + // client-facing URL; the cache key below stays on the canonical backend + // URL, which is the node's identity everywhere else. + location := strings.TrimRight(plan.ProxyNode.ClientURL(), "/") + "/downloads/file/" + url.PathEscape(token) targetKey := target.Path if target.OriginArtifactID != "" { targetKey = target.OriginNodeURL + "\x00" + target.OriginArtifactID diff --git a/internal/api/handlers/library_collections.go b/internal/api/handlers/library_collections.go index 1ffbbf6a1..5a58be2e2 100644 --- a/internal/api/handlers/library_collections.go +++ b/internal/api/handlers/library_collections.go @@ -289,12 +289,10 @@ func (h *LibraryCollectionHandler) GenerateCollectionPoster(ctx context.Context, return nil } - // Delete any existing auto-generated images before uploading new ones. - if err := h.deleteCollectionImages(ctx, collectionID, collectionPosterImageType); err != nil { - slog.WarnContext(ctx, "collage: failed to clean up old poster images", "component", "api", "collection_id", collectionID, "error", err) - } - // Process through the standard image pipeline (generates WebP variants + thumbhash). + // Variant keys are deterministic, so successful puts replace the complete + // prior set. Keeping the old set until then prevents a processing or upload + // failure from breaking the poster currently referenced by the database. s3Path, thumbhash, err := h.processCollectionImage(ctx, collectionID, collectionPosterImageType, composited) if err != nil { return fmt.Errorf("processing collage image: %w", err) @@ -1469,14 +1467,11 @@ func applyCollectionPosterURLUpdate(input *catalog.UpdateLibraryCollectionInput, if input == nil || posterURL == nil { return } - input.PosterURL = posterURL + canonicalURL := strings.TrimSpace(*posterURL) + input.PosterURL = &canonicalURL notGenerated := false - suppressed := strings.TrimSpace(*posterURL) == "" + suppressed := canonicalURL == "" emptyThumbhash := "" - if suppressed { - emptyURL := "" - input.PosterURL = &emptyURL - } input.PosterAutoGenerated = ¬Generated input.PosterSuppressed = &suppressed input.PosterFromTemplate = ¬Generated @@ -1554,7 +1549,12 @@ func (h *LibraryCollectionHandler) deleteServerCollection(ctx context.Context, c if err != nil { return fmt.Errorf("locking collection poster mutation before collection delete: %w", err) } - defer unlock() + locked := true + defer func() { + if locked { + unlock() + } + }() if h.SectionRepo != nil { refs, err := h.SectionRepo.CountLibraryCollectionReferences(ctx, collectionID, "") @@ -1566,9 +1566,18 @@ func (h *LibraryCollectionHandler) deleteServerCollection(ctx context.Context, c } } - // Clean up S3 images before deleting the collection row. Failures here - // only leak storage; the collection row delete must still proceed so the - // admin's request succeeds. + // Delete database state while poster writers are excluded, then release the + // pinned advisory-lock connection before doing best-effort object cleanup. + // A storage failure can leak variants, but cannot resurrect the collection. + if err := h.repo.Delete(ctx, collectionID); err != nil { + return err + } + unlock() + locked = false + + if h.SortPreferenceCleaner != nil { + h.SortPreferenceCleaner.DeleteForCollection(ctx, userstore.CollectionKindLibrary, collectionID) + } if h.s3GP != nil { prefix := fmt.Sprintf("collection-images/%s/", collectionID) keys, err := h.s3GP.ListObjects(ctx, h.s3GP.Bucket(), prefix) @@ -1583,13 +1592,6 @@ func (h *LibraryCollectionHandler) deleteServerCollection(ctx context.Context, c } } } - - if err := h.repo.Delete(ctx, collectionID); err != nil { - return err - } - if h.SortPreferenceCleaner != nil { - h.SortPreferenceCleaner.DeleteForCollection(ctx, userstore.CollectionKindLibrary, collectionID) - } return nil } @@ -3862,15 +3864,16 @@ func (h *LibraryCollectionHandler) replaceCollectionImage( return fmt.Errorf("locking collection poster mutation before replacement: %w", err) } defer unlock() + // Fail before deleting the existing variants if the collection vanished + // while an upload request was in flight. if _, err := h.repo.GetByID(ctx, collectionID); err != nil { return fmt.Errorf("reloading collection before poster replacement: %w", err) } } - if err := h.deleteCollectionImages(ctx, collectionID, imageType); err != nil { - return fmt.Errorf("deleting %s images: %w", imageType, err) - } - + // Variant keys are deterministic. Uploading in place preserves the prior + // complete set when image processing fails, while successful puts replace + // every key used by this image type. s3Path, thumbhash, err := h.processCollectionImage(ctx, collectionID, imageType, fileData) if err != nil { return fmt.Errorf("%s: %w", imageType, err) diff --git a/internal/api/handlers/library_collections_test.go b/internal/api/handlers/library_collections_test.go index 87cd8006d..656b370cd 100644 --- a/internal/api/handlers/library_collections_test.go +++ b/internal/api/handlers/library_collections_test.go @@ -64,19 +64,21 @@ func TestApplyCollectionPosterURLUpdate(t *testing.T) { tests := []struct { name string posterURL string + wantURL string wantSuppressed bool }{ - {name: "empty suppresses automatic poster", posterURL: "", wantSuppressed: true}, - {name: "whitespace suppresses automatic poster", posterURL: " ", wantSuppressed: true}, - {name: "nonempty restores custom poster", posterURL: "https://example.test/poster.jpg", wantSuppressed: false}, + {name: "empty suppresses automatic poster", posterURL: "", wantURL: "", wantSuppressed: true}, + {name: "whitespace suppresses automatic poster", posterURL: " ", wantURL: "", wantSuppressed: true}, + {name: "nonempty restores custom poster", posterURL: "https://example.test/poster.jpg", wantURL: "https://example.test/poster.jpg", wantSuppressed: false}, + {name: "nonempty URL is canonicalized", posterURL: " https://example.test/poster.jpg ", wantURL: "https://example.test/poster.jpg", wantSuppressed: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input := catalog.UpdateLibraryCollectionInput{} applyCollectionPosterURLUpdate(&input, &tt.posterURL) - if tt.wantSuppressed && (input.PosterURL == nil || *input.PosterURL != "") { - t.Fatalf("poster URL = %v, want canonical empty string", input.PosterURL) + if input.PosterURL == nil || *input.PosterURL != tt.wantURL { + t.Fatalf("poster URL = %v, want %q", input.PosterURL, tt.wantURL) } if input.PosterSuppressed == nil || *input.PosterSuppressed != tt.wantSuppressed { t.Fatalf("poster suppressed = %v, want %t", input.PosterSuppressed, tt.wantSuppressed) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 79690c90b..ca49f46c9 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -7,14 +7,19 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "net/http" "strconv" + "strings" "sync" "time" "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/go-chi/chi/v5" "github.com/redis/go-redis/v9" ) @@ -26,7 +31,7 @@ type NodeRepository interface { Create(ctx context.Context, input nodepool.CreateNodeInput) (*nodepool.Node, error) Update(ctx context.Context, id int, input nodepool.UpdateNodeInput) (*nodepool.Node, error) Delete(ctx context.Context, id int) error - UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int) error + UpdateHealth(ctx context.Context, id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte) error } // NodeListEnabled queries enabled nodes by type for pool reload. @@ -34,6 +39,17 @@ type NodeListEnabled interface { ListEnabled(ctx context.Context, nodeType string) ([]*nodepool.Node, error) } +// NodeCapabilityRefresher fetches, persists, and publishes one node's +// capability report immediately rather than waiting for the background sweep. +// +// It is an interface only so the re-probe handler can be tested without a live +// sweep; *nodepool.HealthChecker is the single implementation, and the handler +// deliberately does not reimplement any of the fetch, drift, or persist logic +// that lives behind it. +type NodeCapabilityRefresher interface { + RefreshNodeCapabilities(ctx context.Context, n *nodepool.Node) error +} + // NodeHandler handles CRUD operations and health checks for stream nodes. type NodeHandler struct { repo NodeRepository @@ -43,6 +59,64 @@ type NodeHandler struct { eventBus cache.EventBus redisClient *redis.Client // for reading session keys jwtSecret string // for bearer auth when calling force-reload on nodes + // capabilities refreshes a node's stored inventory on demand; nil in a + // deployment with no health checker, where a re-probe still runs on the node + // and the stored row catches up on the next sweep. + capabilities NodeCapabilityRefresher + // invalidateCapabilityCache drops one node's cached protocol-v3 planning + // inventory. nil outside integrated mode, where there is no playback handler + // holding one. + invalidateCapabilityCache func(nodeURL string) + // afterNodeUpdate fires once the post-commit work an update kicks off has + // finished. Tests wait on it rather than on a sleep; production leaves it + // nil. + afterNodeUpdate func() + // clusterPlayback reports the cluster-wide acceleration policy, which is + // what a node without an override of its own runs. Read live rather than + // snapshotted, because it is hot-reloadable. nil where nothing wired it, and + // then a node's own override is all this handler can price from. + clusterPlayback func() config.PlaybackConfig +} + +// SetClusterPlaybackPolicy wires the live cluster-wide acceleration policy, +// which the re-probe budget needs to resolve a node that carries no override of +// its own. Set after construction like the other collaborators here. +func (h *NodeHandler) SetClusterPlaybackPolicy(policy func() config.PlaybackConfig) { + if h == nil { + return + } + h.clusterPlayback = policy +} + +// playbackPolicy is the cluster-wide acceleration policy, or the zero policy +// where none was wired — which prices the default device set rather than +// nothing. +func (h *NodeHandler) playbackPolicy() config.PlaybackConfig { + if h == nil || h.clusterPlayback == nil { + return config.PlaybackConfig{} + } + return h.clusterPlayback() +} + +// SetCapabilityInvalidator wires the planning-cache drop used after a node's +// acceleration policy changes. Like SetCapabilityRefresher it is set after +// construction, because the playback handler that owns the cache is built +// before the router reaches this route group. +func (h *NodeHandler) SetCapabilityInvalidator(invalidate func(nodeURL string)) { + if h == nil { + return + } + h.invalidateCapabilityCache = invalidate +} + +// SetCapabilityRefresher wires the on-demand capability refresh used after a +// re-probe. It is set after construction because the health checker is built +// before the router and owned by the process, not by this handler. +func (h *NodeHandler) SetCapabilityRefresher(refresher NodeCapabilityRefresher) { + if h == nil { + return + } + h.capabilities = refresher } // NewNodeHandler creates a new NodeHandler. @@ -71,9 +145,19 @@ type checkNodeResult struct { Healthy bool `json:"healthy"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash is what the node advertised on this check. It is not the + // stored hash: an unequal pair means the background sweep has a refetch to + // do, which is exactly what an operator checking a node wants to see. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` } // HandleListNodes handles GET /admin/nodes. +// +// The stored rows are served as they are: physical_gpu_keys is derived by the +// node store when a row is scanned, so the same identities the planner routes +// on are the ones an operator sees. The one thing added is the hash each node +// advertised on its last health check, which lives only in the pools — see +// overlayAdvertisedHashes. func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { nodes, err := h.repo.List(r.Context()) if err != nil { @@ -81,10 +165,44 @@ func (h *NodeHandler) HandleListNodes(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list nodes") return } - + if nodes == nil { + nodes = []*nodepool.Node{} + } + h.overlayAdvertisedHashes(nodes) writeJSON(w, http.StatusOK, nodes) } +// overlayAdvertisedHashes copies each node's last advertised capability hash +// from the pools onto the rows this endpoint serves. +// +// The hash is an observation from the health sweep rather than stored state, so +// it exists only on the pool's copy of a node while the row comes from the +// database. Without this the field is always absent and the admin page cannot +// tell a report the node has already contradicted from a current one — the very +// case a recent last_health_check cannot rule out, because that check keeps +// succeeding while the refetch behind it fails. +func (h *NodeHandler) overlayAdvertisedHashes(nodes []*nodepool.Node) { + advertised := make(map[int]*string, len(nodes)) + collect := func(pooled []*nodepool.Node) { + for _, n := range pooled { + if n != nil && n.AdvertisedCapabilitiesHash != nil { + advertised[n.ID] = n.AdvertisedCapabilitiesHash + } + } + } + if h.proxyPool != nil { + collect(h.proxyPool.Nodes()) + } + if h.transcodePool != nil { + collect(h.transcodePool.Nodes()) + } + for _, n := range nodes { + if n != nil { + n.AdvertisedCapabilitiesHash = advertised[n.ID] + } + } +} + // HandleCreateNode handles POST /admin/nodes. func (h *NodeHandler) HandleCreateNode(w http.ResponseWriter, r *http.Request) { var input nodepool.CreateNodeInput @@ -125,21 +243,184 @@ func (h *NodeHandler) HandleUpdateNode(w http.ResponseWriter, r *http.Request) { return } + // Read the row before the write so the nudge below can tell a real policy + // change from a resubmit: the admin form posts every field on each save, so + // a field being present says nothing about it moving. + // + // The URL counts for the same reason the overrides do. A partial PUT that + // carries only a new url still repoints the row's existing overrides at a + // different worker, and without the old row to compare against + // nodePolicyTargetChanged has nothing to see — the replacement would keep + // running on what it inherited until its own 60s poll. + var previous *nodepool.Node + if input.URL != nil || input.HWAccelOverride != nil || input.HWDeviceOverride != nil { + previous, _ = h.repo.GetByID(r.Context(), id) + } + node, err := h.repo.Update(r.Context(), id, input) if err != nil { if errors.Is(err, nodepool.ErrNodeNotFound) { writeError(w, http.StatusNotFound, "not_found", "Node not found") return } + if errors.Is(err, nodepool.ErrInvalidNodeInput) { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } slog.ErrorContext(r.Context(), "updating node", "component", "api", "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update node") return } writeJSON(w, http.StatusOK, node) - h.reloadPools(r.Context()) + // Order matters: the node has to adopt its policy before this server starts + // dispatching under it. reloadPools publishes the updated row, and + // EffectiveHWAccel then names its backend on every start request; the node + // itself only re-reads its row on the config watcher's 60s poll. Without the + // nudge, an operator switching both overlays at once (QSV on a render node + // to NVENC on a CUDA index, say) — or repointing the row at a worker that + // has been running on what it inherited — gets up to a minute of requests + // pairing one side's backend with the other side's device. + // Off the request goroutine, not merely detached from its context. + // + // writeJSON above does not end the response — the handler returning does — + // so anything after it is latency the operator waits through. The node nudge + // alone is bounded at ten seconds against a worker that may be unreachable, + // which is long enough for the admin form to sit in "Saving..." and time out + // after a database write that already succeeded. The row is committed at + // this point; none of what follows can change the answer already written, so + // none of it belongs in front of the client. + policyChanged := nodePolicyTargetChanged(previous, node) + ctx := context.WithoutCancel(r.Context()) + go func() { + if policyChanged { + if !h.reloadNodeConfig(ctx, node) { + // The node did not confirm. Its backend now comes from this + // server's pool while its device still comes from its own + // configuration, so until its poll catches up (within 60s) a + // start request can pair the new backend with the old device + // and fail. + // + // The policy is published anyway. Withholding it would leave an + // override an operator has saved and can see stored never + // reaching dispatch at all — nothing else re-reads the column — + // which is a silent permanent misconfiguration rather than a + // loud, bounded, self-healing one. Closing the window properly + // means sending the effective device alongside the backend so + // both come from one source; that is a change to the node start + // contract, not something to slip into a policy edit. + slog.WarnContext(ctx, "node has not adopted its new acceleration policy yet; transcodes dispatched to it may fail until its next config poll", + "component", "api", "node_id", node.ID, "name", node.Name) + } + // And drop what this server believes the node can do. The v3 + // planning cache holds the tone-map executors and transformation + // inventory the *old* backend advertised, and it stays valid for + // its own TTL — so without this a session started in the next + // minute is planned against the previous backend's filters and then + // rejected by the worker that has already moved on. Dropped before + // the pool reload, for the same reason the worker is nudged first: + // nothing should dispatch under the new policy while stale + // capabilities are still readable. + if h.invalidateCapabilityCache != nil { + h.invalidateCapabilityCache(node.URL) + } + } + h.reloadPools(ctx) + if h.afterNodeUpdate != nil { + h.afterNodeUpdate() + } + }() +} + +// nodePolicyTargetChanged reports whether this update changed which effective +// acceleration policy applies, or to which worker. +// +// Two ways that happens, and both open the same window. The obvious one is an +// override moving. The other is the row being repointed: the overrides may be +// byte-identical, but they now describe a *different* worker, one that has been +// running under whatever it inherited and will not learn otherwise until its own +// 60s poll. reloadPools publishes the new URL immediately, so between those two +// moments this server dispatches the row's overridden backend to a worker still +// holding its inherited device — the same mismatch, reached by a different edit. +// +// The admin form submits every field on each save, so their presence in the body +// is not evidence of a change; without this, renaming a node or editing its +// capacity would nudge it too. An unreadable previous row reports no change: the +// nudge is an optimization over the node's own config poll, and skipping it +// costs at most that interval. +func nodePolicyTargetChanged(before, after *nodepool.Node) bool { + if before == nil || after == nil { + return false + } + if !sameNodeURL(before.URL, after.URL) { + return true + } + return !sameOptionalString(before.HWAccelOverride, after.HWAccelOverride) || + !sameOptionalString(before.HWDeviceOverride, after.HWDeviceOverride) +} + +// sameNodeURL compares two stored node URLs the way the pools and the database +// fences do, so a trailing slash is not a repoint. +func sameNodeURL(a, b string) bool { + return strings.TrimRight(a, "/") == strings.TrimRight(b, "/") +} + +func sameOptionalString(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +// reloadNodeConfig asks one node to re-read its configuration now, reporting +// whether it confirmed. +// +// It targets /admin/reload-config, never /admin/force-reload: the latter tears +// down every live playback session on a transcode node, which is a reasonable +// thing for an operator to ask for explicitly and an unacceptable side effect +// of saving a policy edit that the UI says applies to new transcodes. +// +// Best effort by design: the override is already stored, and the node's own +// watcher poll is the backstop, so a node that is unreachable or predates the +// route must not turn a successful update into a failed one. It is bounded +// tightly for the same reason — this runs after the response is written, but +// still on the request's goroutine and context. The caller uses the result to +// say how far out of step the node may be, not to fail the update. +func (h *NodeHandler) reloadNodeConfig(ctx context.Context, node *nodepool.Node) bool { + if node == nil || node.URL == "" { + return false + } + ctx, cancel := context.WithTimeout(ctx, nodeConfigReloadTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/reload-config"), nil) + if err != nil { + return false + } + req.Header.Set("Authorization", "Bearer "+h.jwtSecret) + resp, err := (&http.Client{Timeout: nodeConfigReloadTimeout}).Do(req) + if err != nil { + slog.WarnContext(ctx, "node did not reload after an acceleration override change; it will pick it up on its next config poll", + "component", "api", "node_id", node.ID, "name", node.Name, "error", logredact.SanitizeURLError(err)) + return false + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + // The route answers 204; accepting the whole 2xx family keeps this from + // warning about a reload that in fact succeeded, and leaves room for the + // node's answer to gain a body later. + if resp.StatusCode < 200 || resp.StatusCode > 299 { + slog.WarnContext(ctx, "node refused the reload after an acceleration override change; it will pick it up on its next config poll", + "component", "api", "node_id", node.ID, "name", node.Name, "status", resp.StatusCode) + return false + } + return true } +// nodeConfigReloadTimeout bounds the post-update nudge. A config reload is a +// database read on the node, not a probe, so this only has to cover a round +// trip to a node that is up. +const nodeConfigReloadTimeout = 10 * time.Second + // HandleDeleteNode handles DELETE /admin/nodes/{id}. func (h *NodeHandler) HandleDeleteNode(w http.ResponseWriter, r *http.Request) { id, err := parseIDParam(r) @@ -182,19 +463,48 @@ func (h *NodeHandler) HandleCheckNode(w http.ResponseWriter, r *http.Request) { return } - healthy, activeJobs, egressKbps := nodepool.CheckNode(r.Context(), node) + healthy, activeJobs, egressKbps, capabilitiesHash, lastStats := nodepool.CheckNode(r.Context(), node) - if err := h.repo.UpdateHealth(r.Context(), id, healthy, activeJobs, egressKbps); err != nil { + if err := h.repo.UpdateHealth(r.Context(), id, node.URL, healthy, activeJobs, egressKbps, lastStats); err != nil { slog.ErrorContext(r.Context(), "persisting health check result", "component", "api", "node_id", id, "error", err) } + // The pools get it too, exactly as the background sweep would. A manual + // check that only wrote the row would leave the planner admitting work to a + // node whose scratch volume this check just found full, and would pair a + // fresh last_health_check in the database with the pool's older advertised + // hash — which is the combination the Nodes page reads as "this inventory + // was reconfirmed", for up to the next 30 seconds. + h.applyHealthToPools(node, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats) writeJSON(w, http.StatusOK, checkNodeResult{ - Healthy: healthy, - ActiveJobs: activeJobs, - EgressKbps: egressKbps, + Healthy: healthy, + ActiveJobs: activeJobs, + EgressKbps: egressKbps, + CapabilitiesHash: capabilitiesHash, }) } +// applyHealthToPools publishes one check's result to whichever pool holds the +// node, so an operator-triggered check lands everywhere the sweep's would. +// +// Fenced on the node's URL by the pools themselves, like every other health +// write: the row can be repointed while a check is in flight. +func (h *NodeHandler) applyHealthToPools( + node *nodepool.Node, healthy bool, activeJobs, egressKbps int, capabilitiesHash string, lastStats []byte, +) { + checkedAt := time.Now() + switch node.Type { + case nodepool.NodeTypeProxy: + if h.proxyPool != nil { + h.proxyPool.ApplyHealth(node.ID, node.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, checkedAt) + } + case nodepool.NodeTypeTranscode: + if h.transcodePool != nil { + h.transcodePool.ApplyHealth(node.ID, node.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, checkedAt) + } + } +} + // HandleForceReloadNodes handles POST /admin/nodes/force-reload — sends a // force-reload signal to every enabled node in parallel. func (h *NodeHandler) HandleForceReloadNodes(w http.ResponseWriter, r *http.Request) { @@ -219,7 +529,7 @@ func (h *NodeHandler) HandleForceReloadNodes(w http.ResponseWriter, r *http.Requ defer wg.Done() result := ForceReloadResult{NodeID: node.ID, NodeName: node.Name} client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequestWithContext(ctx, http.MethodPost, node.URL+"/admin/force-reload", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/force-reload"), nil) if err != nil { result.Status = "error" result.Error = err.Error() @@ -268,7 +578,7 @@ func (h *NodeHandler) HandleForceReloadNode(w http.ResponseWriter, r *http.Reque } client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, node.URL+"/admin/force-reload", nil) + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/force-reload"), nil) if err != nil { writeError(w, http.StatusInternalServerError, "internal_error", err.Error()) return @@ -300,6 +610,244 @@ func (h *NodeHandler) HandleForceReloadNode(w http.ResponseWriter, r *http.Reque }}}) } +// ReprobeNodeResult is the JSON response for an operator-triggered capability +// re-probe of one node. +type ReprobeNodeResult struct { + NodeID int `json:"node_id"` + NodeName string `json:"node_name"` + // Status is "ok" when the node re-probed successfully, "error" otherwise. + // A node that refused or could not be reached is reported here rather than + // as an HTTP error status, matching the per-node check and force-reload + // routes: the request to the API succeeded, the node is what failed. + Status string `json:"status"` + Error string `json:"error,omitempty"` + // Resolved is the backend the node picked after re-probing. + Resolved string `json:"resolved,omitempty"` + // CapabilityHash identifies the snapshot the node published. Comparing it + // against the hash in the node list before the call is what tells an + // operator whether the re-probe changed anything. + CapabilityHash string `json:"capability_hash,omitempty"` + // CapabilitiesRefreshed reports whether this server also refetched and + // stored the node's new inventory before answering. False means the stored + // row will catch up on a later health sweep instead. + CapabilitiesRefreshed bool `json:"capabilities_refreshed"` +} + +// nodeReprobeFallbackTimeout bounds the re-probe request to a node whose stored +// capability report does not advertise a probe budget — one that has never been +// inventoried, or that runs a build predating the advertisement. +// +// A re-probe deliberately discards the node's probe caches, so it pays the full +// cold cost: a hardware walk plus the whole tone-map matrix, each bounded by +// several ffmpeg execs. This is the same order of magnitude as the health +// sweep's own two-minute capability fetch bound, plus request slack, and is only +// a fallback: a node that has been inventoried once advertises its real budget. +const nodeReprobeFallbackTimeout = 150 * time.Second + +// nodeReprobeTimeout is how long the API waits for one node's re-probe. +// +// The node itself publishes the budget its probe matrix needs in every +// capability report (probe_request_timeout_ms), which is exactly what a cold +// capability fetch is given; a node with different hardware or a slower ffmpeg +// therefore gets its own number rather than a cluster-wide guess. +// +// Floored at what the node's current policy prices, for the same reason the +// planning and download paths are: a report stored before an operator widened +// hw_device_override advertises a budget for the smaller device set, and a +// re-probe deliberately discards every cache on the node, so it runs the larger +// matrix and gets canceled at the old deadline. Nothing here learns its way out +// of that — a re-probe is exactly the request that never completes. +func (h *NodeHandler) nodeReprobeTimeout(n *nodepool.Node) time.Duration { + cluster := h.playbackPolicy() + return playback.ColdCapabilityRequestTimeout( + n.StoredCapabilities(), + n.EffectiveHWAccel(cluster.HWAccel), + n.EffectiveHWDevice(cluster.HWDevice), + nodeReprobeFallbackTimeout, + ) +} + +// capabilityRefreshBound is how long the refresh after a re-probe may run for +// this node, asked of the thing that will enforce it. +// +// The bound is derived from the node's own advertised probe budget, so it is +// node-specific and routinely past the exported floor — a node with a large +// device set asks for well over five minutes. Computing it here from a second +// rule would be the same number derived twice, and the two would disagree +// exactly where it matters. Without a refresher wired there is no refresh to +// wait for, so the floor is all this can promise. +func (h *NodeHandler) capabilityRefreshBound(n *nodepool.Node) time.Duration { + if bounder, ok := h.capabilities.(nodeCapabilityRefreshBounder); ok { + return bounder.CapabilityRefreshBound(n) + } + return nodepool.CapabilityRefreshTimeout +} + +// nodeCapabilityRefreshBounder reports how long a refresh of one node may take. +// Optional on NodeCapabilityRefresher, like the other collaborators here; +// *nodepool.HealthChecker implements it. +type nodeCapabilityRefreshBounder interface { + CapabilityRefreshBound(n *nodepool.Node) time.Duration +} + +// nodeReprobeWriteSlack covers the repository round trips and the JSON write +// that bracket the two long calls this handler makes. +const nodeReprobeWriteSlack = 15 * time.Second + +// extendReprobeWriteDeadline lifts this connection's write deadline to cover the +// whole action: the node's own probe budget, then the capability refetch that +// stores the result. +// +// The API listener's WriteTimeout is 120 seconds, and this route can legitimately +// outlive it — a node with two render devices advertises a probe budget past two +// minutes on its own, before the refetch adds its bound. Without this the +// deadline expires after the node has already re-probed and this server has +// already persisted the report: the operator's browser sees a torn connection, +// the UI toasts a failure for an action that succeeded, and the obvious response +// is to run the whole cold FFmpeg matrix again. The diagnostics upload route +// extends its deadlines for exactly the same reason. +func (h *NodeHandler) extendReprobeWriteDeadline( + w http.ResponseWriter, r *http.Request, n *nodepool.Node, probeBudget time.Duration, +) { + budget := probeBudget + h.capabilityRefreshBound(n) + nodeReprobeWriteSlack + if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(budget)); err != nil { + // A ResponseWriter that cannot carry a deadline (a test recorder, a + // wrapper that does not unwrap) is not a reason to refuse the action. + slog.WarnContext(r.Context(), "node re-probe write deadline not extended", "component", "api", + "budget", budget, "error", err) + } +} + +// HandleReprobeNode handles POST /admin/nodes/{id}/reprobe — asks one node to +// discard its cached hardware-probe verdicts and re-verify against live +// hardware, then stores the resulting inventory immediately. +// +// This is the operator's answer to hardware that stopped working underneath a +// running node. A node caches a successful probe for its whole process lifetime +// (re-verifying per request would put ffmpeg execs on the playback path), so a +// GPU that has since been removed, or whose driver was replaced, keeps reading +// "verified" until the node restarts. Nothing about that is visible in a health +// check, because the node is perfectly healthy either way. The opposite +// direction self-heals: a failed GPU probe is retried after a short negative +// TTL, so a repaired driver reaches the list on the next capability snapshot +// without this action. +// +// A node that is transcoding refuses: the probe smoke-encodes on the GPU, and a +// busy encoder would report working hardware as failed. +// +// Both halves matter: the node recomputes, and this server then refetches and +// persists the report so the admin list, the pools, and the planner's GPU +// identities agree with it without waiting up to a sweep interval. The refresh +// reuses the health checker's own machinery, so drift detection and persistence +// have exactly one implementation. +func (h *NodeHandler) HandleReprobeNode(w http.ResponseWriter, r *http.Request) { + id, err := parseIDParam(r) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid node ID") + return + } + + node, err := h.repo.GetByID(r.Context(), id) + if err != nil { + if errors.Is(err, nodepool.ErrNodeNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Node not found") + return + } + slog.ErrorContext(r.Context(), "fetching node for re-probe", "component", "api", "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch node") + return + } + + h.extendReprobeWriteDeadline(w, r, node, h.nodeReprobeTimeout(node)) + + result := ReprobeNodeResult{NodeID: node.ID, NodeName: node.Name, Status: "ok"} + reprobed, err := h.reprobeNode(r.Context(), node) + if err != nil { + slog.WarnContext(r.Context(), "node capability re-probe failed", "component", "api", + "node_id", node.ID, "name", node.Name, "error", err) + result.Status = "error" + result.Error = err.Error() + writeJSON(w, http.StatusOK, result) + return + } + result.Resolved = reprobed.Resolved + result.CapabilityHash = reprobed.CapabilityHash + + // The node has already recomputed at this point, so a refresh failure is + // reported alongside a successful re-probe rather than turning it into one: + // the next sweep will store the report, and saying the re-probe failed + // would invite an operator to run it again for nothing. + if h.capabilities == nil { + writeJSON(w, http.StatusOK, result) + return + } + if err := h.capabilities.RefreshNodeCapabilities(r.Context(), node); err != nil { + slog.WarnContext(r.Context(), "storing re-probed node capabilities failed", "component", "api", + "node_id", node.ID, "name", node.Name, "error", err) + } else { + result.CapabilitiesRefreshed = true + } + writeJSON(w, http.StatusOK, result) +} + +// nodeReprobeResponse is the node's own answer to /admin/reprobe-capabilities. +type nodeReprobeResponse struct { + Resolved string `json:"resolved"` + CapabilityHash string `json:"capability_hash"` +} + +// reprobeNode performs the bearer-authenticated re-probe call against one node, +// mirroring the force-reload client. +func (h *NodeHandler) reprobeNode(ctx context.Context, node *nodepool.Node) (nodeReprobeResponse, error) { + timeout := h.nodeReprobeTimeout(node) + client := &http.Client{Timeout: timeout} + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/admin/reprobe-capabilities"), nil) + if err != nil { + return nodeReprobeResponse{}, err + } + req.Header.Set("Authorization", "Bearer "+h.jwtSecret) + + resp, err := client.Do(req) + if err != nil { + return nodeReprobeResponse{}, logredact.SanitizeURLError(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + // Read the small error body rather than discarding it: it carries the + // node's own explanation, and reading it also lets the transport reuse + // the connection. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + switch resp.StatusCode { + case http.StatusServiceUnavailable: + // The node's own degraded-snapshot answer: it kept its previous + // hash rather than publishing a partial probe. + return nodeReprobeResponse{}, errors.New("node could not complete its hardware probe; its previous capability report was kept") + case http.StatusConflict: + // The node refused because it is transcoding: a probe's smoke encode + // competes with live sessions for encoder slots, and losing that + // race would be published as failed hardware. + if message := strings.TrimSpace(string(body)); message != "" { + return nodeReprobeResponse{}, errors.New(message) + } + return nodeReprobeResponse{}, errors.New("node is busy transcoding; retry the re-probe when it is idle") + } + return nodeReprobeResponse{}, fmt.Errorf("node re-probe returned status %d", resp.StatusCode) + } + var decoded nodeReprobeResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, maxNodeReprobeResponseBytes)).Decode(&decoded); err != nil { + return nodeReprobeResponse{}, err + } + return decoded, nil +} + +// maxNodeReprobeResponseBytes bounds the node's answer. It carries two short +// strings; the bound is only there because a node is a worker that may run on +// remote hardware. +const maxNodeReprobeResponseBytes = 8 << 10 + // HandleListSessions handles GET /admin/nodes/sessions — lists active playback // sessions from Redis, optionally filtered by node_id query parameter. func (h *NodeHandler) HandleListSessions(w http.ResponseWriter, r *http.Request) { @@ -354,11 +902,27 @@ func (h *NodeHandler) HandleListSessions(w http.ResponseWriter, r *http.Request) writeJSON(w, http.StatusOK, sessionsResponse{Sessions: sessions}) } -// reloadPools refreshes the in-memory proxy and transcode pools from the database. +// nodePostCommitTimeout bounds work that runs after the response is written and +// therefore no longer has a client waiting on it. +const nodePostCommitTimeout = 15 * time.Second + +// reloadPools refreshes the in-memory proxy and transcode pools from the +// database and tells every replica to do the same. +// +// Every caller reaches here after its response is written, so the request +// context is the wrong lifetime: a client that disconnects — or an admin who +// navigates away from a slow save — cancels it, and this would then fail both +// database reads and return without publishing EventNodePoolChanged. The row is +// already committed at that point, so this instance and every replica would go +// on dispatching under the old policy indefinitely; nothing else re-reads the +// column. Detaching from that cancellation, bounded, is what makes the write and +// its publication one outcome instead of two. func (h *NodeHandler) reloadPools(ctx context.Context) { if h.lister == nil { return } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), nodePostCommitTimeout) + defer cancel() proxyNodes, proxyErr := h.lister.ListEnabled(ctx, nodepool.NodeTypeProxy) transcodeNodes, tcErr := h.lister.ListEnabled(ctx, nodepool.NodeTypeTranscode) if proxyErr != nil || tcErr != nil { diff --git a/internal/api/handlers/nodes_reprobe_test.go b/internal/api/handlers/nodes_reprobe_test.go new file mode 100644 index 000000000..30d4f8aba --- /dev/null +++ b/internal/api/handlers/nodes_reprobe_test.go @@ -0,0 +1,382 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" + "github.com/go-chi/chi/v5" +) + +// stubCapabilityRefresher stands in for the health checker's on-demand refresh. +type stubCapabilityRefresher struct { + calls atomic.Int32 + err error +} + +func (s *stubCapabilityRefresher) RefreshNodeCapabilities(context.Context, *nodepool.Node) error { + s.calls.Add(1) + return s.err +} + +func reprobeRequest(t *testing.T) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/admin/nodes/1/reprobe", nil) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("id", "1") + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) +} + +// fakeNodeServer answers the node-side re-probe route with the given status and +// body, recording the bearer token it was given. +func fakeNodeServer(t *testing.T, status int, body string) (url string, authorization *string) { + t.Helper() + seen := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/admin/reprobe-capabilities" || r.Method != http.MethodPost { + w.WriteHeader(http.StatusNotFound) + return + } + seen = r.Header.Get("Authorization") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server.URL, &seen +} + +// The point of the API-side action is that the operator does not have to wait a +// sweep interval: the node re-probes, and this server immediately refetches and +// stores the report so the list, the pools, and the planner agree with it. +func TestHandleReprobeNodeRefreshesStoredCapabilities(t *testing.T) { + url, authorization := fakeNodeServer(t, http.StatusOK, + `{"resolved":"qsv","capability_hash":"sha256:new"}`) + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.Error != "" { + t.Fatalf("result = %+v, want a clean ok", result) + } + if result.Resolved != "qsv" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result did not carry the node's answer: %+v", result) + } + if !result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = false, want the stored row refreshed on success") + } + if got := refresher.calls.Load(); got != 1 { + t.Fatalf("refresh calls = %d, want exactly one", got) + } + if *authorization != "Bearer secret" { + t.Fatalf("node saw authorization %q, want the bearer secret", *authorization) + } +} + +// A node whose probe could not complete answers 503 and keeps its previous +// hash. That has to reach the operator as a named failure, and must not trigger +// a capability refetch — refetching would store the report the node explicitly +// declined to republish. +func TestHandleReprobeNodeSurfacesNodeFailure(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusServiceUnavailable, "capability probe unavailable\n") + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + // The API request itself succeeded; the node is what failed, exactly as the + // per-node check and force-reload routes report it. + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "error" || result.Error == "" { + t.Fatalf("result = %+v, want a reported node error", result) + } + if !strings.Contains(result.Error, "hardware probe") { + t.Fatalf("error = %q, want it to explain the degraded probe", result.Error) + } + if result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = true after a failed re-probe") + } + if got := refresher.calls.Load(); got != 0 { + t.Fatalf("refresh calls = %d, want none after a failed re-probe", got) + } +} + +// A refresh failure is reported beside a successful re-probe rather than turning +// it into one: the node has already recomputed, and the next sweep will store +// the report. +func TestHandleReprobeNodeReportsRefreshFailureSeparately(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusOK, `{"resolved":"none","capability_hash":"sha256:new"}`) + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{err: nodepool.ErrCapabilityRefreshInFlight} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result = %+v, want the re-probe itself reported as ok", result) + } + if result.CapabilitiesRefreshed { + t.Fatal("capabilities_refreshed = true despite a failed refresh") + } +} + +// A node that is transcoding refuses, because a probe's smoke encode competing +// with live sessions would be published as failed hardware. The operator has to +// read the node's own explanation, not a bare status code, or the only sensible +// next step — drain the node and retry — is invisible. +func TestHandleReprobeNodeSurfacesBusyNodeRefusal(t *testing.T) { + url, _ := fakeNodeServer(t, http.StatusConflict, + "node is running 2 transcode job(s); a re-probe smoke-encodes on the GPU. Retry when the node is idle.\n") + repo := &stubNodeRepository{node: &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: url}} + refresher := &stubCapabilityRefresher{} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(refresher) + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + var result ReprobeNodeResult + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "error" { + t.Fatalf("result = %+v, want the refusal reported as an error", result) + } + if !strings.Contains(result.Error, "idle") { + t.Fatalf("error = %q, want the node's own explanation", result.Error) + } + if got := refresher.calls.Load(); got != 0 { + t.Fatalf("refresh calls = %d, want none after a refused re-probe", got) + } +} + +// The action outlives the API listener's 120s WriteTimeout by design: the node's +// advertised probe budget reaches five minutes and the capability refetch adds +// two more. Without a deadline extension the response is written after the +// connection's deadline has passed, so the operator sees a torn connection and a +// "Re-probe failed" toast for an action that succeeded — and re-runs the whole +// cold FFmpeg matrix. This drives a real listener with a deadline far shorter +// than the node takes, which is the only way to observe the write actually +// landing. +func TestHandleReprobeNodeOutlivesTheListenerWriteTimeout(t *testing.T) { + const nodeDelay = 400 * time.Millisecond + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(nodeDelay) + _, _ = w.Write([]byte(`{"resolved":"qsv","capability_hash":"sha256:new"}`)) + })) + defer node.Close() + + // The node advertises a budget, so the handler knows how long to hold the + // connection open — exactly as a node that has been inventoried once does. + report, err := json.Marshal(playback.HWAccelInfo{ProbeRequestTimeoutMillis: 30_000}) + if err != nil { + t.Fatal(err) + } + repo := &stubNodeRepository{node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, + Capabilities: report, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + handler.SetCapabilityRefresher(&stubCapabilityRefresher{}) + + router := chi.NewRouter() + router.Post("/admin/nodes/{id}/reprobe", handler.HandleReprobeNode) + api := httptest.NewUnstartedServer(router) + api.Config.WriteTimeout = nodeDelay / 4 + api.Start() + defer api.Close() + + resp, err := api.Client().Post(api.URL+"/admin/nodes/1/reprobe", "", nil) + if err != nil { + t.Fatalf("re-probe response lost to the listener write deadline: %v", err) + } + defer func() { _ = resp.Body.Close() }() + var result ReprobeNodeResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Status != "ok" || result.CapabilityHash != "sha256:new" { + t.Fatalf("result = %+v, want the node's answer delivered intact", result) + } +} + +func TestHandleReprobeNodeUnknownNode(t *testing.T) { + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 for an unknown node", recorder.Code) + } +} + +// The node publishes the budget its own probe matrix needs, so a node with +// slower hardware is not abandoned on a cluster-wide guess. A node that has +// never been inventoried falls back to the generous constant. +func TestNodeReprobeTimeoutPrefersTheNodeAdvertisedBudget(t *testing.T) { + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + // Above the fallback, which is a floor: a node that says it needs less than + // the constant this handler is willing to spend does not shorten it. + advertised := playback.HWAccelInfo{ProbeRequestTimeoutMillis: 200_000} + payload, err := json.Marshal(advertised) + if err != nil { + t.Fatal(err) + } + if nodeReprobeFallbackTimeout >= 200*time.Second { + t.Fatalf("fixture is inert: the fallback %s must stay under the advertised 200s", nodeReprobeFallbackTimeout) + } + if got := handler.nodeReprobeTimeout(&nodepool.Node{Capabilities: payload}); got != 200*time.Second { + t.Fatalf("timeout = %s, want the node-advertised 200s", got) + } + if got := handler.nodeReprobeTimeout(&nodepool.Node{}); got != nodeReprobeFallbackTimeout { + t.Fatalf("timeout = %s, want the fallback %s for a node with no report", got, nodeReprobeFallbackTimeout) + } + if got := handler.nodeReprobeTimeout(&nodepool.Node{Capabilities: json.RawMessage(`not json`)}); got != nodeReprobeFallbackTimeout { + t.Fatalf("timeout = %s, want the fallback for an unreadable report", got) + } +} + +// A re-probe discards every cache on the node, so it runs the full matrix for +// whatever device set the node is configured for *now*. A report stored before +// an operator widened hw_device_override advertises the old, smaller budget, and +// honoring it would cancel the very request that would have replaced it. +func TestNodeReprobeTimeoutRepricesAWidenedDeviceOverride(t *testing.T) { + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + handler.SetClusterPlaybackPolicy(func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + }) + stored, err := json.Marshal(playback.HWAccelInfo{ + ProbeRequestTimeoutMillis: playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128").Milliseconds(), + }) + if err != nil { + t.Fatal(err) + } + + want := playback.CapabilityRequestTimeout(backend, devices) + got := handler.nodeReprobeTimeout(&nodepool.Node{ + Capabilities: stored, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }) + if got != want { + t.Fatalf("re-probe timeout = %s, want the four-device %s", got, want) + } + if want <= nodeReprobeFallbackTimeout { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the fallback %s", want, nodeReprobeFallbackTimeout) + } +} + +// boundedCapabilityRefresher is a refresher that also reports how long its +// refresh may take, as the real health checker does. +type boundedCapabilityRefresher struct { + stubCapabilityRefresher + bound time.Duration +} + +func (b *boundedCapabilityRefresher) CapabilityRefreshBound(*nodepool.Node) time.Duration { + return b.bound +} + +// The connection has to stay open across both long calls: the node's re-probe +// and the capability refresh that stores its result. The refresh's bound is +// derived from the node's own advertised probe budget, so on a node with many +// devices it runs well past the exported five-minute floor — and reserving the +// floor would let the write deadline fire after the re-probe succeeded but +// before its response was written, telling an operator an action failed that +// has already changed the node. +func TestReprobeWriteDeadlineReservesTheNodesOwnRefreshBound(t *testing.T) { + node := &nodepool.Node{ID: 1, Name: "gpu-1", URL: "http://gpu-1", Type: nodepool.NodeTypeTranscode} + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + + // No refresher: the floor is all this handler can promise. + if got := handler.capabilityRefreshBound(node); got != nodepool.CapabilityRefreshTimeout { + t.Fatalf("bound without a refresher = %s, want the exported floor %s", got, nodepool.CapabilityRefreshTimeout) + } + + // A refresher that will hold the fetch open past the floor. + bound := nodepool.CapabilityRefreshTimeout + 4*time.Minute + handler.SetCapabilityRefresher(&boundedCapabilityRefresher{bound: bound}) + if got := handler.capabilityRefreshBound(node); got != bound { + t.Fatalf("bound = %s, want the refresher's own %s", got, bound) + } + + // And it is what the deadline reserves, on top of the probe budget. + recorder := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + probeBudget := 3 * time.Minute + before := time.Now() + handler.extendReprobeWriteDeadline(recorder, reprobeRequest(t), node, probeBudget) + want := probeBudget + bound + nodeReprobeWriteSlack + if reserved := recorder.deadline.Sub(before); reserved < want { + t.Fatalf("reserved %s, want at least the probe budget plus the refresh bound (%s)", reserved, want) + } +} + +// deadlineRecorder captures the write deadline the handler sets. +type deadlineRecorder struct { + *httptest.ResponseRecorder + deadline time.Time +} + +func (d *deadlineRecorder) SetWriteDeadline(at time.Time) error { + d.deadline = at + return nil +} + +// The route an operator triggers has to reach a node stored with a trailing +// slash, which the pools and the repository already treat as the same worker. +func TestHandleReprobeNodeReachesATrailingSlashBaseURL(t *testing.T) { + var path string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + writeJSON(w, http.StatusOK, map[string]any{"resolved": "qsv", "capability_hash": "sha256:x"}) + })) + defer node.Close() + + repo := &stubNodeRepository{node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL + "/", Enabled: true, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleReprobeNode(recorder, reprobeRequest(t)) + + if path != "/admin/reprobe-capabilities" { + t.Fatalf("node was asked for %q, want /admin/reprobe-capabilities", path) + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } +} diff --git a/internal/api/handlers/nodes_test.go b/internal/api/handlers/nodes_test.go new file mode 100644 index 000000000..928f1e09d --- /dev/null +++ b/internal/api/handlers/nodes_test.go @@ -0,0 +1,795 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/go-chi/chi/v5" +) + +type stubNodeRepository struct { + nodes []*nodepool.Node + // updateResult is what Update returns once validation passes; nil keeps the + // default "unknown node" answer. + updateResult *nodepool.Node + updated *nodepool.UpdateNodeInput + // node is what GetByID returns; nil keeps the default "unknown node" answer. + node *nodepool.Node +} + +func (s *stubNodeRepository) List(context.Context) ([]*nodepool.Node, error) { return s.nodes, nil } + +func (s *stubNodeRepository) GetByID(context.Context, int) (*nodepool.Node, error) { + if s.node == nil { + return nil, nodepool.ErrNodeNotFound + } + return s.node, nil +} + +func (s *stubNodeRepository) Create(context.Context, nodepool.CreateNodeInput) (*nodepool.Node, error) { + return nil, nodepool.ErrNodeNotFound +} + +// Update mirrors Repository.Update's order of operations — validate, then +// write — so handler tests see the same errors production returns. +func (s *stubNodeRepository) Update(_ context.Context, _ int, input nodepool.UpdateNodeInput) (*nodepool.Node, error) { + if err := input.Validate(); err != nil { + return nil, err + } + s.updated = &input + if s.updateResult == nil { + return nil, nodepool.ErrNodeNotFound + } + return s.updateResult, nil +} + +func (s *stubNodeRepository) Delete(context.Context, int) error { return nil } + +func (s *stubNodeRepository) UpdateHealth(context.Context, int, string, bool, int, int, []byte) error { + return nil +} + +// The node list is the admin's inventory view, so it must carry the stored +// capability report, its age, and the derived GPU identities beside the +// existing node fields. +func TestHandleListNodesIncludesCapabilities(t *testing.T) { + hash := "sha256:abc" + refreshedAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + { + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", Enabled: true, Healthy: true, + Capabilities: json.RawMessage(`{"boot_id":"boot-1","resolved":"nvenc","render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`), + CapabilitiesHash: &hash, + CapabilitiesRefreshedAt: &refreshedAt, + // Production derives this in the node store's row scanner (covered + // by TestScanNodeDerivesPhysicalGPUKeys); the stub stands in for it + // so this test can assert the handler passes the field through. + PhysicalGPUKeys: []string{"GPU-aaa"}, + }, + {ID: 2, Name: "old-node", Type: nodepool.NodeTypeProxy, URL: "http://old", Enabled: true}, + }} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var items []map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &items); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(items) != 2 { + t.Fatalf("returned %d nodes, want 2", len(items)) + } + // Existing fields must still be present: this response is embedded, not + // rebuilt, so a client reading it today keeps working. + if items[0]["name"] != "gpu-1" || items[0]["healthy"] != true { + t.Fatalf("node fields were lost: %v", items[0]) + } + if items[0]["capabilities_hash"] != "sha256:abc" { + t.Fatalf("capabilities_hash = %v", items[0]["capabilities_hash"]) + } + if items[0]["capabilities_refreshed_at"] == nil { + t.Fatalf("capabilities_refreshed_at missing: %v", items[0]) + } + if items[0]["capabilities"] == nil { + t.Fatalf("capabilities missing: %v", items[0]) + } + keys, ok := items[0]["physical_gpu_keys"].([]any) + if !ok || len(keys) != 1 || keys[0] != "GPU-aaa" { + t.Fatalf("physical_gpu_keys = %v", items[0]["physical_gpu_keys"]) + } + // A node that never reported capabilities carries none of the new fields + // rather than empty ones a client would have to special-case. + for _, field := range []string{"capabilities", "capabilities_hash", "capabilities_refreshed_at", "physical_gpu_keys"} { + if _, present := items[1][field]; present { + t.Fatalf("node without capabilities carries %q: %v", field, items[1]) + } + } +} + +func updateNodeRequest(t *testing.T, body string) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/admin/nodes/1", strings.NewReader(body)) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("id", "1") + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) +} + +// A per-node override may only name a backend the cluster-wide setting could +// also name. Rejecting it here is what turns a CHECK-constraint violation into +// an answer the admin UI can show. +func TestHandleUpdateNodeRejectsUnknownHWAccelOverride(t *testing.T) { + repo := &stubNodeRepository{updateResult: &nodepool.Node{ID: 1, Name: "gpu-1"}} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"videotoolbox"}`) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body = %s", recorder.Code, recorder.Body.String()) + } + if repo.updated != nil { + t.Fatalf("rejected input still reached the store: %+v", repo.updated) + } + if !strings.Contains(recorder.Body.String(), "hw_accel_override") { + t.Fatalf("error body does not name the field: %s", recorder.Body.String()) + } +} + +// Setting an override and clearing it again are both ordinary updates; the +// clear has to survive JSON decoding as a clear rather than as "unchanged". +func TestHandleUpdateNodeAcceptsAndClearsHWOverrides(t *testing.T) { + accel, device := "vaapi", "/dev/dri/renderD129" + tests := []struct { + name string + body string + wantAccel *string + wantDevice *string + }{ + { + name: "sets both", + body: `{"hw_accel_override":"vaapi","hw_device_override":"/dev/dri/renderD129"}`, + wantAccel: &accel, + wantDevice: &device, + }, + { + name: "explicit null clears both", + body: `{"hw_accel_override":null,"hw_device_override":null}`, + wantAccel: new(string), + wantDevice: new(string), + }, + { + name: "omitted leaves both alone", + body: `{"name":"gpu-1"}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stored := "qsv" + repo := &stubNodeRepository{updateResult: &nodepool.Node{ID: 1, Name: "gpu-1", HWAccelOverride: &stored}} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + awaitNodeUpdate(t, handler, recorder, test.body) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", recorder.Code, recorder.Body.String()) + } + if repo.updated == nil { + t.Fatal("update never reached the store") + } + if !equalStringPointer(repo.updated.HWAccelOverride, test.wantAccel) { + t.Fatalf("HWAccelOverride = %v, want %v", repo.updated.HWAccelOverride, test.wantAccel) + } + if !equalStringPointer(repo.updated.HWDeviceOverride, test.wantDevice) { + t.Fatalf("HWDeviceOverride = %v, want %v", repo.updated.HWDeviceOverride, test.wantDevice) + } + // The response is the stored row, so the admin UI sees the effective + // policy without a second read. + var response map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response["hw_accel_override"] != "qsv" { + t.Fatalf("response hw_accel_override = %v, want the stored value", response["hw_accel_override"]) + } + }) + } +} + +func equalStringPointer(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +// A node re-reads its own row on a 60s config poll, but this server starts +// dispatching the new backend the moment its pool reloads. An operator moving a +// node from QSV on a render node to NVENC on a CUDA index would otherwise get +// up to a minute of start requests pairing the new backend with the old device, +// so the node is nudged to reload before the updated policy is published. +// +// The nudge targets /admin/reload-config, never /admin/force-reload: the latter +// tears down every live playback session on a transcode node. +func TestHandleUpdateNodeReloadsTheNodeAfterAnOverrideChange(t *testing.T) { + reloaded := make(chan string, 4) + var destructive atomic.Bool + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/admin/reload-config": + reloaded <- r.Header.Get("Authorization") + case "/admin/force-reload": + destructive.Store(true) + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(node.Close) + + qsv := "qsv" + before := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &qsv} + nvenc := "nvenc" + after := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &nvenc} + repo := &stubNodeRepository{updateResult: after, node: before} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"nvenc"}`) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + select { + case authorization := <-reloaded: + if authorization != "Bearer secret" { + t.Fatalf("node saw authorization %q, want the bearer secret", authorization) + } + default: + t.Fatal("the node was not asked to reload after its overrides changed") + } + if destructive.Load() { + t.Fatal("a policy edit hit the destructive force-reload route") + } + // The route answers 204, so a client that accepted only 200 would warn on + // every successful reload — a standing false alarm on the ordinary path. + if logged := recorder.Body.String(); strings.Contains(logged, "refused") { + t.Fatalf("body mentions a refusal: %s", logged) + } +} + +// The node reload route answers 204. Treating anything outside 2xx as a refusal +// is what keeps a successful reload from logging a failure an operator would +// then go looking for. +func TestReloadNodeConfigAcceptsNoContent(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusNoContent, http.StatusAccepted} { + t.Run(http.StatusText(status), func(t *testing.T) { + called := make(chan struct{}, 1) + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called <- struct{}{} + w.WriteHeader(status) + })) + t.Cleanup(node.Close) + + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + handler.reloadNodeConfig(context.Background(), &nodepool.Node{ID: 1, Name: "gpu-1", URL: node.URL}) + + select { + case <-called: + default: + t.Fatal("the node was never called") + } + }) + } +} + +// This server's cached view of what a node can do — the v3 planning inventory — +// is keyed by node URL and holds the tone-map executors and transformations the +// *previous* backend advertised. Changing the policy without dropping it plans +// the next minute's sessions against filters the worker has already moved off, +// and the worker then rejects the start. +func TestHandleUpdateNodeInvalidatesCapabilityCacheAfterAnOverrideChange(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(node.Close) + + qsv, nvenc := "qsv", "nvenc" + before := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &qsv} + after := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &nvenc} + repo := &stubNodeRepository{updateResult: after, node: before} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + invalidated := make(chan string, 4) + handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) + + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"hw_accel_override":"nvenc"}`) + + select { + case url := <-invalidated: + if url != node.URL { + t.Fatalf("invalidated %q, want the node's URL %q", url, node.URL) + } + default: + t.Fatal("the capability cache was not dropped after the policy changed") + } +} + +// A node that does not confirm the reload is out of step: its backend now comes +// from this server's pool while its device still comes from its own +// configuration. The policy is published regardless — withholding it would +// strand an override the operator can see stored, since nothing else re-reads +// the column — so what this pins down is that the update still succeeds and the +// caller learns the node did not confirm. +func TestReloadNodeConfigReportsAnUnconfirmedNode(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(node.Close) + + handler := NewNodeHandler(&stubNodeRepository{}, nil, nil, nil, nil, nil, "secret") + if handler.reloadNodeConfig(context.Background(), &nodepool.Node{ID: 1, Name: "gpu-1", URL: node.URL}) { + t.Fatal("a 500 from the node reported the reload as confirmed") + } + if handler.reloadNodeConfig(context.Background(), &nodepool.Node{ID: 1, Name: "gpu-1", URL: "http://127.0.0.1:1"}) { + t.Fatal("an unreachable node reported the reload as confirmed") + } +} + +// The update itself still succeeds and the new policy still reaches the pool: a +// stored override that never reaches dispatch is a silent permanent +// misconfiguration, where the mismatch is loud, bounded by the node's poll, and +// self-healing. +func TestHandleUpdateNodePublishesPolicyEvenWhenTheNodeDoesNotConfirm(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(node.Close) + + qsv, nvenc := "qsv", "nvenc" + before := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &qsv} + after := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, HWAccelOverride: &nvenc} + repo := &stubNodeRepository{updateResult: after, node: before} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + invalidated := make(chan string, 4) + handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) + + recorder := httptest.NewRecorder() + awaitNodeUpdate(t, handler, recorder, `{"hw_accel_override":"nvenc"}`) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want the update to succeed anyway", recorder.Code) + } + select { + case <-invalidated: + default: + t.Fatal("the capability cache was not dropped when the node did not confirm") + } +} + +// An edit that moves neither override leaves the cache alone: re-probing every +// node on every rename would put ffmpeg execs behind an unrelated form save. +func TestHandleUpdateNodeKeepsCapabilityCacheWithoutAnOverrideChange(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(node.Close) + + stored := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL} + repo := &stubNodeRepository{updateResult: stored, node: stored} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + invalidated := make(chan string, 4) + handler.SetCapabilityInvalidator(func(url string) { invalidated <- url }) + + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"name":"gpu-one"}`) + + select { + case url := <-invalidated: + t.Fatalf("a rename dropped the capability cache for %q", url) + default: + } +} + +// The admin form posts both override fields on every transcode-node save, so +// their presence says nothing about them moving. Nudging on presence alone made +// an unrelated edit — a rename, a capacity change, or a plain resubmit — ask the +// node to re-read its config for nothing. +func TestHandleUpdateNodeDoesNotReloadWhenOverridesAreUnchanged(t *testing.T) { + reloaded := make(chan struct{}, 4) + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/reload") || strings.HasPrefix(r.URL.Path, "/admin/force") { + reloaded <- struct{}{} + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(node.Close) + + qsv := "qsv" + device := "/dev/dri/renderD128" + unchanged := func() *nodepool.Node { + accel, path := qsv, device + return &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL, + HWAccelOverride: &accel, HWDeviceOverride: &path, + } + } + repo := &stubNodeRepository{updateResult: unchanged(), node: unchanged()} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + body := `{"name":"gpu-1","hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}` + awaitNodeUpdate(t, handler, httptest.NewRecorder(), body) + + select { + case <-reloaded: + t.Fatal("an edit that moved neither override still asked the node to reload") + default: + } +} + +// An edit that touches no acceleration field must not cost a round trip to the +// node: renaming a node has nothing to do with what it probes. +func TestHandleUpdateNodeDoesNotReloadWithoutAnOverrideChange(t *testing.T) { + reloaded := make(chan struct{}, 4) + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/") { + reloaded <- struct{}{} + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(node.Close) + + stored := &nodepool.Node{ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: node.URL} + repo := &stubNodeRepository{updateResult: stored, node: stored} + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"name":"gpu-one"}`) + + select { + case <-reloaded: + t.Fatal("a rename asked the node to reload its configuration") + default: + } +} + +// recordingEventBus captures publications so a test can assert the pool change +// actually reached the other replicas. +type recordingEventBus struct { + mu sync.Mutex + events []cache.Event +} + +func (b *recordingEventBus) Publish(_ context.Context, _ string, event cache.Event) error { + b.mu.Lock() + defer b.mu.Unlock() + b.events = append(b.events, event) + return nil +} + +func (b *recordingEventBus) Subscribe(context.Context, string, cache.EventHandler) error { return nil } +func (b *recordingEventBus) Close() error { return nil } + +func (b *recordingEventBus) types() []string { + b.mu.Lock() + defer b.mu.Unlock() + types := make([]string, 0, len(b.events)) + for _, event := range b.events { + types = append(types, event.Type) + } + return types +} + +// ctxAwareLister answers only for a live context, the way a database read does. +type ctxAwareLister struct{ nodes []*nodepool.Node } + +func (l *ctxAwareLister) ListEnabled(ctx context.Context, _ string) ([]*nodepool.Node, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return l.nodes, nil +} + +// The pool reload runs after the response is written, so the request context is +// no longer the right lifetime: an admin whose browser gives up on a slow save +// cancels it, and the reload would then fail its reads and never publish. The +// row is already committed at that point, so this instance and every replica +// would keep dispatching under the old acceleration policy indefinitely — +// nothing else re-reads the column. +func TestReloadPoolsSurvivesRequestCancellation(t *testing.T) { + bus := &recordingEventBus{} + lister := &ctxAwareLister{nodes: []*nodepool.Node{{ID: 1, URL: "http://node", Enabled: true}}} + handler := NewNodeHandler(&stubNodeRepository{}, nodepool.NewProxyPool(), nodepool.NewTranscodePool(), lister, bus, nil, "secret") + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + handler.reloadPools(canceled) + + if got := bus.types(); len(got) != 1 || got[0] != string(cache.EventNodePoolChanged) { + t.Fatalf("published %v, want a single %q after a canceled request", got, cache.EventNodePoolChanged) + } + if got := handler.transcodePool.Nodes(); len(got) != 1 { + t.Fatalf("transcode pool holds %d nodes, want the reload to have landed", len(got)) + } +} + +// Repointing a row at a different worker changes which machine those overrides +// apply to, even when the values are byte-identical. reloadPools publishes the +// new URL at once, so between that and the replacement's own 60s config poll +// this server dispatches the row's overridden backend to a worker still running +// on whatever it inherited — the same backend/device mismatch an override edit +// causes, reached by a different edit. +func TestHandleUpdateNodeReloadsTheReplacementWhenAURLMoves(t *testing.T) { + reloaded := make(chan string, 4) + replacement := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/reload") { + reloaded <- r.URL.Path + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(replacement.Close) + + qsv, device := "qsv", "/dev/dri/renderD128" + repo := &stubNodeRepository{ + node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://retired-worker", + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + updateResult: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: replacement.URL, + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + } + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + body := `{"name":"gpu-1","url":"` + replacement.URL + `","hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}` + awaitNodeUpdate(t, handler, httptest.NewRecorder(), body) + + select { + case path := <-reloaded: + if path != "/admin/reload-config" { + t.Fatalf("nudged %q, want the non-destructive reload", path) + } + default: + t.Fatal("a repointed row left the replacement worker on its inherited policy") + } +} + +// A trailing slash is not a repoint: the pools normalize URLs and the database +// column does not, so treating it as one would nudge on every unrelated save. +func TestNodePolicyTargetChangeIgnoresATrailingSlash(t *testing.T) { + qsv := "qsv" + before := &nodepool.Node{ID: 1, URL: "http://node/", HWAccelOverride: &qsv} + sameAccel := qsv + after := &nodepool.Node{ID: 1, URL: "http://node", HWAccelOverride: &sameAccel} + + if nodePolicyTargetChanged(before, after) { + t.Fatal("a trailing-slash difference read as repointing the row") + } +} + +// A partial PUT that carries only a new url still repoints the row's existing +// overrides at a different worker. Loading the previous row only when an +// override field is present left nodePolicyTargetChanged with nothing to compare +// against, so the URL clause it grew for exactly this case never fired. +func TestHandleUpdateNodeReloadsOnAURLOnlyRepoint(t *testing.T) { + reloaded := make(chan string, 4) + replacement := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/admin/reload") { + reloaded <- r.URL.Path + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(replacement.Close) + + qsv, device := "qsv", "/dev/dri/renderD128" + repo := &stubNodeRepository{ + node: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://retired-worker", + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + updateResult: &nodepool.Node{ + ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: replacement.URL, + HWAccelOverride: &qsv, HWDeviceOverride: &device, + }, + } + handler := NewNodeHandler(repo, nil, nil, nil, nil, nil, "secret") + + // Only the url: no acceleration field in the body at all. + awaitNodeUpdate(t, handler, httptest.NewRecorder(), `{"url":"`+replacement.URL+`"}`) + + select { + case path := <-reloaded: + if path != "/admin/reload-config" { + t.Fatalf("nudged %q, want the non-destructive reload", path) + } + default: + t.Fatal("a url-only repoint left the replacement worker on its inherited policy") + } +} + +// awaitNodeUpdate runs one update and waits for the post-commit work it starts. +// +// HandleUpdateNode answers as soon as the row is committed and does the node +// nudge, cache drop and pool reload on their own goroutine — the nudge alone is +// bounded at ten seconds against a worker that may be unreachable, and an +// operator should not wait through that. Tests therefore wait on the handler's +// own completion signal rather than on the call returning. +func awaitNodeUpdate(t *testing.T, handler *NodeHandler, recorder *httptest.ResponseRecorder, body string) { + t.Helper() + done := make(chan struct{}) + handler.afterNodeUpdate = func() { close(done) } + handler.HandleUpdateNode(recorder, updateNodeRequest(t, body)) + if recorder.Code < 200 || recorder.Code > 299 { + // A rejected update commits nothing and starts no post-commit work, so + // there is no signal coming; the test is asserting on the refusal. + return + } + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("timed out waiting for the post-commit node update work") + } +} + +// The advertised hash lives only on the pools — it is an observation from the +// health sweep, not stored state — while this endpoint serves database rows. If +// it is not carried across, the field is always absent and the admin page has +// no way to tell a report the node has already contradicted from a current one. +func TestHandleListNodesCarriesTheAdvertisedHashFromThePools(t *testing.T) { + stored := "sha256:stored" + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + {ID: 1, Name: "gpu-1", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", CapabilitiesHash: &stored}, + {ID: 2, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1"}, + {ID: 3, Name: "quiet", Type: nodepool.NodeTypeTranscode, URL: "http://quiet"}, + }} + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{ + {ID: 1, URL: "http://gpu-1", Enabled: true}, + {ID: 3, URL: "http://quiet", Enabled: true}, + }) + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ID: 2, URL: "http://proxy-1", Enabled: true}}) + // The sweep learns each node's advertised hash on its health check. + transcodes.ApplyHealth(1, "http://gpu-1", true, 0, 0, "sha256:newer", nil, time.Now()) + proxies.ApplyHealth(2, "http://proxy-1", true, 0, 0, "sha256:proxy", nil, time.Now()) + + handler := NewNodeHandler(repo, proxies, transcodes, nil, nil, nil, "secret") + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } + var listed []struct { + ID int `json:"id"` + Hash string `json:"capabilities_hash"` + Advertised string `json:"advertised_capabilities_hash"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &listed); err != nil { + t.Fatalf("decode: %v (%s)", err, recorder.Body.String()) + } + byID := map[int]string{} + for _, node := range listed { + byID[node.ID] = node.Advertised + } + if byID[1] != "sha256:newer" { + t.Fatalf("transcode advertised hash = %q, want the sweep's observation", byID[1]) + } + if byID[2] != "sha256:proxy" { + t.Fatalf("proxy advertised hash = %q, want the sweep's observation", byID[2]) + } + // A node that has not been checked since this process started carries none, + // and the field is omitted rather than reported as a mismatch. + if byID[3] != "" { + t.Fatalf("unchecked node advertised hash = %q, want it absent", byID[3]) + } +} + +// A manual check is a health check like the sweep's: whatever it learns has to +// reach the pool the planner and the Nodes page read, not just the row. +func TestHandleCheckNodePublishesResultToThePool(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "active_jobs": 3, + "egress_kbps": 4200, + "capabilities_hash": "sha256:after", + "system": map[string]any{"scratch_free_gb": 0.5}, + }) + })) + defer node.Close() + + stale := "sha256:before" + pooled := &nodepool.Node{ + ID: 7, Name: "gpu-7", Type: nodepool.NodeTypeTranscode, URL: node.URL, Enabled: true, + Healthy: true, ActiveJobs: 99, CapabilitiesHash: &stale, AdvertisedCapabilitiesHash: &stale, + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{pooled}) + + repo := &stubNodeRepository{node: pooled} + handler := NewNodeHandler(repo, nil, pool, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/admin/nodes/7/check", nil) + routeContext := chi.NewRouteContext() + routeContext.URLParams.Add("id", "7") + handler.HandleCheckNode(recorder, request.WithContext(context.WithValue(request.Context(), chi.RouteCtxKey, routeContext))) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + + updated := pool.Nodes() + if len(updated) != 1 { + t.Fatalf("pool holds %d nodes, want 1", len(updated)) + } + if got := updated[0].AdvertisedCapabilitiesHash; got == nil || *got != "sha256:after" { + t.Errorf("pool advertised hash = %v, want the hash this check just read", got) + } + if got := updated[0].ActiveJobs; got != 3 { + t.Errorf("pool active jobs = %d, want 3", got) + } + // The stats decide whether the planner keeps admitting work; a check that + // found the scratch volume nearly full has to reach the planner's copy. + if len(updated[0].LastStats) == 0 { + t.Error("pool node kept no stats from the manual check") + } + if updated[0].LastHealthCheck == nil { + t.Error("pool node kept no check timestamp, so the row and the pool disagree on freshness") + } +} + +// Three states, not two: a node that answers health checks with no hash is not +// the same as a node nobody has checked yet. The first is no longer standing +// behind the inventory stored for it — a build downgraded past capability +// reports — while the second is every node until the first sweep after a +// restart, and says nothing at all. +func TestHandleListNodesDistinguishesUncheckedFromUnreportedHashes(t *testing.T) { + stored := "sha256:stored" + none := "" + repo := &stubNodeRepository{nodes: []*nodepool.Node{ + {ID: 1, Name: "checked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", CapabilitiesHash: &stored}, + {ID: 2, Name: "unchecked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-2", CapabilitiesHash: &stored}, + }} + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{ + {ID: 1, Name: "checked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-1", AdvertisedCapabilitiesHash: &none}, + {ID: 2, Name: "unchecked", Type: nodepool.NodeTypeTranscode, URL: "http://gpu-2"}, + }) + handler := NewNodeHandler(repo, nil, pool, nil, nil, nil, "secret") + + recorder := httptest.NewRecorder() + handler.HandleListNodes(recorder, httptest.NewRequest(http.MethodGet, "/admin/nodes", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var items []map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &items); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(items) != 2 { + t.Fatalf("returned %d nodes, want 2", len(items)) + } + // Present and empty: the node was asked and named nothing. + advertised, ok := items[0]["advertised_capabilities_hash"] + if !ok || advertised != "" { + t.Errorf("checked node advertised %v (present=%v), want an empty string", advertised, ok) + } + // Absent: nothing has asked, so the field must not claim the node said so. + if _, ok := items[1]["advertised_capabilities_hash"]; ok { + t.Errorf("unchecked node carried an advertised hash: %v", items[1]) + } +} diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 5caf54992..688264800 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -246,14 +246,30 @@ type PlaybackHandler struct { tm *playback.TranscodeManager // PlanStoreV3 owns the short-lived protocol-v3 control-plane state. Router // wiring replaces the in-memory default with PostgreSQL in integrated mode. - PlanStoreV3 playback.PlanStoreV3 - v3RegistryMu sync.Mutex - v3Registry *playback.TransformationRegistryV3 - v3RegistryProbe func(context.Context, string, tonemap.Capabilities) (*playback.TransformationRegistryV3, error) - v3ToneMapProbe func(context.Context, string, string, string) (tonemap.Capabilities, error) - v3NodeCapabilitiesMu sync.Mutex - v3NodeCapabilities map[string]v3NodeCapabilityCache - v3NodeCapabilityRefresh sync.Map + PlanStoreV3 playback.PlanStoreV3 + v3RegistryMu sync.Mutex + v3Registry *playback.TransformationRegistryV3 + v3RegistryProbe func(context.Context, string, tonemap.Capabilities) (*playback.TransformationRegistryV3, error) + v3ToneMapProbe func(context.Context, string, string, string) (tonemap.Capabilities, error) + v3NodeCapabilitiesMu sync.Mutex + v3NodeCapabilities map[string]v3NodeCapabilityCache + // v3NodeProbeBudgets holds what each node last said a capability read of it + // costs, guarded by v3NodeCapabilitiesMu. It is kept apart from the + // inventory above because the two are invalidated for different reasons: an + // acceleration change makes the inventory wrong and the next read slow, + // while how long that node takes to answer is unchanged. See + // remoteToneMapProbeTimeoutV3. + v3NodeProbeBudgets map[string]time.Duration + // v3NodeCapabilityInvalidations counts invalidations per node URL, guarded + // by v3NodeCapabilitiesMu. A probe that started before the count moved + // describes hardware the health sweep has already reported as changed, so + // its result must not be installed. See RefreshNodeCapabilitiesV3. + v3NodeCapabilityInvalidations map[string]uint64 + // v3NodeCapabilityRefresh holds the nodes with a background refresh in + // flight, one at a time each. Guarded by v3NodeCapabilitiesMu, the same + // lock as the invalidation counter, so a refresh cannot release its slot in + // between an invalidation and that invalidation's claim on it. + v3NodeCapabilityRefresh map[string]struct{} v3EventOnce sync.Once v3EventQueue chan playback.RouteEventRecordV3 v3StartEffectsOnce sync.Once @@ -1842,7 +1858,7 @@ func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyN if proxyNode == nil || token == "" { return appendStreamToken(localURL, token) } - return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" + return nodepool.NodeEndpoint(proxyNode.ClientURL(), "/stream/transcode/"+token+"/master.m3u8") } // proxyToTranscodeNode forwards a request to the remote transcode node. diff --git a/internal/api/handlers/playback_transport.go b/internal/api/handlers/playback_transport.go index 18c2b796b..3e41d7c34 100644 --- a/internal/api/handlers/playback_transport.go +++ b/internal/api/handlers/playback_transport.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/logredact" + "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodenode" @@ -36,7 +37,7 @@ func (h *PlaybackHandler) startRemotePlaybackTransport(ctx context.Context, node } requestCtx, cancel := context.WithTimeout(ctx, h.remotePlaybackTransportTimeout(nodeURL, request)) defer cancel() - httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body)) + httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(nodeURL, "/transcode/start"), bytes.NewReader(body)) if err != nil { return transcodenode.TranscodeStartResponse{}, 0, logredact.SanitizeURLError(err) } diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 4eff31595..e16b4e378 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -87,7 +87,6 @@ type v3NodeCapabilityCache struct { toneMapCapabilities tonemap.Capabilities err error expiresAt time.Time - probeRequestTimeout time.Duration } type preparedTransportV3 struct { @@ -342,11 +341,11 @@ func (h *PlaybackHandler) transformationRegistryV3(ctx context.Context) *playbac func (h *PlaybackHandler) localToneMapCapabilitiesV3(ctx context.Context) (tonemap.Capabilities, error) { cfg := h.playbackConfig() ffmpegPath := playback.ResolveFFmpegPath(cfg.FFmpegPath) - resolved := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.HWAccel, cfg.FFmpegPath) + hwDevice := strings.TrimSpace(cfg.HWDevice) + resolved := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.HWAccel, cfg.FFmpegPath, hwDevice) if err := ctx.Err(); err != nil { return nil, err } - hwDevice := strings.TrimSpace(cfg.HWDevice) probe := tonemap.Probe if h.v3ToneMapProbe != nil { probe = h.v3ToneMapProbe @@ -363,17 +362,84 @@ func (h *PlaybackHandler) localToneMapCapabilitiesForTransportV3(ctx context.Con func (h *PlaybackHandler) localToneMapProbeTimeoutV3() time.Duration { cfg := h.playbackConfig() - return tonemap.ProbeEndpointTimeout(cfg.HWAccel, cfg.HWDevice) + // The whole read, not its tone-map half: localToneMapCapabilitiesV3 resolves + // the backend first, and on Linux that is a full hardware walk whose cost + // scales with the configured device set. + return playback.CapabilityEndpointTimeout(cfg.HWAccel, cfg.HWDevice) } +// remoteToneMapProbeTimeoutV3 returns how long to allow one capability read of +// a node, from the budget that node last advertised. +// +// The budget is kept apart from the inventory because it describes the node +// rather than its hardware, and the two are invalidated for different reasons. +// An acceleration change makes the inventory wrong and the matrix cold — which +// is exactly when the next read is slowest — while how long that node takes to +// answer has not changed. Storing them together meant an invalidation dropped +// the budget with the inventory and the refresh it triggered fell back to two +// minutes, short of the ~136 seconds a two-device node legitimately asks for, +// so protocol-v3 planning lost its inventory precisely after an invalidation. func (h *PlaybackHandler) remoteToneMapProbeTimeoutV3(nodeURL string) time.Duration { + nodeURL = nodepool.NormalizeNodeURL(nodeURL) h.v3NodeCapabilitiesMu.Lock() - entry := h.v3NodeCapabilities[nodeURL] + budget := h.v3NodeProbeBudgets[nodeURL] h.v3NodeCapabilitiesMu.Unlock() - if entry.probeRequestTimeout > 0 { - return entry.probeRequestTimeout + // Never less than what this node currently describes, whatever was learned + // from it before. + // + // A learned budget is kept across invalidations on purpose — an invalidation + // is the moment the next read is coldest and slowest, so dropping the budget + // with the inventory would fall back to a figure short of what the node + // needs. But a per-node policy edit invalidates through the same path, and + // widening hw_device_override is precisely a change that makes the learned + // number too small: the node reloads, walks four devices instead of one, and + // gets canceled at the one-device deadline. Nothing recovers from that on its + // own, because a budget is only ever learned from a read that completes. + if cold := h.coldNodeProbeTimeoutV3(nodeURL); cold > budget { + return cold + } + return budget +} + +// coldNodeProbeTimeoutV3 prices one capability read of a node this process has +// not read successfully yet, from that node rather than from a cluster-wide +// guess. Without a pooled record — a planner that cannot look nodes up, or a +// URL that is no longer in the pool — the cluster's own policy is the closest +// description available. +func (h *PlaybackHandler) coldNodeProbeTimeoutV3(nodeURL string) time.Duration { + cfg := h.playbackConfig() + var node *nodepool.Node + if lookup, ok := h.NodePlanner.(transcodeNodeLookupV3); ok { + if found, ok := lookup.TranscodeNodeByURL(nodeURL); ok { + node = found + } } - return remoteNodeProbeFallbackTimeout + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(cfg.HWAccel), + node.EffectiveHWDevice(cfg.HWDevice), + remoteNodeProbeFallbackTimeout, + ) +} + +// transcodeNodeLookupV3 resolves the pooled record behind a transcode node URL, +// which carries that node's stored capability report and its acceleration +// override. Optional, like the planner itself: without it this path falls back +// to the cluster-wide setting. *nodepool.Planner implements it. +type transcodeNodeLookupV3 interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) +} + +// rememberNodeProbeBudgetV3 records what a node says its capability read costs. +// Callers must hold v3NodeCapabilitiesMu. +func (h *PlaybackHandler) rememberNodeProbeBudgetLockedV3(nodeURL string, budget time.Duration) { + if budget <= 0 { + return + } + if h.v3NodeProbeBudgets == nil { + h.v3NodeProbeBudgets = make(map[string]time.Duration) + } + h.v3NodeProbeBudgets[nodeURL] = budget } func (h *PlaybackHandler) toneMapPlanningTimeoutV3(localFallbackAllowed bool) time.Duration { @@ -411,6 +477,10 @@ func (h *PlaybackHandler) lookupRemoteTransformationsV3(ctx context.Context, nod // lookupRemoteCapabilitiesV3 fetches and jointly caches a node's transformation // and tone-map inventory, optionally reusing short-lived fetch failures. func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeURL string, honorCachedFailure bool) (v3NodeCapabilityCache, error) { + // Canonical here and in RefreshNodeCapabilitiesV3, which are the two ways + // into these maps; everything below is reached from one of them and so is + // already keyed the same way. + nodeURL = nodepool.NormalizeNodeURL(nodeURL) now := time.Now() h.v3NodeCapabilitiesMu.Lock() entry, ok := h.v3NodeCapabilities[nodeURL] @@ -432,20 +502,53 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } - requestCtx, cancel := context.WithTimeout(ctx, h.remoteToneMapProbeTimeoutV3(nodeURL)) - defer cancel() - info, err := fetchRemoteTranscodeCapabilities(requestCtx, nodeURL, h.JWTSecret) - completedAt := time.Now() + // An invalidation landing mid-fetch means the report this probe is about to + // return describes the node as it was, not as it is — and this caller is + // about to select transformations and a tone-map executor from it. Re-probe + // rather than plan on it. + // + // Bounded at two attempts, and the second result is used even if it is + // overtaken too. Failing the request instead would be worse than a slightly + // stale inventory: most hash changes are not "the hardware went away" — a + // driver update, a new identity field, a raised probe budget all move it — + // so refusing would reject playback that the report in hand describes + // perfectly well, and on a single-transcode-node deployment there is nothing + // to fall back to. A node changing faster than two probes can read it is a + // different problem, and one this path cannot fix by failing. + var ( + info playback.HWAccelInfo + err error + completedAt time.Time + overtaken bool + ) + for attempt := range v3CapabilityFetchAttempts { + // Snapshot the invalidation count before probing: anything this fetch + // learns describes the node as it was when the request left. + invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) + requestCtx, cancel := context.WithTimeout(ctx, h.remoteToneMapProbeTimeoutV3(nodeURL)) + info, err = fetchRemoteTranscodeCapabilities(requestCtx, nodeURL, h.JWTSecret) + cancel() + completedAt = time.Now() + overtaken = h.nodeCapabilityInvalidationsV3(nodeURL) != invalidations + if !overtaken || attempt+1 == v3CapabilityFetchAttempts { + break + } + } + if err != nil { h.v3NodeCapabilitiesMu.Lock() if h.v3NodeCapabilities == nil { h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) } + if overtaken { + h.v3NodeCapabilitiesMu.Unlock() + return v3NodeCapabilityCache{}, err + } if current, currentOK := h.v3NodeCapabilities[nodeURL]; currentOK && current.err == nil && completedAt.Before(current.expiresAt) { h.v3NodeCapabilitiesMu.Unlock() return current, nil } - h.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{err: err, expiresAt: completedAt.Add(v3NodeCapabilityErrorTTL), probeRequestTimeout: entry.probeRequestTimeout} + h.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{err: err, expiresAt: completedAt.Add(v3NodeCapabilityErrorTTL)} h.v3NodeCapabilitiesMu.Unlock() return v3NodeCapabilityCache{}, err } @@ -453,9 +556,20 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR transformations: append([]playback.TransformationV3(nil), info.Transformations...), toneMapCapabilities: append(tonemap.Capabilities(nil), info.ToneMapCapabilities...), expiresAt: completedAt.Add(v3NodeCapabilityTTL), - probeRequestTimeout: playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout), } h.v3NodeCapabilitiesMu.Lock() + // Recorded even when the entry below is discarded as overtaken: what the + // node says its read costs is true regardless of whether this particular + // answer is still current. + h.rememberNodeProbeBudgetLockedV3(nodeURL, + playback.NormalizeProbeRequestTimeout(info.ProbeRequestTimeoutMillis, remoteNodeProbeFallbackTimeout)) + if overtaken { + // Still overtaken after the retry. Hand the result to this caller, which + // has nothing better, but leave the cache empty so the next lookup + // re-probes instead of serving it for a full TTL to everyone else. + h.v3NodeCapabilitiesMu.Unlock() + return entry, nil + } if h.v3NodeCapabilities == nil { h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) } @@ -464,19 +578,97 @@ func (h *PlaybackHandler) lookupRemoteCapabilitiesV3(ctx context.Context, nodeUR return entry, nil } -func (h *PlaybackHandler) refreshRemoteCapabilitiesV3(nodeURL string) { +// v3CapabilityFetchAttempts is how many times one lookup will re-probe a node +// whose capabilities changed while it was being read. +const v3CapabilityFetchAttempts = 2 + +func (h *PlaybackHandler) nodeCapabilityInvalidationsV3(nodeURL string) uint64 { + h.v3NodeCapabilitiesMu.Lock() + defer h.v3NodeCapabilitiesMu.Unlock() + return h.v3NodeCapabilityInvalidations[nodeURL] +} + +// RefreshNodeCapabilitiesV3 discards one node's cached capability inventory and +// re-probes it in the background. It exists for the node health sweep, which +// learns from a node's advertised capability hash that its hardware changed +// long before this cache's freshness window would expire — and a cache that +// outlives the hardware it describes plans transcodes onto a GPU that is gone. +func (h *PlaybackHandler) RefreshNodeCapabilitiesV3(nodeURL string) { if h == nil || nodeURL == "" { return } - if _, loaded := h.v3NodeCapabilityRefresh.LoadOrStore(nodeURL, struct{}{}); loaded { + // Every map here is keyed by the node's canonical address. This entry point + // is reached from the admin route with the URL exactly as the row stores it, + // which may carry a trailing slash the pools have already dropped — and then + // the entry this deletes is not the entry planning reads, so a node keeps + // serving the backend it was just moved off until the old key expires. + nodeURL = nodepool.NormalizeNodeURL(nodeURL) + h.v3NodeCapabilitiesMu.Lock() + delete(h.v3NodeCapabilities, nodeURL) + if h.v3NodeCapabilityInvalidations == nil { + h.v3NodeCapabilityInvalidations = make(map[string]uint64) + } + // Bumping the count is what makes the delete stick. A refresh already in + // flight fetched this node before its hardware changed, so it must neither + // re-install its answer over this delete nor be mistaken for the re-probe + // this invalidation is owed. + h.v3NodeCapabilityInvalidations[nodeURL]++ + claimed := h.claimCapabilityRefreshLockedV3(nodeURL) + h.v3NodeCapabilitiesMu.Unlock() + if claimed { + h.runCapabilityRefreshV3(nodeURL) + } +} + +func (h *PlaybackHandler) refreshRemoteCapabilitiesV3(nodeURL string) { + if h == nil || nodeURL == "" { return } + h.v3NodeCapabilitiesMu.Lock() + claimed := h.claimCapabilityRefreshLockedV3(nodeURL) + h.v3NodeCapabilitiesMu.Unlock() + if claimed { + h.runCapabilityRefreshV3(nodeURL) + } +} + +// claimCapabilityRefreshLockedV3 takes the node's single background-refresh +// slot. The caller must hold v3NodeCapabilitiesMu: taking the slot under the +// same lock as the invalidation counter is what guarantees that an invalidation +// either starts a refresh or is seen by the one already running. +func (h *PlaybackHandler) claimCapabilityRefreshLockedV3(nodeURL string) bool { + if _, inFlight := h.v3NodeCapabilityRefresh[nodeURL]; inFlight { + return false + } + if h.v3NodeCapabilityRefresh == nil { + h.v3NodeCapabilityRefresh = make(map[string]struct{}) + } + h.v3NodeCapabilityRefresh[nodeURL] = struct{}{} + return true +} + +// runCapabilityRefreshV3 probes one node in the background until its result is +// current: a probe that an invalidation overtook was discarded, so it repeats +// rather than leave the cache empty until the next viewer pays for a probe. +func (h *PlaybackHandler) runCapabilityRefreshV3(nodeURL string) { go func() { - defer h.v3NodeCapabilityRefresh.Delete(nodeURL) - ctx, cancel := context.WithTimeout(context.Background(), h.remoteToneMapProbeTimeoutV3(nodeURL)) - defer cancel() - if _, err := h.lookupRemoteCapabilitiesV3(ctx, nodeURL, false); err != nil { - slog.Debug("protocol v3 background node capability refresh failed", "component", "api", "node", logredact.SanitizeURL(nodeURL), "error", err) + for { + invalidations := h.nodeCapabilityInvalidationsV3(nodeURL) + ctx, cancel := context.WithTimeout(context.Background(), h.remoteToneMapProbeTimeoutV3(nodeURL)) + _, err := h.lookupRemoteCapabilitiesV3(ctx, nodeURL, false) + cancel() + if err != nil { + slog.Debug("protocol v3 background node capability refresh failed", "component", "api", "node", logredact.SanitizeURL(nodeURL), "error", err) + } + h.v3NodeCapabilitiesMu.Lock() + current := h.v3NodeCapabilityInvalidations[nodeURL] == invalidations + if current { + delete(h.v3NodeCapabilityRefresh, nodeURL) + } + h.v3NodeCapabilitiesMu.Unlock() + if current { + return + } } }() } @@ -2072,7 +2264,7 @@ func (h *PlaybackHandler) identityGrantStreamURLV3(ctx context.Context, s *playb if !stored { return h.playbackStreamURL(s), false, nil } - return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + s.ID, true, prior + return strings.TrimRight(proxyNode.ClientURL(), "/") + "/stream/v3/" + s.ID, true, prior } // putProxyGrantV3 stores the recipe a designated proxy origin serves this @@ -2417,7 +2609,7 @@ func (h *PlaybackHandler) identityStreamURLV3(s *playback.Session, file *models. if token == "" { return h.playbackStreamURL(s), false } - base := strings.TrimRight(proxyNode.URL, "/") + base := strings.TrimRight(proxyNode.ClientURL(), "/") if s.PlayMethod == playback.PlayRemux { if claims.PlayMethod == streamtoken.PlayMethodAudioDownmixRemux { return base + "/stream/remux/audio-v2/" + token, true @@ -2836,7 +3028,10 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla } sourceMetadata := sourceExecutionMetadataV3(file, result) sourceProfile, sourceBitDepth := sourceVideoTranscodeFactsV3(file, result) - hwAccel := h.playbackConfig().HWAccel + // The node's own override wins over this host's cluster-wide setting; every + // other node gets the cluster value verbatim, "auto" included, so the node + // still resolves it against live hardware at session start. + hwAccel := node.EffectiveHWAccel(h.playbackConfig().HWAccel) toneMapFilter := "" if result.ToneMapMode != "" { capabilities, err := h.remoteToneMapCapabilitiesV3(r.Context(), node.URL, false) @@ -3025,7 +3220,7 @@ func (h *PlaybackHandler) grantManifestURLV3(ctx context.Context, card playback. if !stored { return localURL, false, nil } - return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + card.SessionID + "/master.m3u8", true, prior + return strings.TrimRight(proxyNode.ClientURL(), "/") + "/stream/v3/" + card.SessionID + "/master.m3u8", true, prior } // sourceExecutionMetadataV3 freezes the source facts used by a remote executor. @@ -4841,10 +5036,18 @@ func (h *PlaybackHandler) enqueueRouteEventV3(event playback.RouteEventRecordV3) } h.v3EventOnce.Do(func() { h.v3EventQueue = make(chan playback.RouteEventRecordV3, 512) + // The store is captured with the queue rather than read per event. The + // goroutine below outlives the request that started it, so re-reading + // the field would be an unsynchronized read of handler state — harmless + // in production, where the router wires PlanStoreV3 once before serving, + // and a real data race against any caller that replaces it afterwards. + // Capturing also matches what the queue is: work batched for the store + // that existed when it was created. + store := h.PlanStoreV3 go func() { for value := range h.v3EventQueue { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - if err := h.PlanStoreV3.RecordRouteEvent(ctx, value); err != nil { + if err := store.RecordRouteEvent(ctx, value); err != nil { slog.Warn("playback route event write failed", "error", err, "event", value.Event) } cancel() diff --git a/internal/api/handlers/playback_v3_capability_retry_test.go b/internal/api/handlers/playback_v3_capability_retry_test.go new file mode 100644 index 000000000..14b244ff2 --- /dev/null +++ b/internal/api/handlers/playback_v3_capability_retry_test.go @@ -0,0 +1,118 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +// A capability lookup overtaken by an invalidation was handing the caller the +// pre-invalidation report, and that caller goes on to pick transformations and +// a tone-map executor from it — possibly hardware the newer report says is +// gone. It re-probes instead. +func TestLookupRemoteCapabilitiesRefetchesWhenOvertakenMidFlight(t *testing.T) { + handler := NewPlaybackHandler(nil) + var fetches atomic.Int32 + + // Captured after the server exists; the cache is keyed by the URL the lookup + // was given, which is not the Host header the node sees. + var nodeURL string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempt := fetches.Add(1) + if attempt == 1 { + // The health sweep notices the node changed while this first read is + // still in flight. The counter is bumped directly rather than through + // RefreshNodeCapabilitiesV3, which would also start a background + // re-probe and make the fetch count say nothing about this lookup. + handler.v3NodeCapabilitiesMu.Lock() + if handler.v3NodeCapabilityInvalidations == nil { + handler.v3NodeCapabilityInvalidations = make(map[string]uint64) + } + handler.v3NodeCapabilityInvalidations[nodeURL]++ + handler.v3NodeCapabilitiesMu.Unlock() + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "resolved": "qsv", + "render_devices": []string{"/dev/dri/renderD128"}, + "transformations": []map[string]any{ + {"name": "hdr_to_sdr_tone_map", "executor": "qsv", "recipe_version": "v3"}, + }, + }) + })) + t.Cleanup(node.Close) + nodeURL = node.URL + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), nodeURL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + if got := fetches.Load(); got != 2 { + t.Fatalf("fetches = %d, want the overtaken read repeated exactly once", got) + } +} + +// The ordinary path must not pay for that: a lookup nothing invalidates reads +// the node exactly once. +func TestLookupRemoteCapabilitiesFetchesOnceWhenNothingInvalidates(t *testing.T) { + handler := NewPlaybackHandler(nil) + var fetches atomic.Int32 + + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fetches.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"resolved": "qsv"}) + })) + t.Cleanup(node.Close) + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), node.URL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + if got := fetches.Load(); got != 1 { + t.Fatalf("fetches = %d, want a single read on the uncontended path", got) + } +} + +// An acceleration change makes a node's inventory wrong and its next capability +// matrix cold — which is exactly when the read is slowest. Dropping the learned +// budget along with the inventory sent the refresh that invalidation triggers +// back to the 120s fallback, short of what a two-device node legitimately asks +// for, so planning lost its inventory precisely after the invalidation. +func TestRefreshNodeCapabilitiesKeepsTheLearnedProbeBudget(t *testing.T) { + handler := NewPlaybackHandler(nil) + + advertised := 136_000 + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "resolved": "qsv", + "probe_request_timeout_ms": advertised, + }) + })) + t.Cleanup(node.Close) + + if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), node.URL, false); err != nil { + t.Fatalf("lookupRemoteCapabilitiesV3: %v", err) + } + learned := handler.remoteToneMapProbeTimeoutV3(node.URL) + if want := 136 * time.Second; learned != want { + t.Fatalf("learned budget = %v, want the advertised %v", learned, want) + } + + handler.RefreshNodeCapabilitiesV3(node.URL) + + if got := handler.remoteToneMapProbeTimeoutV3(node.URL); got != learned { + t.Fatalf("budget after invalidation = %v, want the learned %v — the refresh it triggers is sized from this", + got, learned) + } + // The inventory itself is still discarded; only the budget survives. + handler.v3NodeCapabilitiesMu.Lock() + _, cached := handler.v3NodeCapabilities[node.URL] + handler.v3NodeCapabilitiesMu.Unlock() + if cached { + t.Fatal("invalidation left the inventory cached") + } +} diff --git a/internal/api/handlers/playback_v3_node_hwaccel_test.go b/internal/api/handlers/playback_v3_node_hwaccel_test.go new file mode 100644 index 000000000..da5e7d3a1 --- /dev/null +++ b/internal/api/handlers/playback_v3_node_hwaccel_test.go @@ -0,0 +1,116 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// A node under a per-node acceleration override resolves to its own backend no +// matter what the request says, so a request carrying this API host's +// cluster-wide value would be silently corrected on arrival — and the recipe +// card would then describe an encode that never ran. Every other node keeps the +// cluster value verbatim: a node honors a named backend without re-checking it, +// so "auto" has to survive dispatch to reach live detection on the node, and a +// stale capability report must never stand in for it. +func TestPrepareRemoteTransportV3DispatchesTheNodesEffectiveBackend(t *testing.T) { + override := func(value string) *string { return &value } + tests := []struct { + name string + cluster string + override *string + capabilities string + want string + }{ + { + name: "node overridden to software wins over a qsv cluster", + cluster: "qsv", + override: override("none"), + want: "none", + }, + { + name: "node overridden to other hardware wins too", + cluster: "qsv", + override: override("nvenc"), + want: "nvenc", + }, + { + name: "an override lands immediately, ahead of the stale report it contradicts", + cluster: "qsv", + override: override("none"), + // The node was still reporting qsv when the operator disabled it. + capabilities: `{"resolved":"qsv"}`, + want: "none", + }, + { + name: "a node with no override keeps the cluster value", + cluster: "qsv", + want: "qsv", + }, + { + name: "a blank override is not an override", + cluster: "qsv", + override: override(" "), + want: "qsv", + }, + { + name: "auto reaches the node so it resolves against live hardware", + cluster: "auto", + // A boot-time probe that ran before the render devices were + // attached must not pin later sessions to software. + capabilities: `{"resolved":"none","render_devices":[]}`, + want: "auto", + }, + { + name: "a stale report never overrides the cluster value", + cluster: "qsv", + capabilities: `{"resolved":"none"}`, + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var got transcodenode.TranscodeStartRequest + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/transcode/start" { + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decode remote start: %v", err) + } + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{SessionID: got.SessionID, Status: "started", AudioRecipeVersion: got.AudioRecipeVersion}) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer node.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{TranscodeEnabled: true, HWAccel: test.cluster} + } + pooled := &nodepool.Node{URL: node.URL, HWAccelOverride: test.override} + if test.capabilities != "" { + pooled.Capabilities = json.RawMessage(test.capabilities) + } + transport, transportErr := handler.prepareRemoteTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-node-hwaccel", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), remoteHLSResultV3(), + nodepool.Plan{TranscodeNode: pooled}, preparedTimelineV3{}, mediaAuthModeV3{}, + ) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + if got.HWAccel != test.want { + t.Fatalf("dispatched hw_accel = %q, want %q", got.HWAccel, test.want) + } + }) + } +} diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 37e2224c9..a4ca70b93 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -5746,11 +5747,15 @@ func TestPlaybackV3ToneMapBudgetsCoverColdNodeWork(t *testing.T) { return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} } - if got, want := handler.localToneMapProbeTimeoutV3(), tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { + if got, want := handler.localToneMapProbeTimeoutV3(), playback.CapabilityEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { t.Fatalf("local tone-map probe timeout = %s, want %s", got, want) } - if got, want := handler.remoteToneMapProbeTimeoutV3("https://unknown.example"), remoteNodeProbeFallbackTimeout; got != want { - t.Fatalf("remote tone-map probe timeout = %s, want %s", got, want) + // A node this handler cannot resolve is priced at the cluster's own policy, + // which is what such a node runs unless it carries an override — closer than + // a flat guess, and the flat fallback is reserved for having no policy either. + coldUnknown := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/dev/dri/renderD128") + if got := handler.remoteToneMapProbeTimeoutV3("https://unknown.example"); got != coldUnknown { + t.Fatalf("remote tone-map probe timeout = %s, want %s", got, coldUnknown) } if got := handler.toneMapPlanningTimeoutV3(true); got != v3NodeCapabilityPlanTimeout { t.Fatalf("planning timeout with local fallback = %s, want %s", got, v3NodeCapabilityPlanTimeout) @@ -5765,7 +5770,7 @@ func TestPlaybackV3ToneMapBudgetsCoverColdNodeWork(t *testing.T) { RequireReady: true, HWAccel: tonemap.BackendQSV, } - want := remoteNodeProbeFallbackTimeout + playback.ManifestStartupTimeout + + want := coldUnknown + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout if got := handler.remotePlaybackTransportTimeout("https://unknown.example", request); got != want { t.Fatalf("remote tone-map start timeout = %s, want %s", got, want) @@ -5804,6 +5809,62 @@ func TestLookupRemoteCapabilitiesStartsCacheTTLAfterRequestCompletes(t *testing. } } +// waitForCachedNodeCapabilitiesV3 returns the first successful cache entry the +// background refresh installs for nodeURL. +func waitForCachedNodeCapabilitiesV3(t *testing.T, handler *PlaybackHandler, nodeURL string) v3NodeCapabilityCache { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + handler.v3NodeCapabilitiesMu.Lock() + entry, ok := handler.v3NodeCapabilities[nodeURL] + handler.v3NodeCapabilitiesMu.Unlock() + if ok && entry.err == nil { + return entry + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("no node capability report was cached before the deadline") + return v3NodeCapabilityCache{} +} + +// A report fetched before the health sweep reported the node's hardware changed +// must not be installed after the invalidation that change fired, and the +// invalidation still owes a re-probe: a cache that outlives the hardware it +// describes plans transcodes onto a GPU that is gone. +func TestRefreshNodeCapabilitiesDropsReportFetchedBeforeInvalidation(t *testing.T) { + var requests atomic.Int64 + firstRequest := make(chan struct{}) + release := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + transformation := "post_change" + if requests.Add(1) == 1 { + close(firstRequest) + <-release + transformation = "pre_change" + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + Transformations: []playback.TransformationV3{{Name: transformation}}, + }) + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.refreshRemoteCapabilitiesV3(remote.URL) + <-firstRequest + + // The sweep's callback lands while the earlier probe is still outstanding. + handler.RefreshNodeCapabilitiesV3(remote.URL) + close(release) + + entry := waitForCachedNodeCapabilitiesV3(t, handler, remote.URL) + if len(entry.transformations) != 1 || entry.transformations[0].Name != "post_change" { + t.Fatalf("cached transformations = %v, want the report fetched after the invalidation", entry.transformations) + } + if got := requests.Load(); got != 2 { + t.Fatalf("node was probed %d time(s), want the invalidation's own re-probe and no more", got) + } +} + func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { probeTimeoutMillis := int64(137000) remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -5812,12 +5873,18 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { defer remote.Close() handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) handler.PlaybackConfig = func() config.PlaybackConfig { - return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/central/device/one,/central/device/two"} + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/central/device/one"} } if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), remote.URL, false); err != nil { t.Fatal(err) } + // The node's own figure is what this path exists to use. It has to exceed + // what the cluster setting prices, or the floor below it would be answering + // and this would pass without the target node being consulted at all. + if cluster := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/central/device/one"); cluster >= 137*time.Second { + t.Fatalf("fixture is inert: the cluster price %s must stay under the node's 137s", cluster) + } if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), 137*time.Second; got != want { t.Fatalf("remote probe timeout = %s, want target node budget %s", got, want) } @@ -5829,11 +5896,161 @@ func TestRemoteToneMapProbeTimeoutUsesTargetNodeBudget(t *testing.T) { handler.v3NodeCapabilitiesMu.Lock() delete(handler.v3NodeCapabilities, remote.URL) handler.v3NodeCapabilitiesMu.Unlock() - probeTimeoutMillis = (10 * time.Minute).Milliseconds() + probeTimeoutMillis = (24 * time.Hour).Milliseconds() if _, err := handler.lookupRemoteCapabilitiesV3(context.Background(), remote.URL, false); err != nil { t.Fatal(err) } - if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), 5*time.Minute; got != want { + // Still bounded — the value comes off the wire from a worker — but at the + // ceiling the probe formula produces rather than a round number that a real + // nine-device node already exceeds. + if got, want := handler.remoteToneMapProbeTimeoutV3(remote.URL), playback.MaxCapabilityRequestTimeout(); got != want { t.Fatalf("bounded remote probe timeout = %s, want %s", got, want) } } + +// v3NodeLookupPlanner plans nothing and only resolves nodes by URL, which is all +// a cold probe budget needs from a planner. +type v3NodeLookupPlanner struct { + node *nodepool.Node +} + +func (p *v3NodeLookupPlanner) PlanSession(string, string, bool, int) nodepool.Plan { + return nodepool.Plan{} +} + +func (p *v3NodeLookupPlanner) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if p.node == nil || p.node.URL != nodeURL { + return nil, false + } + return p.node, true +} + +// The first read of a node after an API restart is also its most expensive: no +// probe cache on the node survives, so the whole matrix runs. The flat fallback +// is shorter than a two-device node legitimately takes, and paying it would +// cancel exactly the multi-GPU nodes this path exists to plan onto — so the +// durable report the node last advertised is what prices it. +func TestPlaybackV3ColdNodeProbeBudgetComesFromTheStoredReport(t *testing.T) { + const nodeURL = "https://gpu-1.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, + Capabilities: json.RawMessage(`{"resolved":"qsv","probe_request_timeout_ms":163000}`), + }} + + if got, want := handler.remoteToneMapProbeTimeoutV3(nodeURL), 163*time.Second; got != want { + t.Fatalf("cold probe timeout = %s, want the node's advertised %s", got, want) + } + if remoteNodeProbeFallbackTimeout >= 163*time.Second { + t.Fatal("fixture is inert: the advertised budget must exceed the flat fallback") + } +} + +// A node registered since the last capability fetch has no stored report to read +// a budget from, but it does have its own acceleration override — and that is +// what decides how long its walk takes. +func TestPlaybackV3ColdNodeProbeBudgetFollowsTheOverrideWithoutAReport(t *testing.T) { + const nodeURL = "https://gpu-2.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130" + backend := tonemap.BackendQSV + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 2, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != want { + t.Fatalf("cold probe timeout = %s, want the override's %s", got, want) + } + if cluster := playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); want <= cluster { + t.Fatalf("fixture is inert: the override budget %s must exceed the cluster's %s", want, cluster) + } +} + +// A per-node policy edit invalidates the inventory through the same path a +// hardware change does, and the learned budget deliberately survives that. But +// widening hw_device_override is exactly the change that makes the learned +// number too small: the node reloads, walks four devices instead of one, and is +// canceled at the one-device deadline on every retry — with no way back, since a +// budget is only learned from a read that completes. +func TestPlaybackV3RepricesALearnedBudgetAfterTheDeviceSetGrows(t *testing.T) { + const nodeURL = "https://gpu-3.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ + ID: 3, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + // Learned while the node was still on the cluster's single device. + learned := playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128") + handler.v3NodeCapabilitiesMu.Lock() + handler.rememberNodeProbeBudgetLockedV3(nodeURL, learned) + handler.v3NodeCapabilitiesMu.Unlock() + + handler.RefreshNodeCapabilitiesV3(nodeURL) + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != want { + t.Fatalf("probe timeout after the override grew = %s, want the four-device %s", got, want) + } + if want <= learned { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the learned %s", want, learned) + } +} + +// The reverse must still hold: a node whose own measurement exceeds what this +// replica can price keeps its measurement, which is the whole reason the learned +// budget survives an invalidation. +func TestPlaybackV3KeepsALearnedBudgetLargerThanThePolicyPrice(t *testing.T) { + const nodeURL = "https://gpu-4.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{HWAccel: tonemap.BackendQSV, HWDevice: "/dev/dri/renderD128"} + } + handler.NodePlanner = &v3NodeLookupPlanner{node: &nodepool.Node{ID: 4, URL: nodeURL}} + learned := playback.MaxCapabilityRequestTimeout() + handler.v3NodeCapabilitiesMu.Lock() + handler.rememberNodeProbeBudgetLockedV3(nodeURL, learned) + handler.v3NodeCapabilitiesMu.Unlock() + + if got := handler.remoteToneMapProbeTimeoutV3(nodeURL); got != learned { + t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) + } +} + +// The admin route invalidates with the URL exactly as the row stores it, which +// may carry a trailing slash the pools have already dropped. Keyed verbatim, the +// entry this deletes is not the entry planning reads, so the node keeps serving +// the backend it was just moved off until the old key expires. +func TestRefreshNodeCapabilitiesV3NormalizesTheCacheKey(t *testing.T) { + const nodeURL = "https://gpu-5.example" + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.v3NodeCapabilitiesMu.Lock() + if handler.v3NodeCapabilities == nil { + handler.v3NodeCapabilities = map[string]v3NodeCapabilityCache{} + } + handler.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{expiresAt: time.Now().Add(time.Hour)} + handler.v3NodeCapabilitiesMu.Unlock() + + handler.RefreshNodeCapabilitiesV3(nodeURL + "/") + + handler.v3NodeCapabilitiesMu.Lock() + _, present := handler.v3NodeCapabilities[nodeURL] + invalidations := handler.v3NodeCapabilityInvalidations[nodeURL] + handler.v3NodeCapabilitiesMu.Unlock() + if present { + t.Fatal("the canonical entry survived an invalidation made with a trailing slash") + } + if invalidations == 0 { + t.Fatal("the invalidation was counted under a different key than planning reads") + } +} diff --git a/internal/api/handlers/playback_v3_tokenless_test.go b/internal/api/handlers/playback_v3_tokenless_test.go index feac38e9a..40323a3a0 100644 --- a/internal/api/handlers/playback_v3_tokenless_test.go +++ b/internal/api/handlers/playback_v3_tokenless_test.go @@ -302,3 +302,28 @@ func TestHandleReplanPlaybackV3PinsAttemptStickyFeatures(t *testing.T) { t.Fatalf("durable client features = %v, the software-decode opt-in was dropped", record.NormalizedRequest.ClientFeatures) } } + +// Every client-facing proxy URL builder joins onto the proxy's ClientURL — +// the public URL when one is set — while the backend URL stays what the +// server and the stream-token tnode claim dial. A split-network proxy +// registered by its private address must never leak that address to a player. +func TestProxyURLBuildersUseThePublicURLWhenSet(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-stream-signing-secret" + file := v3HandlerFixtureFile(t) + public := "https://cdn.example.com" + proxy := &nodepool.Node{URL: "http://10.0.0.9:8083", PublicURL: &public} + + session := &playback.Session{ID: "session-public", UserID: 7, ProfileID: "profile-1", MediaFileID: file.ID, PlayMethod: playback.PlayDirect} + if got, servedByProxy := handler.identityStreamURLV3(session, file, proxy); !servedByProxy || !strings.HasPrefix(got, public+"/stream/direct/") { + t.Fatalf("identity URL = %q (proxy %v), want the public origin", got, servedByProxy) + } + + card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", playback.TranscodeOpts{SessionID: session.ID, InputPath: file.FilePath}) + if got := handler.buildProxyManifestURL(card, proxy, false); !strings.HasPrefix(got, public+"/stream/transcode/") { + t.Fatalf("manifest URL = %q, want the public origin", got) + } + if strings.Contains(handler.buildProxyManifestURL(card, proxy, false), "10.0.0.9") { + t.Fatalf("manifest URL leaked the backend address") + } +} diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go index 46db7ef88..a8be43961 100644 --- a/internal/api/handlers/playback_v3_union_test.go +++ b/internal/api/handlers/playback_v3_union_test.go @@ -782,9 +782,9 @@ func TestLookupRemoteCapabilitiesV3PreservesConcurrentFreshSuccessOnRefetchFailu handler.JWTSecret = "test-secret" handler.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) handler.v3NodeCapabilities[remote.URL] = v3NodeCapabilityCache{ - expiresAt: time.Now().Add(-time.Second), - probeRequestTimeout: time.Second, + expiresAt: time.Now().Add(-time.Second), } + handler.v3NodeProbeBudgets = map[string]time.Duration{remote.URL: time.Second} type lookupResult struct { entry v3NodeCapabilityCache err error diff --git a/internal/api/handlers/rate_limits.go b/internal/api/handlers/rate_limits.go index 2b39f5221..9b2b71569 100644 --- a/internal/api/handlers/rate_limits.go +++ b/internal/api/handlers/rate_limits.go @@ -46,6 +46,11 @@ type rateLimitConfigResponse struct { // ActiveBackend is the backend the running limiter actually uses, which // can differ from Backend until the server restarts. ActiveBackend string `json:"active_backend,omitempty"` + // RedisAvailable reports whether the Redis backend can be selected at all, + // using the same rule the save path enforces. Sentinel and REDIS_URL + // deployments have no persisted redis.url row, so admins cannot derive + // this client-side. + RedisAvailable bool `json:"redis_available"` } type tierConfigResponse struct { @@ -83,13 +88,17 @@ type authEndpointConfigRequest struct { // HandleGetConfig handles GET /admin/rate-limits/config. func (h *RateLimitHandler) HandleGetConfig(w http.ResponseWriter, r *http.Request) { - cfg, err := ratelimit.LoadConfig(r.Context(), h.store) + // One read serves the rate values, the stored backend, and the + // Redis-availability bit, so no field of the response can straddle two + // snapshots of the settings table. + values, err := h.store.GetAll(r.Context()) if err != nil { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load rate limit config") return } + cfg := ratelimit.ConfigFromSettings(values) - backend, _ := h.store.Get(r.Context(), "ratelimit.backend") + backend := values["ratelimit.backend"] if backend == "" { backend = "memory" } @@ -104,6 +113,7 @@ func (h *RateLimitHandler) HandleGetConfig(w http.ResponseWriter, r *http.Reques IPBurst: cfg.IPBurst, AuthEndpoints: make(map[string]authEndpointConfigResponse), Active: h.mw != nil, + RedisAvailable: redisConfiguredSettings(values, h.redisBootstrapAvailable), } if h.mw != nil { resp.ActiveBackend = h.mw.ActiveBackend() diff --git a/internal/api/handlers/rate_limits_test.go b/internal/api/handlers/rate_limits_test.go index 976fd3f69..54bdf634d 100644 --- a/internal/api/handlers/rate_limits_test.go +++ b/internal/api/handlers/rate_limits_test.go @@ -175,6 +175,48 @@ func TestRateLimitHandlerWithoutRunningLimiter(t *testing.T) { } } +func TestRateLimitHandlerReportsRedisAvailability(t *testing.T) { + // GET must answer with the same rule the save path enforces, so the UI can + // disable the Redis option instead of failing the admin at save time. + tests := []struct { + name string + values map[string]string + bootstrapAvail bool + wantRedisEnabled bool + }{ + {name: "nothing configured"}, + { + name: "malformed persisted url", + values: map[string]string{"redis.url": "not-a-url"}, + }, + { + name: "canonical persisted url", + values: map[string]string{"redis.url": "redis://cache.example.invalid:6379"}, + wantRedisEnabled: true, + }, + { + name: "bootstrap redis despite stale persisted url", + values: map[string]string{"redis.url": " redis://cache.example.invalid:6379 "}, + bootstrapAvail: true, + wantRedisEnabled: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + store := newFakeRateLimitStore() + for key, value := range tc.values { + store.values[key] = value + } + h := NewRateLimitHandler(store, nil, nil, NewServerRestartStatusTracker(), tc.bootstrapAvail) + + if got := getRateLimitConfig(t, h).RedisAvailable; got != tc.wantRedisEnabled { + t.Errorf("GET redis_available = %v, want %v", got, tc.wantRedisEnabled) + } + }) + } +} + func TestRateLimitHandlerUsesLatestCommittedSettingsAfterAtomicWrite(t *testing.T) { baseStore := newFakeRateLimitStore() baseStore.values["ratelimit.enabled"] = "false" diff --git a/internal/api/handlers/sections.go b/internal/api/handlers/sections.go index 3db2a0ed5..4f7100d2e 100644 --- a/internal/api/handlers/sections.go +++ b/internal/api/handlers/sections.go @@ -381,54 +381,77 @@ func (h *SectionHandler) HandleDeleteSection(w http.ResponseWriter, r *http.Requ return } collectionID := strings.TrimSpace(sections.ParseCollectionConfig(existing.Config).LibraryCollectionID) + var releaseCollectionLock func() + if collectionID != "" && h.CollectionRepo != nil { + unlockLocal := catalog.LockLibraryCollectionPosterMutation(collectionID) + unlockDatabase, err := h.CollectionRepo.AcquirePosterMutationLock(r.Context(), collectionID) + if err != nil { + unlockLocal() + slog.ErrorContext(r.Context(), "failed to lock section-managed collection before section delete", + "component", "api", "section_id", id, "collection_id", collectionID, "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to lock section-managed collection") + return + } + releaseCollectionLock = func() { + unlockDatabase() + unlockLocal() + } + defer func() { + if releaseCollectionLock != nil { + releaseCollectionLock() + } + }() + } if err := h.repo.Delete(r.Context(), id); err != nil { writeError(w, http.StatusNotFound, "not_found", "Section not found") return } - h.deleteUnreferencedSectionManagedCollection(r.Context(), collectionID) + collectionDeleted := h.deleteUnreferencedSectionManagedCollectionLocked(r.Context(), collectionID) + if releaseCollectionLock != nil { + releaseCollectionLock() + releaseCollectionLock = nil + } + if collectionDeleted && h.SortPreferenceCleaner != nil { + h.SortPreferenceCleaner.DeleteForCollection(r.Context(), userstore.CollectionKindLibrary, collectionID) + } w.WriteHeader(http.StatusNoContent) } -func (h *SectionHandler) deleteUnreferencedSectionManagedCollection(ctx context.Context, collectionID string) { +// deleteUnreferencedSectionManagedCollectionLocked removes the collection +// created solely for a deleted section. HandleDeleteSection holds both poster +// mutation locks so a lock failure can never happen after the section delete. +func (h *SectionHandler) deleteUnreferencedSectionManagedCollectionLocked(ctx context.Context, collectionID string) bool { if collectionID == "" || h.CollectionRepo == nil { - return - } - unlockLocal := catalog.LockLibraryCollectionPosterMutation(collectionID) - unlockDatabase, err := h.CollectionRepo.AcquirePosterMutationLock(ctx, collectionID) - if err != nil { - unlockLocal() - slog.WarnContext(ctx, "failed to lock section-managed collection during section delete", "component", "api", "collection_id", collectionID, "error", err) - return + return false } - defer func() { - unlockDatabase() - unlockLocal() - }() collection, err := h.CollectionRepo.GetByID(ctx, collectionID) if err != nil { if !errors.Is(err, catalog.ErrLibraryCollectionNotFound) { slog.WarnContext(ctx, "failed to load section-managed collection during section delete", "component", "api", "collection_id", collectionID, "error", err) } - return + return false } if collection.ManagementMode != "section" { - return + return false } refs, err := h.repo.CountLibraryCollectionReferences(ctx, collectionID, "") if err != nil { slog.WarnContext(ctx, "failed to count section-managed collection references", "component", "api", "collection_id", collectionID, "error", err) - return + return false } if refs > 0 { - return + return false } - if err := h.CollectionRepo.Delete(ctx, collectionID); err != nil && !errors.Is(err, catalog.ErrLibraryCollectionNotFound) { + err = h.CollectionRepo.Delete(ctx, collectionID) + if err == nil { + return true + } + if !errors.Is(err, catalog.ErrLibraryCollectionNotFound) { slog.WarnContext(ctx, "failed to delete unreferenced section-managed collection", "component", "api", "collection_id", collectionID, "error", err) - } else if err == nil && h.SortPreferenceCleaner != nil { - h.SortPreferenceCleaner.DeleteForCollection(ctx, userstore.CollectionKindLibrary, collectionID) } + return false } // HandleReorderSections handles PUT /admin/sections/reorder diff --git a/internal/api/handlers/server_restart_status.go b/internal/api/handlers/server_restart_status.go index 8be7b359d..e1ed2b558 100644 --- a/internal/api/handlers/server_restart_status.go +++ b/internal/api/handlers/server_restart_status.go @@ -1,6 +1,7 @@ package handlers import ( + "slices" "strings" "sync" "time" @@ -15,8 +16,18 @@ type ServerRestartStatusTracker struct { restartRequired bool restartRequiredAt time.Time restartRequiredReason string - restartRequested bool - restartRequestedAt time.Time + // restartMarkCount increments on every MarkRequired call. restartRequired + // latches true for the life of the process, so this counter is the only + // signal that a NEW restart-required save happened — the admin UI keys its + // banner re-arm (after "Later") on it. + restartMarkCount int + // restartReasons accumulates every distinct reason marked since boot, in + // first-seen order. The single restartRequiredReason only remembers the + // LAST save, so a tile scoped to one subsystem cannot trust it: an + // unrelated later save overwrites it. The full set can be scoped. + restartReasons []string + restartRequested bool + restartRequestedAt time.Time } type ServerRestartStatusSnapshot struct { @@ -24,6 +35,8 @@ type ServerRestartStatusSnapshot struct { RestartRequired bool RestartRequiredAt *time.Time RestartRequiredReason string + RestartReasons []string + RestartMarkCount int RestartRequested bool RestartRequestedAt *time.Time } @@ -49,8 +62,12 @@ func (s *ServerRestartStatusTracker) MarkRequired(reason string) { s.restartRequired = true s.restartRequiredAt = now } + s.restartMarkCount++ if reason != "" { s.restartRequiredReason = reason + if !slices.Contains(s.restartReasons, reason) { + s.restartReasons = append(s.restartReasons, reason) + } } } @@ -96,6 +113,8 @@ func (s *ServerRestartStatusTracker) Snapshot() ServerRestartStatusSnapshot { RestartRequired: s.restartRequired, RestartRequiredAt: restartRequiredAt, RestartRequiredReason: s.restartRequiredReason, + RestartReasons: slices.Clone(s.restartReasons), + RestartMarkCount: s.restartMarkCount, RestartRequested: s.restartRequested, RestartRequestedAt: restartRequestedAt, } diff --git a/internal/api/handlers/server_restart_status_test.go b/internal/api/handlers/server_restart_status_test.go new file mode 100644 index 000000000..823dc6576 --- /dev/null +++ b/internal/api/handlers/server_restart_status_test.go @@ -0,0 +1,29 @@ +package handlers + +import "testing" + +// The restart-required boolean latches for the life of the process, so the +// mark count is the only signal the admin UI has that a NEW restart-required +// save happened after the banner was dismissed. +func TestServerRestartStatusMarkCount(t *testing.T) { + tracker := NewServerRestartStatusTracker() + + if got := tracker.Snapshot().RestartMarkCount; got != 0 { + t.Fatalf("RestartMarkCount = %d before any mark, want 0", got) + } + + tracker.MarkRequired("ratelimit_backend") + tracker.MarkRequired("ratelimit_backend") // same reason still counts: it is a new save + tracker.MarkRequired("") + + snapshot := tracker.Snapshot() + if snapshot.RestartMarkCount != 3 { + t.Fatalf("RestartMarkCount = %d after three marks, want 3", snapshot.RestartMarkCount) + } + if !snapshot.RestartRequired { + t.Fatal("RestartRequired = false after marks, want true") + } + if snapshot.RestartRequiredReason != "ratelimit_backend" { + t.Fatalf("RestartRequiredReason = %q, want the last non-empty reason", snapshot.RestartRequiredReason) + } +} diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index dd89e61ca..de1627dda 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -9,30 +9,94 @@ import ( "github.com/Silo-Server/silo-server/internal/buildinfo" "github.com/Silo-Server/silo-server/internal/logredact" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" ) const remoteNodeInventoryProbeTimeout = 5 * time.Second +// resourceSampler is the read side of the local host's resource sampler. +type resourceSampler interface { + Snapshot() nodemetrics.Snapshot +} + +// playbackSettings reports the playback configuration a local probe should run +// under. It is a function rather than three strings because these settings hot +// reload: frozen at construction, /admin/system/hw-accel would keep probing the +// backend and devices the process started with, and the Playback settings page +// would show an operator a verification result for the configuration they just +// replaced. +type playbackSettings func() (ffmpegPath, hwAccel, hwDevice string) + // SystemHandler serves read-only system inspection endpoints. type SystemHandler struct { transcodePool *nodepool.TranscodePool jwtSecret string - ffmpegPath string + playback playbackSettings buildInfo buildinfo.Info + resources resourceSampler } -// NewSystemHandler creates a SystemHandler. -func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string, ffmpegPath string) *SystemHandler { +// NewSystemHandler creates a SystemHandler. playback supplies the current +// playback settings on each call, so a local probe verifies the backend and +// devices this host would transcode on right now. +func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string, playback playbackSettings) *SystemHandler { + if playback == nil { + playback = func() (string, string, string) { return "", "", "" } + } return &SystemHandler{ transcodePool: transcodePool, jwtSecret: jwtSecret, - ffmpegPath: ffmpegPath, + playback: playback, buildInfo: buildinfo.Current(), } } +// SetResourceSampler wires the local host's resource sampler. Without one, +// /admin/system/resources reports the host as unsampled rather than failing: +// the endpoint's answer is "what does this host look like right now", and +// "nothing is measuring it" is a valid answer to that. +func (h *SystemHandler) SetResourceSampler(sampler resourceSampler) { + h.resources = sampler +} + +// SystemResources is the local host's current resource sample. +type SystemResources struct { + // Available is false on a host that cannot be sampled (non-Linux, or before + // the first sample lands), in which case the two fields below are absent. + Available bool `json:"available"` + SampledAt string `json:"sampled_at,omitempty"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` +} + +// HandleSystemResources handles GET /admin/system/resources. +// +// This is the API host's own sample — the counterpart to the per-node +// last_stats on /admin/nodes, which the Nodes page reads. The API host is not a +// registered stream node, so without this route the machine actually serving +// the request is the one machine an operator cannot see. +// +// It reads a snapshot the sampler already published, so it costs nothing and +// cannot hang, no matter what a mount or a GPU query is doing. +func (h *SystemHandler) HandleSystemResources(w http.ResponseWriter, _ *http.Request) { + if h.resources == nil { + writeJSON(w, http.StatusOK, SystemResources{}) + return + } + snapshot := h.resources.Snapshot() + response := SystemResources{ + Available: snapshot.Available, + System: snapshot.System, + GPU: snapshot.GPU, + } + if !snapshot.SampledAt.IsZero() { + response.SampledAt = snapshot.SampledAt.UTC().Format(time.RFC3339) + } + writeJSON(w, http.StatusOK, response) +} + // NodeHWAccel reports one transcode node's GPU inventory. type NodeHWAccel struct { NodeURL string `json:"node_url"` @@ -67,10 +131,22 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if len(healthy) == 0 { - writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: playback.DetectHWAccelWithFFmpeg(h.ffmpegPath)}) + writeJSON(w, http.StatusOK, HWAccelInventory{HWAccelInfo: h.localHWAccel(w, r)}) return } + // The fan-out waits for its slowest node, whose budget can pass the API + // listener's write timeout — the same reason the local walk extends the + // deadline. Sized to the largest per-node budget, since the fetches run + // concurrently. + maxBudget := time.Duration(0) + for _, node := range healthy { + if budget := h.remoteInventoryTimeout(node); budget > maxBudget { + maxBudget = budget + } + } + extendWriteDeadlineBy(w, r, maxBudget+hwAccelWriteSlack) + inventory := HWAccelInventory{Nodes: make([]NodeHWAccel, len(healthy))} infos := make([]playback.HWAccelInfo, len(healthy)) errs := make([]error, len(healthy)) @@ -109,20 +185,82 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) { } } if !primaried { - inventory.HWAccelInfo = playback.DetectHWAccelWithFFmpeg(h.ffmpegPath) + inventory.HWAccelInfo = h.localHWAccel(w, r) } writeJSON(w, http.StatusOK, inventory) } +// localHWAccel probes this host against its current playback settings. +// +// A zero-value handler answers with the ffmpeg on PATH and auto-detection +// rather than panicking: tests build one directly, and the accessor is wiring +// this method should not depend on having received. +func (h *SystemHandler) localHWAccel(w http.ResponseWriter, r *http.Request) playback.HWAccelInfo { + var ffmpegPath, hwAccel, hwDevice string + if h.playback != nil { + ffmpegPath, hwAccel, hwDevice = h.playback() + } + extendHWAccelWriteDeadline(w, r, hwDevice) + return playback.DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice) +} + +// hwAccelWriteSlack covers reading the node list and writing the JSON around the +// walk this route runs. +const hwAccelWriteSlack = 15 * time.Second + +// extendHWAccelWriteDeadline lifts this connection's write deadline to cover a +// synchronous hardware walk. +// +// The walk is bounded by its own budget, which scales with the configured device +// set: eight Intel render devices draw five ffmpeg commands each at three +// seconds apiece, which is already past the API listener's 120-second write +// timeout. Without this the settings page loses its response while every probe +// is still inside its bound, and the operator sees a failed request for a probe +// that is about to succeed. The re-probe route extends its deadline for exactly +// the same reason. +func extendHWAccelWriteDeadline(w http.ResponseWriter, r *http.Request, hwDevice string) { + extendWriteDeadlineBy(w, r, playback.HWAccelWalkTimeout(hwDevice)+hwAccelWriteSlack) +} + +func extendWriteDeadlineBy(w http.ResponseWriter, r *http.Request, budget time.Duration) { + if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(budget)); err != nil { + // A ResponseWriter that cannot carry a deadline (a test recorder, a + // wrapper that does not unwrap) is not a reason to refuse the probe. + slog.WarnContext(r.Context(), "hw-accel write deadline not extended", "component", "api", + "budget", budget, "error", err) + } +} + // HandleBuildInfo handles GET /admin/system/build. func (h *SystemHandler) HandleBuildInfo(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, h.buildInfo) } func (h *SystemHandler) fetchRemoteHWAccel(ctx context.Context, node *nodepool.Node) (playback.HWAccelInfo, error) { - // Inventory is an interactive admin request, so a stalled healthy node must - // fail quickly and surface through the node entry's existing Error field. - requestCtx, cancel := context.WithTimeout(ctx, remoteNodeInventoryProbeTimeout) + requestCtx, cancel := context.WithTimeout(ctx, h.remoteInventoryTimeout(node)) defer cancel() return fetchRemoteTranscodeCapabilities(requestCtx, node.URL, h.jwtSecret) } + +// remoteInventoryTimeout bounds one node's inventory fetch by that node's own +// cold probe budget — its stored report and its effective override, exactly +// how the playback and download paths price a cold read. A warm node answers +// from cache in milliseconds either way; what this sizes is the node whose +// caches were just invalidated (a widened device override is the common case), +// whose full walk legitimately takes past the old flat five seconds — cutting +// it off reported the node as failed on the Playback settings page while it +// was still inside its own advertised budget. The flat constant survives as +// the floor, so a node the cluster prices at nothing still gets a real chance +// to answer, and it is the whole bound only when nothing is known at all. +func (h *SystemHandler) remoteInventoryTimeout(node *nodepool.Node) time.Duration { + var hwAccel, hwDevice string + if h.playback != nil { + _, hwAccel, hwDevice = h.playback() + } + return playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(hwAccel), + node.EffectiveHWDevice(hwDevice), + remoteNodeInventoryProbeTimeout, + ) +} diff --git a/internal/api/handlers/system_resources_test.go b/internal/api/handlers/system_resources_test.go new file mode 100644 index 000000000..1637e0eee --- /dev/null +++ b/internal/api/handlers/system_resources_test.go @@ -0,0 +1,120 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +func TestSystemResourcesReportsLocalSample(t *testing.T) { + t.Parallel() + + sampledAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + total := 71 + video, render := 63, 12 + handler := &SystemHandler{} + handler.SetResourceSampler(nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: sampledAt, + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, + VideoBusyPct: &video, RenderBusyPct: &render, TotalBusyPct: &total, + Source: nodemetrics.SourceFdinfo, + }}, + })) + + rec := httptest.NewRecorder() + handler.HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body struct { + Available bool `json:"available"` + SampledAt string `json:"sampled_at"` + System *struct { + CPUPct int `json:"cpu_pct"` + Disks []struct { + Path string `json:"path"` + } `json:"disks"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + TotalBusyPct *int `json:"total_busy_pct"` + } `json:"gpu"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if !body.Available { + t.Fatalf("available = false: %s", rec.Body) + } + if body.SampledAt != "2026-08-26T12:00:00Z" { + t.Fatalf("sampled_at = %q", body.SampledAt) + } + if body.System == nil || body.System.CPUPct != 41 || len(body.System.Disks) != 1 { + t.Fatalf("system = %+v", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("gpu = %+v", body.GPU) + } + if body.GPU[0].TotalBusyPct == nil || *body.GPU[0].TotalBusyPct != 71 { + t.Fatalf("total_busy_pct = %v, want 71", body.GPU[0].TotalBusyPct) + } +} + +// "Nothing is measuring this host" is a valid answer to "what does this host +// look like", so an unsampled host answers 200 with available:false rather than +// failing the admin page that reads it. +func TestSystemResourcesReportsUnavailableWithoutASampler(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + (&SystemHandler{}).HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if available, _ := body["available"].(bool); available { + t.Fatalf("available = true without a sampler: %s", rec.Body) + } + for _, key := range []string{"system", "gpu", "sampled_at"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sample: %s", key, rec.Body) + } + } +} + +// A host that cannot be sampled (non-Linux) is reported the same way as one +// with no sampler at all, so a client has one case to handle. +func TestSystemResourcesReportsUnavailableHost(t *testing.T) { + t.Parallel() + + handler := &SystemHandler{} + handler.SetResourceSampler(nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{})) + + rec := httptest.NewRecorder() + handler.HandleSystemResources(rec, httptest.NewRequest(http.MethodGet, "/admin/system/resources", nil)) + + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body) + } + if available, _ := body["available"].(bool); available { + t.Fatalf("available = true on an unsampled host: %s", rec.Body) + } +} diff --git a/internal/api/handlers/system_test.go b/internal/api/handlers/system_test.go index 5fe398cbe..425ceebc8 100644 --- a/internal/api/handlers/system_test.go +++ b/internal/api/handlers/system_test.go @@ -3,11 +3,14 @@ package handlers import ( "bytes" "encoding/json" + "fmt" "log/slog" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + "time" "github.com/go-chi/chi/v5" @@ -96,6 +99,37 @@ func TestSystemBuildInfoUnavailableResponseShape(t *testing.T) { } } +// The inventory fetch is bounded per node by that node's own cold probe +// budget. A node whose caches were just invalidated — a widened device +// override is the common case — legitimately walks past the flat floor, and +// cutting it off reported the node as failed on the Playback settings page +// while it was still inside its own advertised budget. +func TestRemoteInventoryTimeoutScalesWithTheNodeBudget(t *testing.T) { + t.Parallel() + + advertisedMillis := (90 * time.Second).Milliseconds() + report := json.RawMessage(fmt.Sprintf(`{"probe_request_timeout_ms":%d}`, advertisedMillis)) + node := &nodepool.Node{URL: "http://node:8082", Capabilities: report} + + handler := &SystemHandler{} + // Derivation-guard: the expected budget is what the shared pricing rule + // answers, asserted to exceed the flat floor so the test cannot pass + // vacuously if the fixture stops out-pricing it. + want := playback.ColdCapabilityRequestTimeout(report, "", "", remoteNodeInventoryProbeTimeout) + if want <= remoteNodeInventoryProbeTimeout { + t.Fatalf("fixture no longer out-prices the floor: got %v, floor %v", want, remoteNodeInventoryProbeTimeout) + } + if got := handler.remoteInventoryTimeout(node); got != want { + t.Fatalf("remoteInventoryTimeout() = %v, want the node's cold budget %v", got, want) + } + + // A node with no report and no override prices from the cluster policy + // over the floor — never below it. + if got := handler.remoteInventoryTimeout(&nodepool.Node{URL: "http://bare:8082"}); got < remoteNodeInventoryProbeTimeout { + t.Fatalf("remoteInventoryTimeout(bare node) = %v, below the %v floor", got, remoteNodeInventoryProbeTimeout) + } +} + func TestHandleHWAccelAggregatesAllHealthyNodes(t *testing.T) { t.Parallel() @@ -247,3 +281,43 @@ func TestHandleHWAccelProbeLogRedactsNodeURLSecrets(t *testing.T) { t.Fatalf("capability probe log lost sanitized node origin: %q", diagnostics) } } + +// The local inventory runs a full hardware walk on the request goroutine, and +// that walk grows with the configured device set — eight Intel render devices +// draw five ffmpeg commands each, already past the API listener's 120-second +// write timeout. Without lifting the deadline the settings page loses its +// response while every probe is still inside its own bound. +func TestHWAccelExtendsTheWriteDeadlineForItsWalk(t *testing.T) { + devices := make([]string, 0, 8) + for i := range 8 { + devices = append(devices, "/dev/dri/renderD"+strconv.Itoa(128+i)) + } + hwDevice := strings.Join(devices, ",") + handler := &SystemHandler{ + playback: func() (string, string, string) { return "/nonexistent/ffmpeg", "auto", hwDevice }, + } + + recorder := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + before := time.Now() + handler.HandleHWAccel(recorder, httptest.NewRequest(http.MethodGet, "/admin/system/hw-accel", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + // Sized from the walk's own budget, so it grows with the device set exactly + // as the walk does. How large that is depends on how the devices classify, + // which depends on the sysfs of the host running this — an Intel host draws + // five commands per device and passes 120 seconds at eight of them, while a + // machine with no such devices prices the same list far lower. So this + // asserts the derivation rather than a number no host agrees on. + want := playback.HWAccelWalkTimeout(hwDevice) + hwAccelWriteSlack + if reserved := recorder.deadline.Sub(before); reserved < want { + t.Fatalf("reserved %s, want at least the walk's own budget plus slack (%s)", reserved, want) + } + // And it is derived from the configured set, not a constant: a single device + // has to price lower than eight. + if single := playback.HWAccelWalkTimeout(devices[0]); single >= playback.HWAccelWalkTimeout(hwDevice) { + t.Fatalf("one device budgets %s and eight budget %s; the walk budget does not follow the device set", + single, playback.HWAccelWalkTimeout(hwDevice)) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 5a5a4b1e3..f47c7f28c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -49,6 +49,7 @@ import ( "github.com/Silo-Server/silo-server/internal/metadata/tmdb" metatrakt "github.com/Silo-Server/silo-server/internal/metadata/trakt" metadatatranslation "github.com/Silo-Server/silo-server/internal/metadata/translation" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/notifications" @@ -113,22 +114,31 @@ type Dependencies struct { StreamTelemetry *streamtelemetry.Registry // local observation-only stream telemetry (may be nil) // StreamTelemetryViewCache serves the merged global view with bounded // staleness so the admin parity endpoint never rebuilds it per request. - StreamTelemetryViewCache *streamtelemetry.ViewCache - SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) - StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) - MovieMatchQueueRepo *metadata.MovieMatchQueueRepository - SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository - Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) - NodeRepo *nodepool.Repository // stream node repository (may be nil) - ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) - TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) - NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) - SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger + StreamTelemetryViewCache *streamtelemetry.ViewCache + SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) + StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) + MovieMatchQueueRepo *metadata.MovieMatchQueueRepository + SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository + Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) + NodeRepo *nodepool.Repository // stream node repository (may be nil) + ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) + TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) + NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) + NodeHealthChecker *nodepool.HealthChecker // periodic node health/capability sweep (may be nil) + // NodeCapabilityInvalidator drops one node's cached capability inventory + // outside the playback handler — the prepared-download preparer holds its + // own. nil where downloads are not wired; set before NewRouter runs. + NodeCapabilityInvalidator func(nodeURL string) + ResourceSampler *nodemetrics.Sampler // this host's own resource sampler (may be nil) + SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger EventBus cache.EventBus AdminStatsProvider handlers.AdminStatsSource Recommender recommendations.Recommender // nil when disabled RecWorker *recommendations.Worker // nil when disabled CatalogSearchVectorizer catalog.CatalogSearchQueryVectorizer + // CatalogSearchSettings is the process-lifetime startup snapshot shared by + // every native/jellycompat provider and the index maintenance worker. + CatalogSearchSettings *catalog.CatalogSearchSettings RatingsRepo *catalog.RatingsRepo PersonRepo *catalog.PersonRepository PersonRefreshQueue handlers.PersonRefreshQueue @@ -159,11 +169,15 @@ type Dependencies struct { MarkerContributionStore *markers.ContributionStore MarkerContributionService *markers.ContributionService WatchProviderService handlers.WatchProviderService - WatchCompletionObserver watchstate.CompletionObserver - PluginService *plugins.Service - PluginHTTPProxy *plugins.HTTPProxy - PluginUserConfig *plugins.UserConfigStore - AuthProviders []auth.RegisteredProvider + // WatchProviderRegistry is the watchsync registry, used by the admin stats + // to list every provider — built-in or plugin-contributed — even when none + // of them has any activity yet. + WatchProviderRegistry handlers.WatchProviderLister + WatchCompletionObserver watchstate.CompletionObserver + PluginService *plugins.Service + PluginHTTPProxy *plugins.HTTPProxy + PluginUserConfig *plugins.UserConfigStore + AuthProviders []auth.RegisteredProvider // PublicURL is the externally-reachable origin (scheme + host) for this // silo instance. Used to build redirect_uri values handed to OAuth // IdPs. Empty disables the /oauth/{install_id}/{init,callback} routes. @@ -194,6 +208,13 @@ type Dependencies struct { // that case rather than failing. MDBListClient *mdblist.Client + // Admin dashboard aggregates, cached like AdminStatsProvider. Each is + // optional: without one, the matching route queries Postgres per request. + AdminPlaybackActivityProvider handlers.AdminPlaybackActivitySource + AdminTopActivityProvider handlers.AdminTopActivitySource + AdminTimeseriesProvider handlers.AdminTimeseriesSource + AdminDownloadsStatsProvider handlers.AdminDownloadsStatsSource + // ABSHandler is the Audiobookshelf-compatible HTTP handler. When non-nil // it is mounted at the root router level (not under /api/v1/) so that ABS // clients hitting /login, /api/*, /abs/api/*, and /abs/socket.io/* all @@ -221,6 +242,24 @@ func (d *Dependencies) CurrentConfig() *config.Config { // NewRouter creates a chi.Router with all middleware and routes mounted // under /api/v1/. ABS-compat routes (/abs/*, /login, /socket.io/*) are // mounted at the root level when deps.ABSHandler is non-nil. +// invalidateNodeCapabilities drops every cached view of one node's hardware. +// +// There is more than one: protocol-v3 planning holds an inventory, and prepared +// downloads hold their own with its own TTL. A policy edit or a capability hash +// change invalidates the node itself, not one reader of it, so anything that +// caches the answer has to be told — otherwise a QSV-to-NVENC edit keeps +// selecting the node for a tone-map executor it no longer has, and the +// reconfigured worker rejects the recipe or the download falls back locally for +// no reason. +func (deps Dependencies) invalidateNodeCapabilities(playbackHandler *handlers.PlaybackHandler) func(nodeURL string) { + return func(nodeURL string) { + playbackHandler.RefreshNodeCapabilitiesV3(nodeURL) + if deps.NodeCapabilityInvalidator != nil { + deps.NodeCapabilityInvalidator(nodeURL) + } + } +} + func NewRouter(deps Dependencies) chi.Router { declareNativeMediaRoutes() r := chi.NewRouter() @@ -578,13 +617,22 @@ func NewRouter(deps Dependencies) chi.Router { browseRepo := catalog.NewBrowseRepository(deps.DB) itemRepo = catalog.NewItemRepository(deps.DB) searchIndexEvents := catalog.NewSearchIndexEventRepository(deps.DB) - catalogSearchService = catalog.NewCatalogSearchService( - context.Background(), - settingsRepo, - itemRepo, - searchIndexEvents, - deps.CatalogSearchVectorizer, - ) + if deps.CatalogSearchSettings != nil { + catalogSearchService = catalog.NewCatalogSearchServiceFromSettings( + *deps.CatalogSearchSettings, + itemRepo, + searchIndexEvents, + deps.CatalogSearchVectorizer, + ) + } else { + catalogSearchService = catalog.NewCatalogSearchService( + context.Background(), + settingsRepo, + itemRepo, + searchIndexEvents, + deps.CatalogSearchVectorizer, + ) + } if catalogSearchService != nil { catalogSearchService.StartCoverageRefresh(deps.AppContext) } @@ -1032,6 +1080,10 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.SetProfileRefreshRequester(deps.RecWorker) } playbackHandler.StartCapabilityWarmupV3(deps.AppContext) + // The health sweep sees a node's capability hash change long before this + // cache would expire, so let it invalidate directly. Wired here rather + // than at checker construction because the handler does not exist yet. + deps.NodeHealthChecker.SetCapabilitiesChangedCallback(deps.invalidateNodeCapabilities(playbackHandler)) realtimeHub := deps.PlaybackRealtimeHub if realtimeHub == nil { @@ -1150,6 +1202,12 @@ func NewRouter(deps Dependencies) chi.Router { adminHandler.EventsHub = deps.EventsHub adminHandler.ImpersonationService = authService adminHandler.StatsSource = deps.AdminStatsProvider + adminHandler.WatchProviders = deps.WatchProviderRegistry + adminHandler.PlaybackActivitySource = deps.AdminPlaybackActivityProvider + adminHandler.TopActivitySource = deps.AdminTopActivityProvider + adminHandler.TimeseriesSource = deps.AdminTimeseriesProvider + adminHandler.DownloadsStatsSource = deps.AdminDownloadsStatsProvider + adminHandler.RedisClient = deps.RedisClient adminHandler.RealtimeHub = deps.RealtimeHub adminHandler.AccessGroups = accessGroupStore adminHandler.BootstrapSensitiveConfigured = deps.BootstrapSensitiveConfigured @@ -1158,6 +1216,11 @@ func NewRouter(deps Dependencies) chi.Router { adminHandler.RestartStatus = restartStatus adminHandler.CatalogSearchStatus = catalogSearchService adminHandler.DiagnosticsStore = diagnosticsStore + // Same source branding asset uploads and the metadata image cacher use: + // the public S3 client only exists when a public bucket is configured, + // and both features are wired off it. + publicAssetStore := deps.S3Public + adminHandler.PublicStorageConfigured = func() bool { return publicAssetStore != nil } if settingsRepo != nil { adminHandler.SettingsRepo = settingsRepo } @@ -1528,19 +1591,12 @@ func NewRouter(deps Dependencies) chi.Router { client: tmdb.NewClient(apiKey, 40), } } - traktClientID := "" - if settingsRepo != nil { - ctx := deps.AppContext - if ctx == nil { - ctx = context.Background() - } - if value, err := settingsRepo.Get(ctx, "watchsync.trakt.client_id"); err == nil { - traktClientID = value - } - } if libraryCollectionService.TraktCollections == nil { + // The client ID is resolved per call rather than captured here, so + // saving new Trakt credentials applies without a server restart. libraryCollectionService.TraktCollections = &traktCollectionAdapter{ - client: metatrakt.NewClient(traktClientID, 5), + client: metatrakt.NewClient("", 5), + settings: settingsRepo, } } if libraryCollectionService.TraktTokenResolver == nil && deps.DB != nil && settingsRepo != nil { @@ -2922,7 +2978,28 @@ func NewRouter(deps Dependencies) chi.Router { r.Get("/playback-history", adminHandler.HandleListPlaybackHistory) r.Get("/unmatched", adminHandler.HandleListUnmatched) r.Get("/stats", adminHandler.HandleGetStats) + // Dashboard aggregates. Both are cached reads over the + // same catalog/playback tables /stats uses, split out + // because their windows and refresh rates differ. + r.Get("/stats/playback-activity", adminHandler.HandleGetPlaybackActivity) + r.Get("/stats/top-activity", adminHandler.HandleGetTopActivity) + // Offline-download aggregate for the dashboard's + // downloads widget. Reads the downloads table, which + // exists whether or not the feature is enabled, so a + // download-less deployment answers zeros. + r.Get("/stats/downloads", adminHandler.HandleGetDownloadsStats) + // Minute-resolution samples written by the dashboard + // metrics sampler (internal/dashmetrics); the only + // history for concurrent streams and egress. + r.Get("/stats/timeseries", adminHandler.HandleGetTimeseries) r.Get("/server/status", adminHandler.HandleGetServerStatus) + // Per-admin-account dashboard arrangement. The server + // stores it as an opaque JSON object; the web client + // owns widget-id and span validation. + r.Get("/dashboard/capabilities", adminHandler.HandleGetDashboardCapabilities) + r.Get("/dashboard/layout", adminHandler.HandleGetDashboardLayout) + r.Put("/dashboard/layout", adminHandler.HandlePutDashboardLayout) + r.Delete("/dashboard/layout", adminHandler.HandleDeleteDashboardLayout) r.Get("/catalog/search/status", adminHandler.HandleGetCatalogSearchStatus) if policyHandler != nil { r.Route("/policy", func(r chi.Router) { @@ -2956,6 +3033,7 @@ func NewRouter(deps Dependencies) chi.Router { r.Post("/jellyfin-compat/web/update", adminHandler.HandleUpdateJellyfinCompatWeb) r.Post("/jellyfin-compat/web/remove", adminHandler.HandleRemoveJellyfinCompatWeb) r.Get("/settings/sensitive-status", adminHandler.HandleGetSensitiveStatus) + r.Get("/settings/restart-keys", adminHandler.HandleGetRestartKeys) r.Post("/settings/check/{kind}", adminHandler.HandleCheckSettingsConnection) if sectionSettingsHandler != nil { r.Get("/settings/sections", sectionSettingsHandler.HandleGet) @@ -3171,6 +3249,23 @@ func NewRouter(deps Dependencies) chi.Router { jwtSecret = deps.Config.Auth.JWTSecret } nodeHandler := handlers.NewNodeHandler(deps.NodeRepo, deps.ProxyPool, deps.TranscodePool, deps.NodeRepo, deps.EventBus, deps.RedisClient, jwtSecret) + // A re-probe stores the node's new inventory through the + // sweep's own refresh, so the drift and persist rules have + // one implementation. Without a health checker the node + // still re-probes and the row catches up on a later sweep. + if deps.NodeHealthChecker != nil { + nodeHandler.SetCapabilityRefresher(deps.NodeHealthChecker) + } + // An acceleration override change makes this + // server's cached view of the node wrong the + // moment it lands; the same invalidation the + // health sweep uses drops it. + nodeHandler.SetCapabilityInvalidator(deps.invalidateNodeCapabilities(playbackHandler)) + // A node with no override of its own runs the + // cluster's acceleration policy, and how many + // devices that names is what decides how long its + // re-probe may take. + nodeHandler.SetClusterPlaybackPolicy(playbackHandler.PlaybackConfig) r.Route("/nodes", func(r chi.Router) { r.Get("/", nodeHandler.HandleListNodes) r.Post("/", nodeHandler.HandleCreateNode) @@ -3179,6 +3274,7 @@ func NewRouter(deps Dependencies) chi.Router { r.Post("/{id}/check", nodeHandler.HandleCheckNode) r.Post("/force-reload", nodeHandler.HandleForceReloadNodes) r.Post("/{id}/force-reload", nodeHandler.HandleForceReloadNode) + r.Post("/{id}/reprobe", nodeHandler.HandleReprobeNode) }) // Live node sessions (reads from Redis) // Note: /admin/sessions is already used for playback sessions from PostgreSQL. @@ -3188,15 +3284,29 @@ func NewRouter(deps Dependencies) chi.Router { // System inspection. { sysJWTSecret := "" - sysFFmpegPath := "" if deps.Config != nil { sysJWTSecret = deps.Config.Auth.JWTSecret - sysFFmpegPath = deps.Config.Playback.FFmpegPath } - systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath) + // Read per request, not captured: playback + // settings hot reload, and a probe against the + // values this process started with would show an + // operator a result for the configuration they + // just replaced. + systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, + func() (string, string, string) { + cfg := deps.CurrentConfig() + if cfg == nil { + return "", "", "" + } + return cfg.Playback.FFmpegPath, cfg.Playback.HWAccel, cfg.Playback.HWDevice + }) + if deps.ResourceSampler != nil { + systemHandler.SetResourceSampler(deps.ResourceSampler) + } r.Route("/system", func(r chi.Router) { r.Get("/build", systemHandler.HandleBuildInfo) r.Get("/hw-accel", systemHandler.HandleHWAccel) + r.Get("/resources", systemHandler.HandleSystemResources) }) } @@ -3668,11 +3778,33 @@ func (a *tmdbDiscoverAdapter) Discover(ctx context.Context, mediaType string, pa return entries, nil } +// traktClientIDSettingKey holds the Trakt app client ID. It is deliberately +// not in config.restartRequiredKeys: the adapter re-reads it before every +// upstream call, so a saved change converges without a restart. +const traktClientIDSettingKey = "watchsync.trakt.client_id" + type traktCollectionAdapter struct { client *metatrakt.Client + // settings is the live source of the app client ID. Nil only where no + // settings store exists (tests), where the client ID stays empty and the + // upstream call fails the same way it always did. + settings catalog.SettingsStore +} + +// refreshClientID pushes the currently saved app client ID onto the shared +// client. A read failure leaves the last known value in place: failing the +// request at Trakt is more useful than failing it here on a transient DB blip. +func (a *traktCollectionAdapter) refreshClientID(ctx context.Context) { + if a.settings == nil { + return + } + if clientID, err := a.settings.Get(ctx, traktClientIDSettingKey); err == nil { + a.client.SetClientID(clientID) + } } func (a *traktCollectionAdapter) GetCollectionPreset(ctx context.Context, preset, mediaType string, limit int, accessToken string) ([]catalog.TraktCollectionEntry, error) { + a.refreshClientID(ctx) results, err := a.client.GetCollectionPreset(ctx, preset, mediaType, limit, accessToken) if err != nil { return nil, err @@ -3694,6 +3826,7 @@ func (a *traktCollectionAdapter) GetCollectionPreset(ctx context.Context, preset } func (a *traktCollectionAdapter) GetUserList(ctx context.Context, user, list string, limit int, accessToken string) ([]catalog.TraktCollectionEntry, error) { + a.refreshClientID(ctx) results, err := a.client.GetUserList(ctx, user, list, limit, accessToken) if err != nil { return nil, err diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt index 39c0431ac..f5c1a8f75 100644 --- a/internal/api/testdata/media_routes.txt +++ b/internal/api/testdata/media_routes.txt @@ -40,6 +40,10 @@ PUT /api/v1/admin/collections/{id}/items/order non-media DELETE /api/v1/admin/collections/{id}/items/{item_id} non-media PUT /api/v1/admin/collections/{id}/items/{item_id} non-media POST /api/v1/admin/collections/{id}/sync non-media +GET /api/v1/admin/dashboard/capabilities non-media +DELETE /api/v1/admin/dashboard/layout non-media +GET /api/v1/admin/dashboard/layout non-media +PUT /api/v1/admin/dashboard/layout non-media GET /api/v1/admin/devices non-media GET /api/v1/admin/devices/{user_id}/{device_id} non-media GET /api/v1/admin/diagnostics/reports/ non-media @@ -143,12 +147,17 @@ GET /api/v1/admin/settings non-media PUT /api/v1/admin/settings non-media POST /api/v1/admin/settings/check/{kind} non-media GET /api/v1/admin/settings/effective non-media +GET /api/v1/admin/settings/restart-keys non-media GET /api/v1/admin/settings/sections non-media PUT /api/v1/admin/settings/sections non-media GET /api/v1/admin/settings/sensitive-status non-media GET /api/v1/admin/settings/{key} non-media PUT /api/v1/admin/settings/{key} non-media GET /api/v1/admin/stats non-media +GET /api/v1/admin/stats/downloads non-media +GET /api/v1/admin/stats/playback-activity non-media +GET /api/v1/admin/stats/timeseries non-media +GET /api/v1/admin/stats/top-activity non-media GET /api/v1/admin/stream-telemetry/parity non-media GET /api/v1/admin/subtitle-providers/ non-media PUT /api/v1/admin/subtitle-providers/{provider}/ non-media @@ -159,6 +168,7 @@ PATCH /api/v1/admin/subtitles/{id}/ non-media GET /api/v1/admin/subtitles/{id}/download non-media GET /api/v1/admin/system/build non-media GET /api/v1/admin/system/hw-accel non-media +GET /api/v1/admin/system/resources non-media GET /api/v1/admin/unmatched non-media GET /api/v1/admin/users non-media POST /api/v1/admin/users non-media diff --git a/internal/catalog/folder_delete_test.go b/internal/catalog/folder_delete_test.go index 311165371..4a5f8400d 100644 --- a/internal/catalog/folder_delete_test.go +++ b/internal/catalog/folder_delete_test.go @@ -23,6 +23,70 @@ func withFastDeadlockRetry(t *testing.T, maxAttempts int) { }) } +// withFastFolderCollectionLockSetRetry shrinks stabilization timing/attempts +// for tests and restores the originals on cleanup. Tests using it must not call +// t.Parallel(). +func withFastFolderCollectionLockSetRetry(t *testing.T, maxAttempts int) { + t.Helper() + oldMax, oldBackoff := folderCollectionLockSetMaxAttempts, folderCollectionLockSetBaseBackoff + folderCollectionLockSetMaxAttempts = maxAttempts + folderCollectionLockSetBaseBackoff = time.Millisecond + t.Cleanup(func() { + folderCollectionLockSetMaxAttempts = oldMax + folderCollectionLockSetBaseBackoff = oldBackoff + }) +} + +func TestRetryFolderCollectionLockSetStabilizes(t *testing.T) { + withFastFolderCollectionLockSetRetry(t, 3) + calls := 0 + deletedIDs, err := retryFolderCollectionLockSet(context.Background(), 42, func() (bool, []string, error) { + calls++ + if calls == 1 { + return true, nil, nil + } + return false, []string{"collection-1"}, nil + }) + if err != nil { + t.Fatalf("retryFolderCollectionLockSet: %v", err) + } + if calls != 2 || len(deletedIDs) != 1 || deletedIDs[0] != "collection-1" { + t.Fatalf("calls = %d, deleted IDs = %v", calls, deletedIDs) + } +} + +func TestRetryFolderCollectionLockSetStopsWhenUnstable(t *testing.T) { + withFastFolderCollectionLockSetRetry(t, 3) + calls := 0 + _, err := retryFolderCollectionLockSet(context.Background(), 42, func() (bool, []string, error) { + calls++ + return true, nil, nil + }) + if !errors.Is(err, errFolderCollectionLockSetNeverStable) { + t.Fatalf("error = %v, want unstable collection-set error", err) + } + if calls != 3 { + t.Fatalf("calls = %d, want 3", calls) + } +} + +func TestRetryFolderCollectionLockSetStopsOnCanceledContext(t *testing.T) { + withFastFolderCollectionLockSetRetry(t, 3) + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + _, err := retryFolderCollectionLockSet(ctx, 42, func() (bool, []string, error) { + calls++ + cancel() + return true, nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + if calls != 1 { + t.Fatalf("calls = %d, want 1", calls) + } +} + func TestRetryOnDeadlockRetriesThenSucceeds(t *testing.T) { withFastDeadlockRetry(t, 5) calls := 0 diff --git a/internal/catalog/folder_repo.go b/internal/catalog/folder_repo.go index e1a2da9f0..c21c284b5 100644 --- a/internal/catalog/folder_repo.go +++ b/internal/catalog/folder_repo.go @@ -15,14 +15,17 @@ import ( "github.com/Silo-Server/silo-server/internal/models" ) -// Retry parameters for transient serialization/deadlock failures. They are -// package vars (not consts) only so tests can shrink them; production code -// never mutates them. +// Retry parameters are package vars (not consts) only so tests can shrink +// them; production code never mutates them. var ( - deadlockMaxAttempts = 5 - deadlockBaseBackoff = 50 * time.Millisecond + deadlockMaxAttempts = 5 + deadlockBaseBackoff = 50 * time.Millisecond + folderCollectionLockSetMaxAttempts = 5 + folderCollectionLockSetBaseBackoff = 10 * time.Millisecond ) +var errFolderCollectionLockSetNeverStable = errors.New("library collection set kept changing during folder delete") + const ( // orphanDeleteBatch is small because each media_items row cascades across // ~15 child tables. @@ -714,14 +717,10 @@ func (r *FolderRepository) DeleteWithStats( // holding a pooled connection; after the lifecycle lock makes the child set // stable, the set is re-read and the attempt is retried if a new child appeared. func (r *FolderRepository) deleteFolderRowWithCollectionPosterLocks(ctx context.Context, folderID int) ([]string, error) { - for { - if err := ctx.Err(); err != nil { - return nil, err - } - + return retryFolderCollectionLockSet(ctx, folderID, func() (bool, []string, error) { collectionIDs, err := r.listOwnedLibraryCollectionIDs(ctx, folderID) if err != nil { - return nil, err + return false, nil, err } unlockLocal := make([]func(), 0, len(collectionIDs)) for _, collectionID := range collectionIDs { @@ -732,12 +731,42 @@ func (r *FolderRepository) deleteFolderRowWithCollectionPosterLocks(ctx context. for i := len(unlockLocal) - 1; i >= 0; i-- { unlockLocal[i]() } + if err != nil { + return false, nil, err + } + return retry, deletedIDs, nil + }) +} + +// retryFolderCollectionLockSet bounds stabilization when concurrent creates or +// reparents change the child set between its initial read and lifecycle-lock +// acquisition. It never permits deletion without every discovered poster lock. +func retryFolderCollectionLockSet( + ctx context.Context, + folderID int, + attempt func() (retry bool, deletedIDs []string, err error), +) ([]string, error) { + backoff := folderCollectionLockSetBaseBackoff + for attemptNumber := 1; ; attemptNumber++ { + if err := ctx.Err(); err != nil { + return nil, err + } + retry, deletedIDs, err := attempt() if err != nil { return nil, err } if !retry { return deletedIDs, nil } + if attemptNumber >= folderCollectionLockSetMaxAttempts { + return nil, fmt.Errorf("folder %d: %w after %d attempts", folderID, errFolderCollectionLockSetNeverStable, attemptNumber) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + } + backoff *= 2 } } @@ -1159,6 +1188,30 @@ func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deleti return unreferenced, nil } +// DistinctLibraryPaths returns every configured library folder path, once each +// and in a stable order. +// +// media_folder_paths is the authoritative list of roots the server was told +// about; a folder can have several. Host resource sampling uses this to decide +// which mounts to report free space on, which is why it reads paths only and +// deliberately does not care which library owns them. +func (r *FolderRepository) DistinctLibraryPaths(ctx context.Context) ([]string, error) { + rows, err := r.pool.Query(ctx, `SELECT DISTINCT path FROM media_folder_paths ORDER BY path`) + if err != nil { + return nil, fmt.Errorf("querying library paths: %w", err) + } + defer rows.Close() + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("scanning library path: %w", err) + } + paths = append(paths, path) + } + return paths, rows.Err() +} + // UpdateLastScanned sets the last_scanned_at timestamp for the given folder. // LibraryRootsForContent returns the media folder root paths the given // content belongs to, resolved through media_item_libraries. The metadata diff --git a/internal/catalog/item_repo_test.go b/internal/catalog/item_repo_test.go index b24fa83de..7c859505c 100644 --- a/internal/catalog/item_repo_test.go +++ b/internal/catalog/item_repo_test.go @@ -491,6 +491,9 @@ func TestItemRepo_Search_AliasScoresUseOneUncorrelatedPass(t *testing.T) { if strings.Contains(sql, "FROM media_item_aliases mia WHERE mia.content_id = mi.content_id") { t.Fatalf("search must not rescan every alias row per media candidate; got:\n%s", sql) } + if strings.Contains(sql, "title_rank") { + t.Fatalf("search must not retain the removed constant title_rank sort key; got:\n%s", sql) + } } } diff --git a/internal/catalog/library_collection_repo.go b/internal/catalog/library_collection_repo.go index 459800198..d4af50ec0 100644 --- a/internal/catalog/library_collection_repo.go +++ b/internal/catalog/library_collection_repo.go @@ -158,6 +158,10 @@ const ( libraryCollectionLifecycleLockSQL = `SELECT pg_advisory_xact_lock(hashtextextended('library_collection_lifecycle:' || $1::text, 0))` ) +// Package variable only so the PostgreSQL integration test can use a short +// bound. Production code never mutates it. +var libraryCollectionPosterLockTimeout = 15 * time.Second + func acquireLibraryCollectionAdvisoryTransaction( ctx context.Context, pool *pgxpool.Pool, @@ -204,6 +208,10 @@ func (r *LibraryCollectionRepository) AcquirePosterMutationLock(ctx context.Cont if err != nil { return nil, err } + if _, err := tx.Exec(ctx, `SELECT set_config('lock_timeout', $1, true)`, libraryCollectionPosterLockTimeout.String()); err != nil { + release() + return nil, fmt.Errorf("setting library collection poster lock timeout: %w", err) + } if _, err := tx.Exec(ctx, libraryCollectionPosterAdvisoryLockSQL, collectionID); err != nil { release() return nil, fmt.Errorf("acquiring library collection poster advisory lock: %w", err) @@ -564,6 +572,9 @@ func libraryCollectionLifecycleLockIDs(oldLibraryID, newLibraryID int) []int { return ids } +// When LibraryIDs is non-nil, Update acquires the local poster lock followed by +// the database poster lock. Callers must not hold either non-reentrant lock when +// invoking Update with LibraryIDs set. func (r *LibraryCollectionRepository) Update(ctx context.Context, input UpdateLibraryCollectionInput) error { var ( updatedLibraryIDs []int diff --git a/internal/catalog/library_collection_repo_test.go b/internal/catalog/library_collection_repo_test.go index 9ca536e1f..9f35fe5af 100644 --- a/internal/catalog/library_collection_repo_test.go +++ b/internal/catalog/library_collection_repo_test.go @@ -1,11 +1,156 @@ package catalog import ( + "context" + "errors" + "fmt" + "os" "slices" + "strings" "testing" "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" ) +func TestLibraryCollectionColumnsKeepPosterSuppressionInScanOrder(t *testing.T) { + want := "lc.poster_thumbhash, lc.backdrop_thumbhash, lc.poster_auto_generated, lc.poster_suppressed, lc.poster_from_template" + if !strings.Contains(libraryCollectionColumns, want) { + t.Fatalf("library collection scan columns do not contain %q in order", want) + } +} + +func TestLibraryCollectionPosterSuppressionRoundTrip(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + suffix := time.Now().UnixNano() + var libraryID int + if err := pool.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ('movies', $1, true) RETURNING id`, + fmt.Sprintf("poster-suppression-%d", suffix), + ).Scan(&libraryID); err != nil { + t.Fatalf("seed library: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, libraryID) + }) + + repo := NewLibraryCollectionRepository(pool) + collection, err := repo.Create(ctx, CreateLibraryCollectionInput{ + LibraryID: libraryID, + LibraryIDs: []int{libraryID}, + Slug: fmt.Sprintf("poster-suppression-%d", suffix), + Title: "Poster suppression mapping", + CollectionType: "manual", + Visibility: "visible", + PosterSuppressed: true, + }) + if err != nil { + t.Fatalf("create collection: %v", err) + } + if !collection.PosterSuppressed { + t.Fatal("created collection lost poster suppression") + } + + loaded, err := repo.GetByID(ctx, collection.ID) + if err != nil { + t.Fatalf("reload collection: %v", err) + } + if !loaded.PosterSuppressed { + t.Fatal("scanned collection lost poster suppression") + } + + updated, err := repo.UpdateGeneratedPosterIfAllowed(ctx, collection.ID, "collection-images/generated.webp", "thumbhash") + if err != nil { + t.Fatalf("conditionally update suppressed poster: %v", err) + } + if updated { + t.Fatal("suppressed collection accepted an automatic poster") + } + + notSuppressed := false + if err := repo.Update(ctx, UpdateLibraryCollectionInput{ + ID: collection.ID, + PosterSuppressed: ¬Suppressed, + }); err != nil { + t.Fatalf("clear poster suppression: %v", err) + } + updated, err = repo.UpdateGeneratedPosterIfAllowed(ctx, collection.ID, "collection-images/generated.webp", "thumbhash") + if err != nil { + t.Fatalf("conditionally update unsuppressed poster: %v", err) + } + if !updated { + t.Fatal("unsuppressed empty collection rejected an automatic poster") + } +} + +func TestAcquirePosterMutationLockTimesOutAndReleasesResources(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + holder, err := pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock-holder connection: %v", err) + } + t.Cleanup(holder.Release) + holderTx, err := holder.Begin(ctx) + if err != nil { + t.Fatalf("begin lock-holder transaction: %v", err) + } + t.Cleanup(func() { _ = holderTx.Rollback(ctx) }) + + collectionID := fmt.Sprintf("poster-lock-timeout-%d", time.Now().UnixNano()) + if _, err := holderTx.Exec(ctx, libraryCollectionPosterAdvisoryLockSQL, collectionID); err != nil { + t.Fatalf("hold poster advisory lock: %v", err) + } + + oldTimeout := libraryCollectionPosterLockTimeout + libraryCollectionPosterLockTimeout = 50 * time.Millisecond + t.Cleanup(func() { libraryCollectionPosterLockTimeout = oldTimeout }) + + repo := NewLibraryCollectionRepository(pool) + started := time.Now() + release, err := repo.AcquirePosterMutationLock(ctx, collectionID) + if release != nil { + release() + t.Fatal("contended poster lock unexpectedly returned a release function") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "55P03" { + t.Fatalf("lock error = %v, want PostgreSQL lock timeout (55P03)", err) + } + if elapsed := time.Since(started); elapsed > 2*time.Second { + t.Fatalf("poster lock timeout took %s, want a bounded failure", elapsed) + } + + if err := holderTx.Rollback(ctx); err != nil { + t.Fatalf("release held poster lock: %v", err) + } + release, err = repo.AcquirePosterMutationLock(ctx, collectionID) + if err != nil { + t.Fatalf("acquire poster lock after timeout cleanup: %v", err) + } + release() +} + func TestLibraryCollectionPosterMutationLocksSerializePerCollection(t *testing.T) { var locks libraryCollectionPosterMutationLocks unlockFirst := locks.lock("collection-1") diff --git a/internal/catalog/search_indexer.go b/internal/catalog/search_indexer.go index 120aa1711..6767cd1b3 100644 --- a/internal/catalog/search_indexer.go +++ b/internal/catalog/search_indexer.go @@ -6,6 +6,8 @@ import ( "fmt" "log/slog" "math" + "slices" + "strconv" "strings" "time" @@ -22,16 +24,19 @@ type SearchIndexProgressReporter interface { } type CatalogSearchIndexSyncStats struct { - Configured bool `json:"configured"` - Skipped bool `json:"skipped"` - Reason string `json:"reason,omitempty"` - Events int `json:"events"` - Upserted int `json:"upserted"` - Deleted int `json:"deleted"` - ActiveIndexUID string `json:"active_index_uid,omitempty"` - DocumentCount int `json:"document_count"` - VectorDocCount int `json:"vector_document_count"` - LastProcessedID int64 `json:"last_processed_event_id,omitempty"` + Configured bool `json:"configured"` + Skipped bool `json:"skipped"` + Reason string `json:"reason,omitempty"` + RebuildAttempted bool `json:"rebuild_attempted"` + Rebuilt bool `json:"rebuilt"` + Events int `json:"events"` + Upserted int `json:"upserted"` + Deleted int `json:"deleted"` + ActiveIndexUID string `json:"active_index_uid,omitempty"` + DocumentCount int `json:"document_count"` + VectorDocCount int `json:"vector_document_count"` + RemovedIndexes int `json:"removed_indexes"` + LastProcessedID int64 `json:"last_processed_event_id,omitempty"` } type CatalogSearchIndexRebuildStats struct { @@ -70,6 +75,7 @@ const catalogSearchExcludeMangaChaptersSQL = `NOT EXISTS (SELECT 1 FROM manga_ch type CatalogSearchIndexer struct { pool *pgxpool.Pool settingsStore SettingsStore + runtime *CatalogSearchSettings events *SearchIndexEventRepository } @@ -81,17 +87,53 @@ func NewCatalogSearchIndexer(pool *pgxpool.Pool, settingsStore SettingsStore) *C } } +// NewCatalogSearchIndexerFromSettings binds scheduled and manual maintenance +// to the same process-lifetime settings snapshot as the serving search +// provider. Catalog search settings are restart-bound; rereading saved values +// on the minute interval could otherwise publish an index the live provider +// does not understand before the server restarts. +func NewCatalogSearchIndexerFromSettings(pool *pgxpool.Pool, settingsStore SettingsStore, settings CatalogSearchSettings) *CatalogSearchIndexer { + settings.IndexTypes = slices.Clone(settings.IndexTypes) + return &CatalogSearchIndexer{ + pool: pool, + settingsStore: settingsStore, + runtime: new(settings), + events: NewSearchIndexEventRepository(pool), + } +} + func (i *CatalogSearchIndexer) ShouldSyncRun(ctx context.Context) (bool, error) { settings, ok, err := i.loadMeilisearchRuntime(ctx) if err != nil || !ok || settings.Provider != SearchProviderMeilisearch { return false, err } + client, err := newMeilisearchClient(settings.MeilisearchURL, settings.MeilisearchAPIKey, settings.Timeout) + if err != nil { + return false, err + } state, err := i.events.GetState(ctx, SearchProviderMeilisearch) if err != nil { return false, err } - if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) { - return false, nil + rebuildRequired, err := catalogSearchIndexRequiresRebuildOnTarget( + ctx, + client, + state, + catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized), + settings.MeilisearchIndex, + ) + if err != nil { + return false, err + } + if rebuildRequired { + if catalogSearchIndexHasNewerSchema(state) { + return false, nil + } + matches, err := i.runtimeMatchesDesiredSettings(ctx, settings) + if err != nil || !matches { + return false, err + } + return true, nil } pending, err := i.events.PendingCount(ctx, SearchProviderMeilisearch) if err != nil { @@ -137,12 +179,46 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn if err != nil { return stats, err } - if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) { - stats.Skipped = true - stats.Reason = "active search index is missing or stale; run rebuild_catalog_search_index" + rebuildRequired, err := catalogSearchIndexRequiresRebuildOnTarget( + ctx, + client, + state, + catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized), + settings.MeilisearchIndex, + ) + if err != nil { + return stats, err + } + if rebuildRequired { + if catalogSearchIndexHasNewerSchema(state) { + stats.Skipped = true + stats.Reason = "active search index was built by a newer server version" + setSearchIndexTaskResult(progress, stats) + reportSearchIndexProgress(progress, 100, "A newer server version owns the active search index") + return stats, nil + } + matches, matchErr := i.runtimeMatchesDesiredSettings(ctx, settings) + if matchErr != nil { + return stats, matchErr + } + if !matches { + stats.Skipped = true + stats.Reason = "saved search settings are waiting for a server restart" + setSearchIndexTaskResult(progress, stats) + reportSearchIndexProgress(progress, 100, "Restart Silo before rebuilding the catalog search index") + return stats, nil + } + stats.RebuildAttempted = true + rebuildStats, rebuildErr := i.rebuildLocked(ctx, progress, settings, client) + stats.Rebuilt = !rebuildStats.Skipped && rebuildErr == nil + stats.Skipped = rebuildStats.Skipped + stats.Reason = rebuildStats.Reason + stats.ActiveIndexUID = rebuildStats.ActiveIndexUID + stats.DocumentCount = rebuildStats.DocumentCount + stats.VectorDocCount = rebuildStats.VectorDocCount + stats.RemovedIndexes = rebuildStats.RemovedIndexes setSearchIndexTaskResult(progress, stats) - reportSearchIndexProgress(progress, 100, "Catalog search index needs a rebuild") - return stats, nil + return stats, rebuildErr } stats.ActiveIndexUID = state.ActiveIndexUID @@ -238,6 +314,55 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn return stats, nil } +func catalogSearchIndexRequiresRebuildOnTarget( + ctx context.Context, + client *meilisearchClient, + state SearchIndexState, + expectedSchemaVersion int, + indexPrefix string, +) (bool, error) { + if catalogSearchIndexStateRequiresRebuild(state, expectedSchemaVersion, indexPrefix) { + return true, nil + } + if client == nil { + return false, fmt.Errorf("checking active Meilisearch index %q: client is not configured", state.ActiveIndexUID) + } + if _, err := client.Stats(ctx, state.ActiveIndexUID); err != nil { + if isMeilisearchIndexNotFound(err) { + return true, nil + } + return false, fmt.Errorf("checking active Meilisearch index %q: %w", state.ActiveIndexUID, err) + } + return false, nil +} + +func catalogSearchIndexStateRequiresRebuild(state SearchIndexState, expectedSchemaVersion int, indexPrefix string) bool { + return state.ActiveIndexUID == "" || + state.SchemaVersion != expectedSchemaVersion || + !catalogSearchIndexUIDBelongsToPrefix(state.ActiveIndexUID, indexPrefix) +} + +func catalogSearchIndexUIDBelongsToPrefix(uid, indexPrefix string) bool { + uid = strings.TrimSpace(uid) + indexPrefix = strings.TrimSpace(indexPrefix) + if uid == "" || indexPrefix == "" { + return false + } + if uid == indexPrefix { + return true + } + suffix, ok := strings.CutPrefix(uid, indexPrefix+"_rebuild_") + if !ok || suffix == "" { + return false + } + timestamp, err := strconv.ParseInt(suffix, 10, 64) + return err == nil && timestamp > 0 +} + +func catalogSearchIndexHasNewerSchema(state SearchIndexState) bool { + return state.ActiveIndexUID != "" && state.SchemaVersion/1_000_000 > SearchMeilisearchSchemaVersion +} + func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndexProgressReporter) (CatalogSearchIndexRebuildStats, error) { stats := CatalogSearchIndexRebuildStats{} settings, client, ok, err := i.loadClient(ctx) @@ -252,6 +377,17 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex return stats, nil } stats.Configured = true + matches, err := i.runtimeMatchesDesiredSettings(ctx, settings) + if err != nil { + return stats, err + } + if !matches { + stats.Skipped = true + stats.Reason = "saved search settings are waiting for a server restart" + setSearchIndexTaskResult(progress, stats) + reportSearchIndexProgress(progress, 100, "Restart Silo before rebuilding the catalog search index") + return stats, nil + } lock, locked, err := pglock.TryAcquire(ctx, i.pool, searchIndexMaintenanceLockID) if err != nil { @@ -270,6 +406,31 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex "component", "catalog", "error", err) } }() + state, err := i.events.GetState(ctx, SearchProviderMeilisearch) + if err != nil { + return stats, err + } + if catalogSearchIndexHasNewerSchema(state) { + stats.Skipped = true + stats.Reason = "active search index was built by a newer server version" + setSearchIndexTaskResult(progress, stats) + reportSearchIndexProgress(progress, 100, "A newer server version owns the active search index") + return stats, nil + } + + return i.rebuildLocked(ctx, progress, settings, client) +} + +// rebuildLocked builds and atomically publishes a replacement index. The +// caller must hold searchIndexMaintenanceLockID so the scheduled sync path and +// manual forced rebuilds cannot race across server nodes. +func (i *CatalogSearchIndexer) rebuildLocked( + ctx context.Context, + progress SearchIndexProgressReporter, + settings CatalogSearchSettings, + client *meilisearchClient, +) (CatalogSearchIndexRebuildStats, error) { + stats := CatalogSearchIndexRebuildStats{Configured: true} rebuildEventHighWater, err := i.events.MaxEventID(ctx, SearchProviderMeilisearch) if err != nil { @@ -522,6 +683,21 @@ func (i *CatalogSearchIndexer) loadClient(ctx context.Context) (CatalogSearchSet } func (i *CatalogSearchIndexer) loadMeilisearchRuntime(ctx context.Context) (CatalogSearchSettings, bool, error) { + if i != nil && i.runtime != nil { + settings := *i.runtime + if i.settingsStore != nil { + var err error + settings, err = LoadCatalogSearchSettings(ctx, i.settingsStore) + if err != nil { + return settings, false, err + } + } + applyCatalogSearchStartupSettings(&settings, *i.runtime) + if settings.Provider != SearchProviderMeilisearch || strings.TrimSpace(settings.MeilisearchURL) == "" { + return settings, false, nil + } + return settings, true, nil + } settings, err := LoadCatalogSearchSettings(ctx, i.settingsStore) if err != nil { return settings, false, err @@ -532,6 +708,39 @@ func (i *CatalogSearchIndexer) loadMeilisearchRuntime(ctx context.Context) (Cata return settings, true, nil } +func (i *CatalogSearchIndexer) runtimeMatchesDesiredSettings(ctx context.Context, runtime CatalogSearchSettings) (bool, error) { + if i == nil || i.runtime == nil || i.settingsStore == nil { + return true, nil + } + desired, err := LoadCatalogSearchSettings(ctx, i.settingsStore) + if err != nil { + return false, err + } + return catalogSearchMaintenanceTargetEqual(runtime, desired), nil +} + +func applyCatalogSearchStartupSettings(settings *CatalogSearchSettings, startup CatalogSearchSettings) { + settings.Provider = startup.Provider + settings.MeilisearchURL = startup.MeilisearchURL + settings.MeilisearchAPIKey = startup.MeilisearchAPIKey + settings.MeilisearchIndex = startup.MeilisearchIndex + settings.IndexTypes = slices.Clone(startup.IndexTypes) + settings.SemanticEnabled = startup.SemanticEnabled + settings.Embedder = startup.Embedder + settings.BinaryQuantized = startup.BinaryQuantized +} + +func catalogSearchMaintenanceTargetEqual(left, right CatalogSearchSettings) bool { + return left.Provider == right.Provider && + left.MeilisearchURL == right.MeilisearchURL && + left.MeilisearchAPIKey == right.MeilisearchAPIKey && + left.MeilisearchIndex == right.MeilisearchIndex && + slices.Equal(left.IndexTypes, right.IndexTypes) && + left.SemanticEnabled == right.SemanticEnabled && + left.Embedder == right.Embedder && + left.BinaryQuantized == right.BinaryQuantized +} + func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings CatalogSearchSettings) error { if settings.Provider != SearchProviderMeilisearch { return nil diff --git a/internal/catalog/search_indexer_test.go b/internal/catalog/search_indexer_test.go index 063047afb..30c76da26 100644 --- a/internal/catalog/search_indexer_test.go +++ b/internal/catalog/search_indexer_test.go @@ -3,6 +3,8 @@ package catalog import ( "context" "fmt" + "net/http" + "net/http/httptest" "os" "slices" "sort" @@ -109,6 +111,184 @@ func TestStaleCatalogSearchIndexUIDs(t *testing.T) { } } +func TestCatalogSearchIndexStateRequiresRebuild(t *testing.T) { + const expected = 123 + tests := []struct { + name string + state SearchIndexState + indexPrefix string + want bool + }{ + {name: "missing index", state: SearchIndexState{SchemaVersion: expected}, indexPrefix: "index", want: true}, + {name: "stale schema", state: SearchIndexState{ActiveIndexUID: "index", SchemaVersion: expected - 1}, indexPrefix: "index", want: true}, + {name: "changed prefix", state: SearchIndexState{ActiveIndexUID: "old_rebuild_123", SchemaVersion: expected}, indexPrefix: "new", want: true}, + {name: "current legacy uid", state: SearchIndexState{ActiveIndexUID: "index", SchemaVersion: expected}, indexPrefix: "index", want: false}, + {name: "current rebuild uid", state: SearchIndexState{ActiveIndexUID: "index_rebuild_123", SchemaVersion: expected}, indexPrefix: "index", want: false}, + {name: "similar prefix", state: SearchIndexState{ActiveIndexUID: "index_rebuild_rebuild_123", SchemaVersion: expected}, indexPrefix: "index", want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := catalogSearchIndexStateRequiresRebuild(tc.state, expected, tc.indexPrefix); got != tc.want { + t.Fatalf("catalogSearchIndexStateRequiresRebuild() = %t, want %t", got, tc.want) + } + }) + } +} + +func TestCatalogSearchIndexRequiresRebuildWhenActiveUIDIsMissingFromTarget(t *testing.T) { + const ( + expectedSchema = 123 + indexPrefix = "index" + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/indexes/index_rebuild_123/stats": + _, _ = w.Write([]byte(`{"numberOfDocuments":42}`)) + case "/indexes/index_rebuild_456/stats": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"Index not found.","code":"index_not_found"}`)) + case "/indexes/index_rebuild_789/stats": + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message":"Unavailable.","code":"internal"}`)) + default: + t.Errorf("unexpected Meilisearch request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := newMeilisearchClient(server.URL, "", time.Second) + if err != nil { + t.Fatalf("newMeilisearchClient() error = %v", err) + } + tests := []struct { + name string + uid string + want bool + wantErr bool + }{ + {name: "active uid exists", uid: "index_rebuild_123", want: false}, + {name: "active uid missing after target move", uid: "index_rebuild_456", want: true}, + {name: "target temporarily unavailable", uid: "index_rebuild_789", wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := catalogSearchIndexRequiresRebuildOnTarget( + t.Context(), + client, + SearchIndexState{ActiveIndexUID: tc.uid, SchemaVersion: expectedSchema}, + expectedSchema, + indexPrefix, + ) + if (err != nil) != tc.wantErr { + t.Fatalf("catalogSearchIndexRequiresRebuildOnTarget() error = %v", err) + } + if got != tc.want { + t.Fatalf("catalogSearchIndexRequiresRebuildOnTarget() = %t, want %t", got, tc.want) + } + }) + } +} + +func TestCatalogSearchIndexerFreezesRuntimeSettings(t *testing.T) { + settings := DefaultCatalogSearchSettings() + settings.Provider = SearchProviderMeilisearch + settings.MeilisearchURL = "http://meilisearch-at-startup:7700" + settings.IndexTypes = []string{"movie"} + indexer := NewCatalogSearchIndexerFromSettings(nil, nil, settings) + + settings.MeilisearchURL = "http://changed-after-startup:7700" + settings.IndexTypes[0] = "series" + got, configured, err := indexer.loadMeilisearchRuntime(t.Context()) + if err != nil { + t.Fatalf("loadMeilisearchRuntime() error = %v", err) + } + if !configured { + t.Fatal("loadMeilisearchRuntime() configured = false, want true") + } + if got.MeilisearchURL != "http://meilisearch-at-startup:7700" { + t.Fatalf("MeilisearchURL = %q, want startup value", got.MeilisearchURL) + } + if len(got.IndexTypes) != 1 || got.IndexTypes[0] != "movie" { + t.Fatalf("IndexTypes = %#v, want startup copy", got.IndexTypes) + } +} + +func TestCatalogSearchIndexerWaitsForRestartBeforeUsingSavedSchemaSettings(t *testing.T) { + startup := DefaultCatalogSearchSettings() + startup.Provider = SearchProviderMeilisearch + startup.MeilisearchURL = "http://meilisearch:7700" + store := &memSettings{m: map[string]string{ + SearchSettingProvider: SearchProviderMeilisearch, + SearchSettingMeilisearchURL: startup.MeilisearchURL, + SearchSettingMeilisearchSemanticEnabled: "true", + SearchSettingMeilisearchRebuildBatchSize: "17", + }} + indexer := NewCatalogSearchIndexerFromSettings(nil, store, startup) + + runtime, configured, err := indexer.loadMeilisearchRuntime(t.Context()) + if err != nil { + t.Fatalf("loadMeilisearchRuntime() error = %v", err) + } + if !configured { + t.Fatal("loadMeilisearchRuntime() configured = false, want true") + } + if runtime.SemanticEnabled { + t.Fatal("saved semantic enablement became active before restart") + } + if runtime.RebuildBatchSize != 17 { + t.Fatalf("RebuildBatchSize = %d, want hot-reloaded 17", runtime.RebuildBatchSize) + } + matches, err := indexer.runtimeMatchesDesiredSettings(t.Context(), runtime) + if err != nil { + t.Fatalf("runtimeMatchesDesiredSettings() error = %v", err) + } + if matches { + t.Fatal("saved schema settings should fence rebuilds until restart") + } +} + +func TestCatalogSearchMaintenanceTargetEqual(t *testing.T) { + startup := DefaultCatalogSearchSettings() + startup.Provider = SearchProviderMeilisearch + startup.MeilisearchURL = "http://meilisearch:7700" + startup.IndexTypes = []string{"movie", "series"} + + tuningOnly := startup + tuningOnly.SyncBatchSize++ + tuningOnly.RebuildBatchSize++ + tuningOnly.RebuildQueueDepth++ + if !catalogSearchMaintenanceTargetEqual(startup, tuningOnly) { + t.Fatal("hot-reloadable maintenance tuning must not require a restart fence") + } + + desired := startup + desired.SemanticEnabled = true + if catalogSearchMaintenanceTargetEqual(startup, desired) { + t.Fatal("semantic setting change must wait for the startup configuration") + } + + desired = startup + desired.IndexTypes = []string{"movie"} + if catalogSearchMaintenanceTargetEqual(startup, desired) { + t.Fatal("index scope change must wait for the startup configuration") + } +} + +func TestCatalogSearchIndexHasNewerSchema(t *testing.T) { + newer := SearchIndexState{ + ActiveIndexUID: "new-index", + SchemaVersion: (SearchMeilisearchSchemaVersion + 1) * 1_000_000, + } + if !catalogSearchIndexHasNewerSchema(newer) { + t.Fatal("newer schema should fence an older server from rebuilding") + } + if catalogSearchIndexHasNewerSchema(SearchIndexState{ActiveIndexUID: "index", SchemaVersion: SearchMeilisearchSchemaVersion * 1_000_000}) { + t.Fatal("current schema generation should not be treated as newer") + } +} + func TestRebuildIndexingPercent(t *testing.T) { if got := rebuildIndexingPercent(0, 0); got != 50 { t.Fatalf("unknown total should report midpoint, got %v", got) diff --git a/internal/catalog/search_meilisearch_provider.go b/internal/catalog/search_meilisearch_provider.go index 9c5c552d5..1e6788d3a 100644 --- a/internal/catalog/search_meilisearch_provider.go +++ b/internal/catalog/search_meilisearch_provider.go @@ -189,16 +189,54 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc if strings.TrimSpace(state.ActiveIndexUID) == "" { return p.fallbackSearch(ctx, req, "meilisearch index has not been built") } - if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled, p.config.BinaryQuantized) { + compatibility := catalogSearchMeilisearchIndexCompatibility( + state.SchemaVersion, + p.config.Embedder, + p.config.IndexTypes, + p.config.SemanticEnabled, + p.config.BinaryQuantized, + ) + if compatibility == catalogSearchIndexIncompatible { return p.fallbackSearch(ctx, req, "meilisearch index schema mismatch") } - result, err := p.searchMeilisearch(ctx, req, state.ActiveIndexUID) + keywordOnly := compatibility == catalogSearchIndexKeywordOnly + result, err := p.searchMeilisearch(ctx, req, state.ActiveIndexUID, keywordOnly) if err != nil { // The cached state may point at an index a rebuild just swapped away - // and deleted; drop it so the next request refetches instead of - // failing for the rest of the TTL. + // and deleted. Refetch and retry Meilisearch once so every API node's + // first post-swap request does not pay the much slower Postgres fallback. p.invalidateIndexState() + if isMeilisearchIndexNotFound(err) { + refreshedState, refreshedPending, stateErr := p.indexState(ctx) + if stateErr == nil && refreshedState.ActiveIndexUID != "" && refreshedState.ActiveIndexUID != state.ActiveIndexUID { + refreshedCompatibility := catalogSearchMeilisearchIndexCompatibility( + refreshedState.SchemaVersion, + p.config.Embedder, + p.config.IndexTypes, + p.config.SemanticEnabled, + p.config.BinaryQuantized, + ) + if refreshedCompatibility != catalogSearchIndexIncompatible { + result, retryErr := p.searchMeilisearch( + ctx, + req, + refreshedState.ActiveIndexUID, + refreshedCompatibility == catalogSearchIndexKeywordOnly, + ) + if retryErr == nil { + result.IndexPendingEvents = refreshedPending + if result.FallbackReason != "" { + p.markFallback(result.FallbackReason) + } else { + p.clearFallback() + } + return result, nil + } + err = retryErr + } + } + } if p.shouldTripCircuit(err) { p.tripCircuit(err) } @@ -213,6 +251,12 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc return result, nil } +func isMeilisearchIndexNotFound(err error) bool { + httpErr, ok := errors.AsType[*meilisearchHTTPError](err) + return ok && + (httpErr.StatusCode == http.StatusNotFound || strings.EqualFold(strings.TrimSpace(httpErr.Code), "index_not_found")) +} + func (p *MeilisearchSearchProvider) indexCoversRequest(itemTypes []string) bool { if p == nil || len(p.config.IndexTypes) == 0 { return true @@ -269,7 +313,7 @@ func (p *MeilisearchSearchProvider) invalidateIndexState() { p.stateMu.Unlock() } -func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req CatalogSearchRequest, indexUID string) (*CatalogSearchResult, error) { +func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req CatalogSearchRequest, indexUID string, keywordOnly bool) (*CatalogSearchResult, error) { target := req.Offset + req.Limit + 1 if target <= 0 { target = 1 @@ -284,7 +328,14 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C scanned := 0 estimatedTotalHits := 0 exhausted := false - baseSearchReq, semanticFallback := p.buildMeilisearchSearchRequest(ctx, req) + var baseSearchReq meilisearchSearchRequest + var semanticFallback string + if keywordOnly { + baseSearchReq = p.buildMeilisearchKeywordSearchRequest(req) + semanticFallback = "search index rebuild required; using Meilisearch keyword search" + } else { + baseSearchReq, semanticFallback = p.buildMeilisearchSearchRequest(ctx, req) + } useFederation := baseSearchReq.Hybrid != nil && searchRequestMixesEpisodeAndMedia(req.ItemTypes) if useFederation && p.isFederationUnsupported() { semanticFallback = "meilisearch federation unsupported; using keyword search" @@ -416,13 +467,7 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co AttributesToRetrieve: []string{"content_id"}, }, "" } - searchReq := meilisearchSearchRequest{ - Query: strings.TrimSpace(req.Query), - Filter: meilisearchSearchFilter(req.ItemTypes, req.Access), - AttributesToRetrieve: []string{"content_id"}, - AttributesToSearchOn: p.attributesToSearchOnForRequest(req), - MatchingStrategy: p.matchingStrategyForRequest(req), - } + searchReq := p.buildMeilisearchKeywordSearchRequest(req) if !p.shouldUseSemanticSearch(req) { return searchReq, "" } @@ -449,6 +494,16 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co return searchReq, "" } +func (p *MeilisearchSearchProvider) buildMeilisearchKeywordSearchRequest(req CatalogSearchRequest) meilisearchSearchRequest { + return meilisearchSearchRequest{ + Query: strings.TrimSpace(req.Query), + Filter: meilisearchSearchFilter(req.ItemTypes, req.Access), + AttributesToRetrieve: []string{"content_id"}, + AttributesToSearchOn: p.attributesToSearchOnForRequest(req), + MatchingStrategy: p.matchingStrategyForRequest(req), + } +} + func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchRequest) bool { if p == nil || !p.config.SemanticEnabled { return false diff --git a/internal/catalog/search_postgres_mixed.go b/internal/catalog/search_postgres_mixed.go index 79ea8581b..8b46615d3 100644 --- a/internal/catalog/search_postgres_mixed.go +++ b/internal/catalog/search_postgres_mixed.go @@ -78,7 +78,7 @@ const mediaSearchTitleVector = `( const mediaSearchOverviewVector = `to_tsvector('english', COALESCE(mi.overview, ''))` const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, year_match DESC, - phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, + phrase_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC` // buildMixedSearchSQLFromParsed builds one ranked candidate set from the two @@ -382,7 +382,6 @@ func buildMixedSearchAliasScoresCTE(exactIdx int, exactShortTitle, leadingShortT mia.content_id, MAX(CASE WHEN mia.normalized_title = $%d THEN 1 ELSE 0 END) AS exact_title_match, MAX(CASE WHEN mia.normalized_title LIKE '%%' || $%d || '%%' THEN 1 ELSE 0 END) AS contiguous_title_match, - 0::real AS title_rank, MAX(CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END) AS title_prefix_rank FROM media_item_aliases mia WHERE %s @@ -419,7 +418,6 @@ func buildMixedSearchCandidateBranch( contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx)) } prefixQuery := `to_tsquery('simple', $2)` - titleRankExpr := "0::real" prefixRankExpr := fmt.Sprintf("ts_rank_cd(%s, %s)", titleVector, prefixQuery) if aliasArms != nil { // Alias exact/contiguous arms are hashed subplans in the CASE select @@ -445,7 +443,6 @@ func buildMixedSearchCandidateBranch( CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match, CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match, CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match, - %s AS title_rank, CASE WHEN $2 <> '' THEN %s ELSE 0 END AS title_prefix_rank, %s AS overview_rank, CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank @@ -455,7 +452,6 @@ func buildMixedSearchCandidateBranch( exactIdx, strings.Join(exactArms, " OR "), exactIdx, strings.Join(contiguousArms, " OR "), yearIdx, yearExpr, yearIdx, - titleRankExpr, prefixRankExpr, overviewRankExpr, phraseIdx, titleVector, phraseIdx, diff --git a/internal/catalog/search_provider.go b/internal/catalog/search_provider.go index 41c87e364..c78b51566 100644 --- a/internal/catalog/search_provider.go +++ b/internal/catalog/search_provider.go @@ -357,6 +357,8 @@ func normalizeCatalogSearchItemTypes(itemTypes []string) []string { type CatalogSearchRuntimeStatus struct { ConfiguredProvider string `json:"configured_provider"` ActiveProvider string `json:"active_provider"` + Degraded bool `json:"degraded"` + DegradedReason string `json:"degraded_reason,omitempty"` Meilisearch CatalogSearchMeiliStatus `json:"meilisearch"` Index CatalogSearchIndexStateStatus `json:"index"` Tasks []CatalogSearchTaskLink `json:"tasks"` @@ -419,6 +421,7 @@ type CatalogSearchIndexStateStatus struct { ActiveIndexUID string `json:"active_index_uid"` SchemaVersion int `json:"schema_version"` ExpectedSchemaVersion int `json:"expected_schema_version"` + RebuildRequired bool `json:"rebuild_required"` DocumentCount int `json:"document_count"` VectorDocumentCount int `json:"vector_document_count"` PendingEvents int `json:"pending_events"` @@ -462,8 +465,10 @@ func catalogSearchMeilisearchEmbedderSettings(embedder string, binaryQuantized b // attachDocumentVectors omits _vectors entirely when semantic is off; toggling it // on must therefore make the existing (vector-less) index look stale so it is // rebuilt rather than serving silently degraded hybrid ranking. A mismatch forces -// a rebuild (SyncOutbox skips, the provider falls back to keyword) and surfaces as -// an ExpectedSchemaVersion divergence in the admin status. +// a rebuild (SyncOutbox skips) and surfaces as an ExpectedSchemaVersion divergence +// in the admin status. An index whose hash proves that only semantic/vector +// settings changed remains safe for keyword-only Meilisearch queries while the +// replacement is built; all other mismatches fall back to Postgres. func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, semanticEnabled, binaryQuantized bool) int { embedder, err := NormalizeCatalogSearchEmbedderName(embedder) if err != nil { @@ -489,3 +494,41 @@ func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, _, _ = h.Write([]byte(identity)) return SearchMeilisearchSchemaVersion*1_000_000 + int(h.Sum32()%1_000_000) } + +type catalogSearchIndexCompatibility uint8 + +const ( + catalogSearchIndexIncompatible catalogSearchIndexCompatibility = iota + catalogSearchIndexKeywordOnly + catalogSearchIndexCurrent +) + +// catalogSearchMeilisearchIndexCompatibility classifies an active index using +// only schema identities Silo itself can have published. If the active hash +// matches the same embedder, vector dimensions, and media scope with a different +// semantic/binary setting, its document fields and filters are still valid for +// keyword search. Unknown hashes are not trusted because they may represent a +// different indexed media scope or document shape. +func catalogSearchMeilisearchIndexCompatibility( + activeVersion int, + embedder string, + itemTypes []string, + semanticEnabled bool, + binaryQuantized bool, +) catalogSearchIndexCompatibility { + expected := catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled, binaryQuantized) + if activeVersion == expected { + return catalogSearchIndexCurrent + } + compatibleVersions := [...]int{ + catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, false, false), + catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, true, false), + catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, true, true), + } + for _, version := range compatibleVersions { + if activeVersion == version { + return catalogSearchIndexKeywordOnly + } + } + return catalogSearchIndexIncompatible +} diff --git a/internal/catalog/search_provider_test.go b/internal/catalog/search_provider_test.go index bc7c26dd1..1908b935b 100644 --- a/internal/catalog/search_provider_test.go +++ b/internal/catalog/search_provider_test.go @@ -736,14 +736,62 @@ func TestMeilisearchProviderInvalidatesStateCacheOnSearchFailure(t *testing.T) { t.Fatalf("Search %d should surface the fallback error in this setup", i) } } - if store.getStateCalls != 2 { - t.Fatalf("getStateCalls = %d, want 2 (cache invalidated after failed search)", store.getStateCalls) + if store.getStateCalls != 3 { + t.Fatalf("getStateCalls = %d, want 3 (each missing-index failure refreshes state immediately)", store.getStateCalls) } if reason, blocked := provider.circuitBlocked(time.Now()); blocked { t.Fatalf("HTTP 404 must not trip the circuit, got open circuit: %s", reason) } } +func TestMeilisearchProviderRetriesNewActiveIndexAfterSwap(t *testing.T) { + var paths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/indexes/old-index/search" { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"Index old-index not found.","code":"index_not_found"}`)) + return + } + _, _ = w.Write([]byte(`{"hits":[],"estimatedTotalHits":0}`)) + })) + defer server.Close() + + client, err := newMeilisearchClient(server.URL, "", time.Second) + if err != nil { + t.Fatalf("newMeilisearchClient: %v", err) + } + schemaVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false) + store := &countingMeilisearchIndexStateStore{state: SearchIndexState{ + ActiveIndexUID: "new-index", + SchemaVersion: schemaVersion, + }} + provider := &MeilisearchSearchProvider{ + stateRepo: store, + fallback: &PostgresSearchProvider{}, + client: client, + config: MeilisearchProviderConfig{MatchingStrategy: DefaultMeilisearchMatchingStrategy, Embedder: DefaultMeilisearchEmbedder}, + cachedState: SearchIndexState{ActiveIndexUID: "old-index", SchemaVersion: schemaVersion}, + stateCachedAt: time.Now(), + } + + result, err := provider.Search(t.Context(), CatalogSearchRequest{Query: "space opera", Limit: 10}) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + wantPaths := []string{"/indexes/old-index/search", "/indexes/new-index/search"} + if !reflect.DeepEqual(paths, wantPaths) { + t.Fatalf("Meilisearch paths = %#v, want %#v", paths, wantPaths) + } + if store.getStateCalls != 1 { + t.Fatalf("state refresh calls = %d, want 1", store.getStateCalls) + } + if result.Provider != SearchProviderMeilisearch { + t.Fatalf("provider = %q, want Meilisearch", result.Provider) + } +} + func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T) { requests := 0 var gotMethod, gotPath string @@ -801,6 +849,59 @@ func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T) } } +func TestMeilisearchProviderUsesStaleVectorlessIndexForKeywordSearch(t *testing.T) { + var searchRequest meilisearchSearchRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/indexes/search-index/search" { + t.Fatalf("unexpected Meilisearch request %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&searchRequest); err != nil { + t.Fatalf("decode search request: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"hits":[],"estimatedTotalHits":0}`)) + })) + defer server.Close() + + client, err := newMeilisearchClient(server.URL, "", time.Second) + if err != nil { + t.Fatalf("newMeilisearchClient: %v", err) + } + vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.1, 0.2}} + provider := &MeilisearchSearchProvider{ + stateRepo: fakeMeilisearchIndexStateStore{state: SearchIndexState{ + ActiveIndexUID: "search-index", + SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false), + }}, + fallback: &PostgresSearchProvider{}, + client: client, + config: MeilisearchProviderConfig{ + MatchingStrategy: DefaultMeilisearchMatchingStrategy, + Embedder: DefaultMeilisearchEmbedder, + SemanticEnabled: true, + SemanticRatio: 0.5, + Vectorizer: vectorizer, + }, + } + + result, err := provider.Search(t.Context(), CatalogSearchRequest{Query: "space opera", Limit: 10}) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if vectorizer.calls != 0 { + t.Fatalf("vectorizer calls = %d, want 0 while the semantic index rebuilds", vectorizer.calls) + } + if searchRequest.Hybrid != nil || len(searchRequest.Vector) != 0 { + t.Fatalf("stale index request used semantic search: hybrid=%#v vector=%v", searchRequest.Hybrid, searchRequest.Vector) + } + if result.Provider != SearchProviderMeilisearch || result.Mode != "keyword" || result.SemanticUsed { + t.Fatalf("result diagnostics = provider %q, mode %q, semantic %t", result.Provider, result.Mode, result.SemanticUsed) + } + if !strings.Contains(result.FallbackReason, "rebuild required") { + t.Fatalf("fallback reason = %q, want rebuild diagnostic", result.FallbackReason) + } +} + func TestMeilisearchProviderFallsBackWhenScopedIndexCannotSatisfyRequest(t *testing.T) { provider := &MeilisearchSearchProvider{ config: MeilisearchProviderConfig{ @@ -872,6 +973,37 @@ func TestMeilisearchSchemaVersionChangesWithSemanticEnabled(t *testing.T) { } } +func TestMeilisearchIndexCompatibility(t *testing.T) { + itemTypes := []string{"movie", "series"} + semanticOff := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, itemTypes, false, false) + semanticOn := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, itemTypes, true, false) + tests := []struct { + name string + activeVersion int + embedder string + itemTypes []string + semantic bool + binary bool + want catalogSearchIndexCompatibility + }{ + {name: "current", activeVersion: semanticOn, embedder: DefaultMeilisearchEmbedder, itemTypes: itemTypes, semantic: true, want: catalogSearchIndexCurrent}, + {name: "semantic enablement", activeVersion: semanticOff, embedder: DefaultMeilisearchEmbedder, itemTypes: itemTypes, semantic: true, want: catalogSearchIndexKeywordOnly}, + {name: "semantic disablement", activeVersion: semanticOn, embedder: DefaultMeilisearchEmbedder, itemTypes: itemTypes, want: catalogSearchIndexKeywordOnly}, + {name: "binary setting change", activeVersion: semanticOn, embedder: DefaultMeilisearchEmbedder, itemTypes: itemTypes, semantic: true, binary: true, want: catalogSearchIndexKeywordOnly}, + {name: "different embedder", activeVersion: semanticOff, embedder: "other_embedder", itemTypes: itemTypes, semantic: true, want: catalogSearchIndexIncompatible}, + {name: "different media scope", activeVersion: semanticOff, embedder: DefaultMeilisearchEmbedder, itemTypes: []string{"movie"}, semantic: true, want: catalogSearchIndexIncompatible}, + {name: "unknown schema", activeVersion: 0, embedder: DefaultMeilisearchEmbedder, itemTypes: itemTypes, semantic: true, want: catalogSearchIndexIncompatible}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := catalogSearchMeilisearchIndexCompatibility(tc.activeVersion, tc.embedder, tc.itemTypes, tc.semantic, tc.binary) + if got != tc.want { + t.Fatalf("compatibility = %v, want %v", got, tc.want) + } + }) + } +} + func TestPostgresSearchProviderNilRepoStillErrors(t *testing.T) { provider := NewPostgresSearchProvider(nil) if _, err := provider.Search(context.Background(), CatalogSearchRequest{}); err == nil { diff --git a/internal/catalog/search_service.go b/internal/catalog/search_service.go index 77ee064d5..963995e1a 100644 --- a/internal/catalog/search_service.go +++ b/internal/catalog/search_service.go @@ -158,21 +158,55 @@ func (s *CatalogSearchService) Status(ctx context.Context) CatalogSearchRuntimeS }, } if s != nil { - if _, ok := s.provider.(*MeilisearchSearchProvider); ok { - status.ActiveProvider = SearchProviderMeilisearch - } + _, meilisearchProviderActive := s.provider.(*MeilisearchSearchProvider) if s.meili != nil { status.Meilisearch = s.meili.Status() } + stateAvailable := false if s.state != nil { state, err := s.state.GetState(ctx, SearchProviderMeilisearch) if err == nil { + stateAvailable = true status.Index.ActiveIndexUID = state.ActiveIndexUID status.Index.SchemaVersion = state.SchemaVersion status.Index.DocumentCount = state.DocumentCount status.Index.LastRebuildAt = state.LastRebuildAt status.Index.LastSyncAt = state.LastSyncAt status.Index.LastProcessedEventID = state.LastProcessedEventID + if meilisearchProviderActive { + switch { + case state.ActiveIndexUID == "": + status.Degraded = true + status.DegradedReason = "Meilisearch index has not been built; using Postgres search" + status.Index.RebuildRequired = true + case catalogSearchMeilisearchIndexCompatibility( + state.SchemaVersion, + settings.Embedder, + settings.IndexTypes, + settings.SemanticEnabled, + settings.BinaryQuantized, + ) == catalogSearchIndexCurrent: + status.ActiveProvider = SearchProviderMeilisearch + case catalogSearchMeilisearchIndexCompatibility( + state.SchemaVersion, + settings.Embedder, + settings.IndexTypes, + settings.SemanticEnabled, + settings.BinaryQuantized, + ) == catalogSearchIndexKeywordOnly: + status.ActiveProvider = SearchProviderMeilisearch + status.Degraded = true + status.DegradedReason = "Search index rebuild required; using Meilisearch keyword search" + status.Index.RebuildRequired = true + case catalogSearchIndexHasNewerSchema(state): + status.Degraded = true + status.DegradedReason = "A newer server version owns the Meilisearch index; using Postgres search on this node" + default: + status.Degraded = true + status.DegradedReason = "Meilisearch index schema mismatch; using Postgres search" + status.Index.RebuildRequired = true + } + } } if pending, err := s.state.PendingCount(ctx, SearchProviderMeilisearch); err == nil { status.Index.PendingEvents = pending @@ -181,6 +215,19 @@ func (s *CatalogSearchService) Status(ctx context.Context) CatalogSearchRuntimeS status.Index.DeadLetteredEvents = deadLettered } } + if meilisearchProviderActive && !stateAvailable { + status.Degraded = true + status.DegradedReason = "Meilisearch index state is unavailable; using Postgres search" + } + if meilisearchProviderActive && !status.Meilisearch.Healthy { + status.ActiveProvider = SearchProviderPostgres + status.Degraded = true + if status.Meilisearch.CircuitReason != "" { + status.DegradedReason = "Meilisearch is unavailable; using Postgres search: " + status.Meilisearch.CircuitReason + } else { + status.DegradedReason = "Meilisearch is unavailable; using Postgres search" + } + } if s.itemRepo != nil && s.itemRepo.pool != nil { if vectorCount, err := countCatalogSearchVectorDocuments(ctx, s.itemRepo.pool, settings.IndexTypes, ""); err == nil { status.Index.VectorDocumentCount = vectorCount @@ -196,7 +243,14 @@ func (s *CatalogSearchService) Status(ctx context.Context) CatalogSearchRuntimeS } else { status.Semantic = CatalogSearchSemanticStatus{Ready: false, DisabledReason: "semantic search disabled"} } - if s.meili != nil { + if status.Index.RebuildRequired { + status.Semantic.Ready = false + status.Semantic.DisabledReason = "search index rebuild required" + status.Semantic.Capability = CatalogSearchSemanticCapability{ + OK: false, + Reason: "search index rebuild required", + } + } else if s.meili != nil { status.Semantic.Capability = s.meili.SemanticCapability(ctx) } } diff --git a/internal/chapterthumbs/extractor.go b/internal/chapterthumbs/extractor.go index b023dc733..6d7b5859d 100644 --- a/internal/chapterthumbs/extractor.go +++ b/internal/chapterthumbs/extractor.go @@ -27,7 +27,7 @@ type FrameExtractOptions struct { RunFunc func(ctx context.Context, ffmpegPath string, args []string) ([]byte, error) softwareToneMapResolver *softwareToneMapFilterResolver - resolveHWAccel func(ctx context.Context, hwAccel, ffmpegPath string) string + resolveHWAccel func(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string } const ( @@ -189,7 +189,7 @@ func ExtractFrame(ctx context.Context, opts FrameExtractOptions) ([]byte, string softwareToneMapResolver: softwareToneMapResolver, } - resolvedAccel := resolveHWAccel(ctx, opts.HWAccel, ffmpegPath) + resolvedAccel := resolveHWAccel(ctx, opts.HWAccel, ffmpegPath, opts.HWDevice) if supportsHardwareFrameExtract(resolvedAccel) { softwareToneMapFilter := "" if resolvedAccel == hwAccelVideoToolbox && opts.ToneMap { @@ -419,9 +419,8 @@ func buildFrameExtractArgs( if hwDevice == "" { return nil, fmt.Errorf("qsv requires a render device") } + args = append(args, tonemap.QSVInitDeviceArgs(hwDevice)...) args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=va:%s,driver=iHD,kernel_driver=i915,vendor_id=0x8086", hwDevice), - "-init_hw_device", "qsv=qs@va", "-filter_hw_device", "va", "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", @@ -430,8 +429,8 @@ func buildFrameExtractArgs( if hwDevice == "" { return nil, fmt.Errorf("vaapi requires a render device") } + args = append(args, tonemap.VAAPIInitDeviceArgs("hw", hwDevice)...) args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=hw:%s", hwDevice), "-filter_hw_device", "hw", "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", diff --git a/internal/chapterthumbs/extractor_test.go b/internal/chapterthumbs/extractor_test.go index b3c73edf8..7dd1d3421 100644 --- a/internal/chapterthumbs/extractor_test.go +++ b/internal/chapterthumbs/extractor_test.go @@ -250,7 +250,7 @@ func TestExtractFramePassesCallerContextToHWAccelResolution(t *testing.T) { InputPath: "/media/movie.mkv", FFmpegPath: "/test/ffmpeg", HWAccel: "auto", - resolveHWAccel: func(gotCtx context.Context, hwAccel, ffmpegPath string) string { + resolveHWAccel: func(gotCtx context.Context, hwAccel, ffmpegPath, _ string) string { called = true if gotCtx != ctx { t.Fatal("hardware probe did not receive the extraction context") diff --git a/internal/chapterthumbs/remote.go b/internal/chapterthumbs/remote.go index ba6e2c4b3..a3b581463 100644 --- a/internal/chapterthumbs/remote.go +++ b/internal/chapterthumbs/remote.go @@ -70,7 +70,7 @@ func (e *httpRemoteFrameExtractor) ExtractFrame( requestCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, node.URL+"/chapter-thumbnails/extract", bytes.NewReader(body)) + httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(node.URL, "/chapter-thumbnails/extract"), bytes.NewReader(body)) if err != nil { return nil, chapterThumbnailNodeUnavailableReason, fmt.Errorf("chapter thumbnail remote extract: build request: %w", err) } diff --git a/internal/chapterthumbs/service.go b/internal/chapterthumbs/service.go index e1d7cc12e..ed4abf74c 100644 --- a/internal/chapterthumbs/service.go +++ b/internal/chapterthumbs/service.go @@ -37,6 +37,14 @@ const ( chapterThumbnailHDRPolicyDisabled = "disabled" chapterThumbnailHDRPolicyBestEffort = "best_effort" chapterThumbnailSoftwareToneMapSetting = "playback.chapter_thumbnail_software_tone_map_enabled" + + // Hardware acceleration is read per extraction from these keys, not frozen + // at startup, so an admin change applies to the next chapter thumbnail. + // playbackHWAccelDefault mirrors the config loader's default for an unset + // row (internal/config/admin_settings.go). + playbackHWAccelSetting = "playback.hw_accel" + playbackHWDeviceSetting = "playback.hw_device" + playbackHWAccelDefault = "auto" ) var chapterThumbnailRetrySchedule = []time.Duration{ @@ -94,17 +102,35 @@ type ChapterThumbnailRequest struct { } type Service struct { - fileRepo FileRepository - folderRepo FolderRepository - probeEnsurer ProbeEnsurer - settings SettingsReader - store ObjectStore - notifier ThumbnailNotifier - ffmpegPath string - hwAccel string - hwDevice string - hwResolveOnce sync.Once - resolvedHWAccel string + fileRepo FileRepository + folderRepo FolderRepository + probeEnsurer ProbeEnsurer + settings SettingsReader + store ObjectStore + notifier ThumbnailNotifier + ffmpegPath string + // hwAccel and hwDevice hold the playback.hw_accel / playback.hw_device + // values captured when the service was built. They are only the fallback: + // resolveHWConfig re-reads both settings per extraction so an admin who + // changes hardware acceleration does not have to restart the server for + // chapter-thumbnail extraction to follow. + hwAccel string + hwDevice string + + // hwMu guards the resolved-accelerator memo below. Resolving "auto" execs + // an FFmpeg capability probe and logs the verdict, so the result is cached + // against the configured values that produced it and recomputed only when + // they actually change. + // + // Both values, not just the backend: the walk is over the configured device + // set, so a device edit changes which backends have candidates to verify and + // therefore what "auto" resolves to. Keying on the backend alone would hold + // a verdict taken against the old device list. + hwMu sync.Mutex + hwResolved bool + hwResolvedFrom string + hwResolvedDevice string + resolvedHWAccel string notifyNormal chan struct{} notifyPriority chan struct{} @@ -207,7 +233,9 @@ func (s *Service) Start(ctx context.Context) { return } - resolvedAccel, resolvedDevice := s.resolveHWConfig() + // Logged for the boot record only: both values are re-read per extraction, + // so a later settings change is honored without a restart. + resolvedAccel, resolvedDevice := s.resolveHWConfig(ctx) slog.InfoContext(ctx, "chapter thumbnail service started", "component", "chapterthumbs", "workers", @@ -650,7 +678,7 @@ func (s *Service) extractFrameLocal( toneMap bool, allowSoftwareToneMap bool, ) ([]byte, string, error) { - resolvedAccel, resolvedDevice := s.resolveHWConfig() + resolvedAccel, resolvedDevice := s.resolveHWConfig(ctx) return ExtractFrame(ctx, FrameExtractOptions{ InputPath: inputPath, SeekSeconds: seekSeconds, @@ -663,13 +691,65 @@ func (s *Service) extractFrameLocal( }) } -func (s *Service) resolveHWConfig() (string, string) { - s.hwResolveOnce.Do(func() { - s.resolvedHWAccel = playback.ResolveHWAccelWithFFmpeg(s.hwAccel, s.ffmpegPath) - }) +// resolveHWConfig returns the accelerator and device this extraction should +// use. Both come from the live settings repo rather than from a value frozen at +// startup, which is what lets playback.hw_accel / playback.hw_device take +// effect without a server restart. +func (s *Service) resolveHWConfig(ctx context.Context) (string, string) { + configuredAccel, configuredDevice := s.configuredHWConfig(ctx) + + s.hwMu.Lock() + defer s.hwMu.Unlock() + if !s.hwResolved || s.hwResolvedFrom != configuredAccel || s.hwResolvedDevice != configuredDevice { + // The device set is an input to the walk, not just to execution: it is + // what decides which backends have candidates to probe. + s.resolvedHWAccel = playback.ResolveHWAccelWithFFmpeg(configuredAccel, s.ffmpegPath, configuredDevice) + s.hwResolvedFrom = configuredAccel + s.hwResolvedDevice = configuredDevice + s.hwResolved = true + } // The configured device value passes through raw: ExtractFrame resolves it // (multi-device balancing, empty-value auto-detection) per extraction. - return s.resolvedHWAccel, s.hwDevice + return s.resolvedHWAccel, configuredDevice +} + +// configuredHWConfig reads playback.hw_accel / playback.hw_device from the +// settings repo, mirroring how the config loader defaults them. A settings repo +// that is absent (test doubles) or failing falls back to the values captured at +// construction, so a database blip keeps the boot configuration instead of +// silently dropping extraction to software. +func (s *Service) configuredHWConfig(ctx context.Context) (string, string) { + if s == nil { + return "", "" + } + accel := s.hwAccel + if value, ok := s.readSetting(ctx, playbackHWAccelSetting); ok { + accel = value + if accel == "" { + accel = playbackHWAccelDefault + } + } + device := s.hwDevice + if value, ok := s.readSetting(ctx, playbackHWDeviceSetting); ok { + // An empty device is a meaningful value ("auto-detect one"), so unlike + // the accelerator it is not replaced by a default. + device = value + } + return accel, device +} + +// readSetting reports the trimmed setting value and whether the settings repo +// answered at all. The second result is what lets callers tell "configured +// empty" apart from "could not read". +func (s *Service) readSetting(ctx context.Context, key string) (string, bool) { + if s == nil || s.settings == nil { + return "", false + } + value, err := s.settings.Get(ctx, key) + if err != nil { + return "", false + } + return strings.TrimSpace(value), true } func (s *Service) chapterThumbnailExecutionMode(ctx context.Context) string { diff --git a/internal/chapterthumbs/service_test.go b/internal/chapterthumbs/service_test.go index 52985ea30..bf54d33b7 100644 --- a/internal/chapterthumbs/service_test.go +++ b/internal/chapterthumbs/service_test.go @@ -874,3 +874,64 @@ func TestExtractFrameResolvesMultiDeviceListToOneDevice(t *testing.T) { t.Fatalf("ffmpeg args missing a resolved device:\n%s", joined) } } + +// failingSettingsReader stands in for a settings repo that cannot answer, so +// the fallback path can be told apart from a configured-empty value. +type failingSettingsReader struct{} + +func (failingSettingsReader) Get(_ context.Context, _ string) (string, error) { + return "", errors.New("settings unavailable") +} + +// TestResolveHWConfigFollowsLiveSettings is the regression guard for the +// restart-required conversion: hardware acceleration is read from the settings +// repo per extraction, so an admin changing playback.hw_accel or +// playback.hw_device does not have to restart the server for chapter +// thumbnails to follow. +func TestResolveHWConfigFollowsLiveSettings(t *testing.T) { + values := map[string]string{ + "playback.hw_accel": "vaapi", + "playback.hw_device": "/dev/dri/renderD128", + } + service := &Service{ + // Deliberately different from the settings rows: the boot values must + // not win over the live configuration. + hwAccel: "none", + hwDevice: "/dev/dri/renderD200", + settings: testSettingsReader{values: values}, + } + + accel, device := service.resolveHWConfig(context.Background()) + if accel != "vaapi" || device != "/dev/dri/renderD128" { + t.Fatalf("resolveHWConfig() = (%q, %q), want (vaapi, /dev/dri/renderD128)", accel, device) + } + + values["playback.hw_accel"] = "qsv" + values["playback.hw_device"] = "/dev/dri/renderD129" + + accel, device = service.resolveHWConfig(context.Background()) + if accel != "qsv" || device != "/dev/dri/renderD129" { + t.Fatalf("resolveHWConfig() after settings change = (%q, %q), want (qsv, /dev/dri/renderD129)", accel, device) + } + + // An emptied device row means "auto-detect", not "keep the previous one". + values["playback.hw_device"] = "" + if _, device = service.resolveHWConfig(context.Background()); device != "" { + t.Fatalf("resolveHWConfig() device after clearing = %q, want empty", device) + } +} + +// TestResolveHWConfigFallsBackWhenSettingsUnavailable keeps a database blip +// from silently switching extraction off the configured accelerator. +func TestResolveHWConfigFallsBackWhenSettingsUnavailable(t *testing.T) { + service := &Service{ + hwAccel: "vaapi", + hwDevice: "/dev/dri/renderD128", + settings: failingSettingsReader{}, + } + + accel, device := service.resolveHWConfig(context.Background()) + if accel != "vaapi" || device != "/dev/dri/renderD128" { + t.Fatalf("resolveHWConfig() = (%q, %q), want the boot values", accel, device) + } +} diff --git a/internal/config/admin_settings.go b/internal/config/admin_settings.go index 117b424cc..9915d4786 100644 --- a/internal/config/admin_settings.go +++ b/internal/config/admin_settings.go @@ -43,6 +43,10 @@ const ArtworkStorageReconcileCheckpointKey = "s3.public_storage_reconcile_checkp // readers that own each setting. The UI must never invent a second set of // defaults: an untouched form should describe the behavior the server is // actually running. +// Setting keys and default values are a data table; naming each repeated +// literal would bury what the table says. +// +//nolint:goconst var adminSettingDefaults = map[string]string{ "auth.access_token_expiry": "8h", "auth.refresh_token_expiry": "30d", @@ -53,23 +57,30 @@ var adminSettingDefaults = map[string]string{ "clientip.trusted_proxies": "10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 127.0.0.0/8, ::1/128", "theme.catalog_url": DefaultThemeCatalogURL, - "database.max_connections": "20", - "s3.public_path_style": "true", - "s3.public_url_auth": "presigned", - "s3.public_token_param": "verify", - "s3.public_token_ttl": "10800", - "s3.private_path_style": "true", - "s3.user_db_path_style": "true", - "userdb.backend": "postgres", - "userdb.pool_max_open": "500", - "userdb.idle_timeout": "12h", - - "scanner.workers": "8", - "matcher.workers": "8", - "matcher.batch_size": "500", - "metadata.cache_images": "false", - "markers.mode": "local", - "markers.lazy_playback": "false", + "database.max_connections": "20", + "s3.public_path_style": "true", + "s3.public_url_auth": "presigned", + "s3.public_token_param": "verify", + "s3.public_token_ttl": "10800", + "s3.private_path_style": "true", + "s3.metadata_presign_expiry": "4h", + "s3.user_db_path_style": "true", + "userdb.backend": "postgres", + "userdb.pool_max_open": "500", + "userdb.idle_timeout": "12h", + + "scanner.workers": "8", + "scanner.max_concurrent_libraries": "1", + "scanner.max_concurrent_scoped": "2", + "scanner.file_removal_grace": "24h", + "scanner.empty_trash_after_scan": "true", + "matcher.workers": "8", + "matcher.batch_size": "500", + "matcher.enable_tv_series_root_queue": "true", + "matcher.enable_tv_series_group_queue": "false", + "metadata.cache_images": "false", + "markers.mode": "local", + "markers.lazy_playback": "false", "playback.ffmpeg_path": "", playbackTranscodeDirSettingKey: DefaultTranscodeDir, @@ -101,6 +112,8 @@ var adminSettingDefaults = map[string]string{ "jellyfin_compat.playback_session_ttl": "6h", "recommendations.enabled": "false", + "recommendations.embedding_provider": "ollama", + "recommendations.embeddings_job_timeout": "24h", "recommendations.embedding_base_url": "http://ollama:11434", "recommendations.embedding_model": "all-minilm", "recommendations.embeddings_cron": "0 3 * * *", @@ -119,6 +132,7 @@ var adminSettingDefaults = map[string]string{ "subtitle_ai.batch_size": "40", "subtitle_ai.context_neighbors": "2", "subtitle_ai.asr_chunk_seconds": "600", + "subtitle_ai.live_asr_chunk_seconds": "30", "subtitle_ai.transcribe_quota_jobs": "0", "subtitle_ai.transcribe_quota_period": "day", "metadata_ai.enabled": "false", @@ -134,6 +148,7 @@ var adminSettingDefaults = map[string]string{ "download.max_concurrent_prepares": "2", "download.artifact_max_bytes": "0", + "policy.editor_enabled": "false", "policy.decision_log_verbosity": "digest", "policy.decision_log_scope_sample_rate": "50", "policy.decision_log_retention_days": "14", @@ -174,6 +189,7 @@ var adminSettingDefaults = map[string]string{ "taskmanager.history_retention_days": "30", "taskmanager.history_keep_per_task": "1000", + "opslog.capture_level": "info", "opslog.retention_days": "7", "opslog.cleanup_interval_minutes": "15", "opslog.max_rows": "1000000", @@ -303,6 +319,8 @@ func NormalizeAdminSetting(key, raw string) (string, error) { "jellyfin_compat.enabled", "jellyfin_compat.web_enabled", "recommendations.enabled", "subtitle_ai.enabled", "subtitle_ai.transcribe_enabled", "metadata_ai.enabled", "download.enabled", "download.transcode_enabled", "email.enabled", "signup.enabled", + "scanner.empty_trash_after_scan", "matcher.enable_tv_series_root_queue", + "matcher.enable_tv_series_group_queue", "policy.editor_enabled", "overlays.enabled", "notifications.release_events_enabled", "notifications.fanout_enabled", "notifications.ui_enabled", "notifications.webhooks_enabled", "notifications.webhooks.allow_private_destinations", "notifications.email_enabled", @@ -318,7 +336,8 @@ func NormalizeAdminSetting(key, raw string) (string, error) { return normalizeAdminInt(key, value, 1, 10000) case "userdb.pool_max_open": return normalizeAdminInt(key, value, 1, 100000) - case "scanner.workers", "matcher.workers": + case "scanner.workers", "matcher.workers", + "scanner.max_concurrent_libraries", "scanner.max_concurrent_scoped": return normalizeAdminInt(key, value, 1, 1024) case "matcher.batch_size": return normalizeAdminInt(key, value, 1, 100000) @@ -338,6 +357,10 @@ func NormalizeAdminSetting(key, raw string) (string, error) { return normalizeAdminInt(key, value, 0, 100) case "subtitle_ai.asr_chunk_seconds": return normalizeAdminInt(key, value, 60, 600) + case "subtitle_ai.live_asr_chunk_seconds": + // 15s is the transcriber's hard floor (clampASRChunkSeconds); accepting + // less would store a value the runtime silently raises. + return normalizeAdminInt(key, value, 15, 600) case "subtitle_ai.transcribe_quota_jobs": return normalizeAdminInt(key, value, 0, math.MaxInt32) case "download.server_bandwidth_mbps", "download.user_bandwidth_mbps": @@ -394,11 +417,24 @@ func NormalizeAdminSetting(key, raw string) (string, error) { case "auth.access_token_expiry", "auth.refresh_token_expiry", "userdb.idle_timeout", "download.period_duration", "jellyfin_compat.session_ttl", - "jellyfin_compat.playback_session_ttl": + "jellyfin_compat.playback_session_ttl", "s3.metadata_presign_expiry", + "recommendations.embeddings_job_timeout": return normalizeAdminDuration(key, value) + case "scanner.file_removal_grace": + // The scanner deliberately tolerates a zero or negative grace as + // "remove missing files immediately" (LoadFromDB warns and clamps to + // zero), so only require a parseable duration here. + if _, err := parseDuration(value); err != nil { + return "", fmt.Errorf("%s must be a duration", key) + } + return value, nil case "server.log_level": return normalizeAdminEnum(key, value, "debug", "info", "warn", "error") + case "opslog.capture_level": + // "warning" is accepted because the startup reader in cmd/silo treats + // it as an alias for "warn". + return normalizeAdminEnum(key, value, "debug", "info", "warn", "warning", "error") case "userdb.backend": return normalizeAdminEnum(key, value, "postgres", "sqlite") case "playback.hw_accel": diff --git a/internal/config/admin_settings_test.go b/internal/config/admin_settings_test.go index 69fe89eff..69c79a20c 100644 --- a/internal/config/admin_settings_test.go +++ b/internal/config/admin_settings_test.go @@ -243,6 +243,18 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) { {key: "theme.catalog_url", value: "http://raw.githubusercontent.com/Silo-Server/silo-themes/main/catalog.json"}, {key: "theme.catalog_url", value: "https://example.com/catalog.json"}, {key: "redis.url", value: "not-a-url"}, + {key: "scanner.max_concurrent_libraries", value: "0"}, + {key: "scanner.max_concurrent_scoped", value: "-1"}, + {key: "scanner.empty_trash_after_scan", value: "sometimes"}, + {key: "scanner.file_removal_grace", value: "a while"}, + {key: "matcher.enable_tv_series_root_queue", value: "yes please"}, + {key: "matcher.enable_tv_series_group_queue", value: "yes please"}, + {key: "policy.editor_enabled", value: "maybe"}, + {key: "policy.eval_timeout_ms", value: "0"}, + {key: "subtitle_ai.live_asr_chunk_seconds", value: "0"}, + {key: "opslog.capture_level", value: "chatty"}, + {key: "s3.metadata_presign_expiry", value: "0s"}, + {key: "recommendations.embeddings_job_timeout", value: "soon"}, } for _, tc := range tests { t.Run(tc.key, func(t *testing.T) { @@ -326,3 +338,59 @@ func TestNormalizeAdminSettingCanonicalizesRedisURL(t *testing.T) { t.Fatalf("normalized Redis URL = %q", got) } } + +// TestNormalizeAdminSettingKeepsPermissiveScannerGrace locks the loader's +// documented behavior: a zero or negative grace means "remove missing files +// immediately", so the admin API must not reject it. +func TestNormalizeAdminSettingKeepsPermissiveScannerGrace(t *testing.T) { + for _, value := range []string{"0s", "-1h", "72h"} { + got, err := NormalizeAdminSetting("scanner.file_removal_grace", " "+value+" ") + if err != nil { + t.Fatalf("NormalizeAdminSetting(scanner.file_removal_grace, %q): %v", value, err) + } + if got != value { + t.Fatalf("normalized grace = %q, want %q", got, value) + } + } +} + +// TestOpslogCaptureLevelAcceptsWarningAlias mirrors the startup reader in +// cmd/silo, which treats "warning" as "warn". +func TestOpslogCaptureLevelAcceptsWarningAlias(t *testing.T) { + got, err := NormalizeAdminSetting("opslog.capture_level", "WARNING") + if err != nil { + t.Fatal(err) + } + if got != "warning" { + t.Fatalf("normalized capture level = %q, want warning", got) + } +} + +// TestHiddenTierDefaultsAreExposed guards the keys that have no admin UI: the +// API must still report the value the server is actually running. +func TestHiddenTierDefaultsAreExposed(t *testing.T) { + effective := EffectiveAdminSettings(nil) + want := map[string]string{ + "recommendations.embedding_provider": "ollama", + "recommendations.embeddings_job_timeout": "24h", + "policy.editor_enabled": "false", + "policy.eval_timeout_ms": "250", + "subtitle_ai.live_asr_chunk_seconds": "30", + "scanner.max_concurrent_libraries": "1", + "scanner.max_concurrent_scoped": "2", + "scanner.file_removal_grace": "24h", + "scanner.empty_trash_after_scan": "true", + "matcher.enable_tv_series_root_queue": "true", + "matcher.enable_tv_series_group_queue": "false", + "opslog.capture_level": "info", + "s3.metadata_presign_expiry": "4h", + } + for key, value := range want { + if got := effective[key]; got != value { + t.Errorf("effective[%q] = %q, want %q", key, got, value) + } + if _, err := NormalizeAdminSetting(key, value); err != nil { + t.Errorf("default for %q is rejected by NormalizeAdminSetting: %v", key, err) + } + } +} diff --git a/internal/config/artifact_dir_test.go b/internal/config/artifact_dir_test.go new file mode 100644 index 000000000..746465df5 --- /dev/null +++ b/internal/config/artifact_dir_test.go @@ -0,0 +1,31 @@ +package config + +import "testing" + +func TestEffectiveDownloadArtifactDir(t *testing.T) { + cases := []struct { + name string + artifactDir string + transcodeDir string + want string + }{ + {"explicit artifact dir wins", "/mnt/downloads", "/mnt/fast/transcode", "/mnt/downloads"}, + {"both blank uses the default transcode dir", "", "", "/tmp/silo-download-artifacts"}, + {"sibling of a custom transcode dir", "", "/mnt/fast/transcode", "/mnt/fast/silo-download-artifacts"}, + // A trailing slash must not nest the artifact root inside the + // transcode dir: the orphaned-transcode sweep deletes non-active + // subdirectories of the transcode root, so nesting is data loss. + {"trailing slash still yields the sibling", "", "/mnt/fast/transcode/", "/mnt/fast/silo-download-artifacts"}, + {"root transcode dir", "", "/", "/silo-download-artifacts"}, + {"repeated separators are cleaned", "", "/mnt//fast/transcode", "/mnt/fast/silo-download-artifacts"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := EffectiveDownloadArtifactDir(tc.artifactDir, tc.transcodeDir); got != tc.want { + t.Fatalf("EffectiveDownloadArtifactDir(%q, %q) = %q, want %q", + tc.artifactDir, tc.transcodeDir, got, tc.want) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index bb39dba84..52b4e79dd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -425,7 +425,11 @@ func EffectiveDownloadArtifactDir(artifactDir, transcodeDir string) string { if transcodeDir == "" { transcodeDir = DefaultTranscodeDir } - return filepath.Join(filepath.Dir(transcodeDir), "silo-download-artifacts") + // Clean first: with a trailing slash, filepath.Dir("/srv/transcode/") is + // "/srv/transcode", which would nest the artifact root INSIDE the transcode + // dir — where the orphaned-transcode sweep deletes non-active + // subdirectories, i.e. it would delete prepared downloads. + return filepath.Join(filepath.Dir(filepath.Clean(transcodeDir)), "silo-download-artifacts") } const DefaultJellyfinCompatEmulatedServerVersion = "10.12.0" diff --git a/internal/config/restart_keys.go b/internal/config/restart_keys.go index 27d6267ae..f14596e11 100644 --- a/internal/config/restart_keys.go +++ b/internal/config/restart_keys.go @@ -1,6 +1,9 @@ package config -import "strings" +import ( + "sort" + "strings" +) // restartRequiredKeys lists server_settings keys whose values are captured at // process startup (listeners, connection pools, HTTP clients, worker pools) @@ -30,12 +33,21 @@ var restartRequiredKeys = map[string]bool{ "ratelimit.enabled": true, "ratelimit.backend": true, - // Playback transcode infrastructure. The playback/stream handlers read - // ffmpeg path and hwaccel live (new transcode sessions), but several - // startup-built consumers still freeze them (scanner ffprobe, chapter - // thumbnails, audiobook enricher) — keep restart-required until those - // convert. transcode_dir is also captured by dedicated transcode nodes for - // session and prepared-download storage. A configured download.artifact_dir is + // Playback transcode infrastructure. The native playback/stream handlers, + // the transcode nodes, and the download artifact managers read ffmpeg path + // and hwaccel live (new transcode sessions), and chapter-thumbnail + // extraction and the playback probe ensurer now do too. + // + // What still freezes them is the jellycompat playback handler: it captures + // FFmpegPath/HWAccel and the boot *config.Config (for hw_device) when the + // compat router is built, so a Jellyfin-client transcode keeps the boot + // values until restart. ffmpeg_path has three more startup-frozen consumers + // in cmd/silo/main.go — the intro-marker analyzer, the scanner's own ffprobe + // path, and the audiobook enricher. Keep these restart-required until those + // convert; converting them is what lets the badge go away. + // + // transcode_dir is also captured by dedicated transcode nodes for session + // and prepared-download storage. A configured download.artifact_dir is // likewise captured by both API and transcode-node artifact managers. The // chapter-thumbnail worker pool is sized at construction. "playback.ffmpeg_path": true, @@ -55,12 +67,11 @@ var restartRequiredKeys = map[string]bool{ "matcher.enable_tv_series_root_queue": true, "matcher.enable_tv_series_group_queue": true, - // External API clients built once at startup. + // External API clients built once at startup. watchsync.trakt.client_id is + // deliberately absent: the collection adapter re-reads it from the settings + // repo before each upstream call (atomic setter on the shared client), and + // the watch-sync OAuth flows always read both credentials live. "tmdb.api_key": true, - // The Trakt collection browser captures its public client ID when the - // router is built. Watch-sync flows read both credentials live, but a - // restart is still required for the collection adapter to converge. - "watchsync.trakt.client_id": true, // Compat listeners and session stores. "audiobookshelf_compat.enabled": true, @@ -118,3 +129,28 @@ func RestartRequired(key string) bool { } return false } + +// RestartRequiredKeys returns the sorted list of exact server_settings keys +// that require a restart. The admin UI reads this over +// GET /admin/settings/restart-keys to render its restart badges, so it must +// stay a copy: callers must not be able to mutate the registry. +func RestartRequiredKeys() []string { + keys := make([]string, 0, len(restartRequiredKeys)) + for key, required := range restartRequiredKeys { + if required { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +// RestartRequiredPrefixes returns the sorted list of key prefixes whose whole +// namespace requires a restart. Pair it with RestartRequiredKeys: a key needs a +// restart when it is listed exactly or carries one of these prefixes. +func RestartRequiredPrefixes() []string { + prefixes := make([]string, len(restartRequiredPrefixes)) + copy(prefixes, restartRequiredPrefixes) + sort.Strings(prefixes) + return prefixes +} diff --git a/internal/config/restart_keys_test.go b/internal/config/restart_keys_test.go index a81b773ad..4924bca4f 100644 --- a/internal/config/restart_keys_test.go +++ b/internal/config/restart_keys_test.go @@ -58,3 +58,36 @@ func TestRestartRequired(t *testing.T) { } } } + +// The exported accessors feed the admin API, so they must agree with +// RestartRequired, stay sorted, and hand out copies rather than the registry. +func TestRestartRequiredAccessors(t *testing.T) { + keys := RestartRequiredKeys() + if len(keys) != len(restartRequiredKeys) { + t.Fatalf("RestartRequiredKeys() returned %d keys, want %d", len(keys), len(restartRequiredKeys)) + } + for i, key := range keys { + if i > 0 && keys[i-1] >= key { + t.Fatalf("RestartRequiredKeys() is not sorted at %d: %v", i, keys) + } + if !RestartRequired(key) { + t.Errorf("RestartRequiredKeys() reported %q, which RestartRequired rejects", key) + } + } + + prefixes := RestartRequiredPrefixes() + if len(prefixes) != len(restartRequiredPrefixes) { + t.Fatalf("RestartRequiredPrefixes() returned %d prefixes, want %d", len(prefixes), len(restartRequiredPrefixes)) + } + for _, prefix := range prefixes { + if !RestartRequired(prefix + "anything") { + t.Errorf("RestartRequiredPrefixes() reported %q, which RestartRequired rejects", prefix) + } + } + + // Mutating a returned slice must not corrupt the registry. + prefixes[0] = "mutated." + if RestartRequiredPrefixes()[0] == "mutated." { + t.Error("RestartRequiredPrefixes() exposes the package-level slice") + } +} diff --git a/internal/dashmetrics/egress.go b/internal/dashmetrics/egress.go new file mode 100644 index 000000000..6a6d55e5d --- /dev/null +++ b/internal/dashmetrics/egress.go @@ -0,0 +1,67 @@ +package dashmetrics + +import "github.com/Silo-Server/silo-server/internal/streamtelemetry" + +// egressDelta is the viewer bytes one process served between two telemetry +// snapshots. Total covers every viewer-egress byte; Download is the subset +// served by file-transfer routes (telemetry transfers: offline/direct +// downloads, ebook and ABS file fetches) rather than streaming playback. +// Download is always <= Total, so a reader can derive playback as the +// difference without ever going negative. +type egressDelta struct { + Total int64 + Download int64 +} + +// computeEgressDelta returns the viewer bytes this process served between two +// telemetry snapshots, together with the cumulative counters the next call must +// compare against. +// +// Only RoleViewerEgress routes and transfers count. A proxy node's viewer +// traffic also traverses the API node as RoleInternalRelay, and counting both +// would report every relayed byte twice. +// +// The split leans on the registry's own taxonomy: playback traffic aggregates +// into logical sessions (ClassPlayback/ClassManifest routes), while +// file-transfer traffic aggregates into transfers (ClassTransfer routes), so +// session growth is playback egress and transfer growth is download egress. A +// route added to either class is classified automatically. +// +// Counters only ever grow, but a session can be pruned and re-created under the +// same id, and a restarted registry starts from zero. A shrinking counter is +// therefore read as a fresh start and contributes nothing rather than a +// negative rate. Entries that vanished from the snapshot are dropped, which +// keeps the map bounded by the live session count. +func computeEgressDelta(prev map[string]int64, snapshot streamtelemetry.Snapshot) (egressDelta, map[string]int64) { + next := make(map[string]int64, len(snapshot.Sessions)+len(snapshot.Transfers)) + var delta egressDelta + + record := func(key string, cumulative int64) int64 { + next[key] = cumulative + if grown := cumulative - prev[key]; grown > 0 { + return grown + } + return 0 + } + + for _, session := range snapshot.Sessions { + var bytes int64 + for _, route := range session.Routes { + if route.Role == streamtelemetry.RoleViewerEgress { + bytes += route.BytesAccepted + } + } + delta.Total += record("session:"+session.SessionID, bytes) + } + + for _, transfer := range snapshot.Transfers { + if transfer.Role != streamtelemetry.RoleViewerEgress { + continue + } + grown := record("transfer:"+transfer.ID, transfer.BytesAccepted) + delta.Total += grown + delta.Download += grown + } + + return delta, next +} diff --git a/internal/dashmetrics/sampler.go b/internal/dashmetrics/sampler.go new file mode 100644 index 000000000..e33486e39 --- /dev/null +++ b/internal/dashmetrics/sampler.go @@ -0,0 +1,257 @@ +// Package dashmetrics records the admin dashboard time series that cannot be +// reconstructed after the fact: how many streams were running (split by play +// method) and how much egress the deployment served. Live sessions leave no +// per-minute trace once they end, and node egress is a rolling average that is +// overwritten on every health check, so both have to be sampled as they happen. +// +// One row per minute per source lands in dashboard_metric_samples, and rows +// older than the retention window below are pruned once an hour: +// +// - "shared" is the cluster-wide snapshot. Every replica writes it with +// INSERT ... ON CONFLICT DO NOTHING, so the first writer for a minute wins +// and the others collapse. Replica snapshots differ only by sub-second +// timing, which is below the resolution a dashboard chart can show, so this +// is deliberately cheaper than coordinating with an advisory lock. +// - "proc:" carries the viewer egress served by one API process, +// measured from the local stream-telemetry registry. stream_nodes only +// describes external stream nodes, so without these rows a single-server +// deployment would chart zero egress forever. egress_kbps is the process +// total; download_egress_kbps is the file-transfer subset of that total +// (see computeEgressDelta), so the dashboard can split playback from +// download traffic. +// +// Sampling is best-effort: every failure is logged and swallowed. A missed +// minute is a gap in the chart, never a failed request or a dead server. +package dashmetrics + +import ( + "context" + "log/slog" + "math" + "os" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +const ( + // component is the slog component key every line from this package carries. + component = "dashmetrics" + + // sampleInterval matches the minute resolution of the samples table. + sampleInterval = time.Minute + + // sampleTickTimeout bounds one tick's database work, comfortably under the + // interval so a wedged pool costs missed minutes, not a stuck sampler. + sampleTickTimeout = 30 * time.Second + + // retentionDays is how much history the charts can show — a month, so the + // dashboard's widest range has samples to draw. 1440 minutes a day times 31 + // days is ~45k rows per source, and sources are (1 + replicas), so the table + // stays in the low hundreds of thousands of rows at most. Reads bucket the + // minutes down before returning them (internal/api/handlers), so a wide + // window costs the same on the wire as a narrow one. + retentionDays = 31 +) + +// Sampler writes one dashboard_metric_samples row per minute for as long as it +// runs. Its state is owned by the single goroutine Start launches; nothing else +// reads or mutates it. +type Sampler struct { + pool *pgxpool.Pool + telemetry *streamtelemetry.Registry // nil when stream telemetry is disabled + source string // "proc:" + interval time.Duration + + // lastBucket is the minute the last tick wrote, so a ticker that fires + // twice inside one minute does not spend an INSERT that ON CONFLICT would + // only discard — which would silently drop the egress bytes it carried. + lastBucket time.Time + + // prevBytes holds the cumulative viewer bytes per telemetry session and + // transfer at the previous tick; lastEgressAt is when it was taken. + prevBytes map[string]int64 + lastEgressAt time.Time + + stopOnce sync.Once + stop chan struct{} +} + +// NewSampler builds a sampler for this process. telemetry may be nil, in which +// case only the shared cluster row is written. nodeID identifies this process +// among the replicas; it falls back to the hostname when empty. +func NewSampler(pool *pgxpool.Pool, telemetry *streamtelemetry.Registry, nodeID string) *Sampler { + if nodeID == "" { + nodeID, _ = os.Hostname() + } + if nodeID == "" { + nodeID = "unknown" + } + return &Sampler{ + pool: pool, + telemetry: telemetry, + source: "proc:" + nodeID, + interval: sampleInterval, + stop: make(chan struct{}), + } +} + +// Start samples once immediately — which also establishes the egress baseline — +// and then every minute until ctx is canceled or Stop is called. +func (s *Sampler) Start(ctx context.Context) { + if s == nil || s.pool == nil { + return + } + go func() { + s.sampleOnce(ctx, time.Now()) + + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case at := <-ticker.C: + s.sampleOnce(ctx, at) + } + } + }() +} + +// Stop ends the sampling goroutine. It is safe to call more than once. +func (s *Sampler) Stop() { + if s == nil { + return + } + s.stopOnce.Do(func() { + close(s.stop) + }) +} + +// sampleOnce writes this minute's rows and, once an hour, prunes expired ones. +// +// Every tick's database work runs under its own deadline: the sampler is one +// goroutine, and an Exec left on the lifetime context during a wedged pool +// would block it — and with it every later sample and the retention prune — +// for as long as the outage lasts. A bounded tick turns that into a bounded +// gap in the chart instead. +func (s *Sampler) sampleOnce(ctx context.Context, at time.Time) { + bucket := sampleBucket(at) + if bucket.Equal(s.lastBucket) { + return + } + s.lastBucket = bucket + + ctx, cancel := context.WithTimeout(ctx, sampleTickTimeout) + defer cancel() + + s.sampleShared(ctx) + s.sampleProcessEgress(ctx, at) + + // Retention runs in-band rather than as its own timer: the table is tiny + // and one DELETE an hour costs less than another goroutine. + if at.Minute() == 0 { + s.pruneExpired(ctx) + } +} + +// sampleShared records the cluster-wide stream counts and node egress. Counting +// and inserting happen in one statement so no replica can read one minute's +// state and write it into another's bucket. +func (s *Sampler) sampleShared(ctx context.Context) { + _, err := s.pool.Exec(ctx, ` + INSERT INTO dashboard_metric_samples + (bucket, source, streams_total, streams_direct, streams_remux, streams_transcode, egress_kbps) + SELECT date_trunc('minute', now()), 'shared', + (SELECT COUNT(*) FROM playback_sessions_sync), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'direct'), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'remux'), + (SELECT COUNT(*) FROM playback_sessions_sync WHERE play_method = 'transcode'), + (SELECT COALESCE(SUM(egress_kbps), 0) FROM stream_nodes WHERE enabled AND healthy) + ON CONFLICT (bucket, source) DO NOTHING + `) + if err != nil { + slog.WarnContext(ctx, "failed to sample shared dashboard metrics", "component", component, "error", err) + } +} + +// sampleProcessEgress records the viewer egress this process served since the +// previous tick. The row is bucketed on the database clock — the same clock the +// shared row and the read window use — so a skewed host cannot land streams and +// their egress in adjacent minutes, or write a "future" row the dashboard's +// server-anchored grid would drop. Two ticks that map onto one DB minute merge +// by GREATEST, which keeps the peak (the read side is peak-preserving anyway) +// instead of silently discarding the second delta; taking each column's max +// independently preserves download <= total because it holds per row. +func (s *Sampler) sampleProcessEgress(ctx context.Context, at time.Time) { + if s.telemetry == nil { + return + } + + // Sweep rather than Snapshot: Snapshot reports byte totals as of the last + // telemetry sweep, and a sweep interval configured above one minute would + // make ticks in between read zero growth and the next one attribute + // several minutes of bytes to a single minute — a spike that never + // happened. Sweep collects the live counters now. + delta, next := computeEgressDelta(s.prevBytes, s.telemetry.Sweep()) + previous, previousAt := s.prevBytes, s.lastEgressAt + s.prevBytes, s.lastEgressAt = next, at + + // The very first snapshot carries every byte served since the process + // started. Charting that as one minute of egress would draw a spike that + // never happened, so the first tick only establishes the baseline. + if previous == nil || previousAt.IsZero() { + return + } + + // Zero minutes are written too: an idle server should draw a line along the + // baseline, not a gap that reads as "no data". egress_kbps stays the total + // this process served (its pre-split meaning), while download_egress_kbps + // carries the file-transfer subset. The subset is clamped under the total + // after rounding so a reader deriving playback as total - download can + // never see a negative minute from two independent roundings. + elapsed := at.Sub(previousAt) + totalKbps := egressKbps(delta.Total, elapsed) + downloadKbps := min(egressKbps(delta.Download, elapsed), totalKbps) + _, err := s.pool.Exec(ctx, ` + INSERT INTO dashboard_metric_samples (bucket, source, egress_kbps, download_egress_kbps) + VALUES (date_trunc('minute', now()), $1, $2, $3) + ON CONFLICT (bucket, source) DO UPDATE + SET egress_kbps = GREATEST(dashboard_metric_samples.egress_kbps, EXCLUDED.egress_kbps), + download_egress_kbps = GREATEST(dashboard_metric_samples.download_egress_kbps, EXCLUDED.download_egress_kbps) + `, s.source, totalKbps, downloadKbps) + if err != nil { + slog.WarnContext(ctx, "failed to sample process egress", "component", component, "source", s.source, "error", err) + } +} + +// pruneExpired drops samples older than the retention window. +func (s *Sampler) pruneExpired(ctx context.Context) { + _, err := s.pool.Exec(ctx, ` + DELETE FROM dashboard_metric_samples + WHERE bucket < now() - make_interval(days => $1) + `, retentionDays) + if err != nil { + slog.WarnContext(ctx, "failed to prune dashboard metric samples", "component", component, "error", err) + } +} + +// sampleBucket truncates a sample time to the minute it belongs to, in UTC. +func sampleBucket(at time.Time) time.Time { + return at.UTC().Truncate(time.Minute) +} + +// egressKbps converts a byte delta over an elapsed period into kilobits per +// second. A non-positive delta or elapsed period is reported as zero rather +// than as a negative rate. +func egressKbps(deltaBytes int64, elapsed time.Duration) int64 { + if deltaBytes <= 0 || elapsed <= 0 { + return 0 + } + return int64(math.Round(float64(deltaBytes) * 8 / 1000 / elapsed.Seconds())) +} diff --git a/internal/dashmetrics/sampler_test.go b/internal/dashmetrics/sampler_test.go new file mode 100644 index 000000000..c4f152519 --- /dev/null +++ b/internal/dashmetrics/sampler_test.go @@ -0,0 +1,267 @@ +package dashmetrics + +import ( + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +func viewerSession(id string, bytes int64) streamtelemetry.SessionView { + return streamtelemetry.SessionView{ + SessionID: id, + Routes: []streamtelemetry.RouteActivityView{ + {Method: "GET", Pattern: "/stream", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: bytes}, + }, + } +} + +func TestComputeEgressDelta(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prev map[string]int64 + snapshot streamtelemetry.Snapshot + wantDelta egressDelta + wantNext map[string]int64 + }{ + { + name: "empty snapshot yields nothing", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{}, + wantDelta: egressDelta{}, + wantNext: map[string]int64{}, + }, + { + name: "a new session contributes all of its bytes", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 4_000)}, + }, + wantDelta: egressDelta{Total: 4_000}, + wantNext: map[string]int64{"session:s1": 4_000}, + }, + { + name: "a grown session contributes only the growth", + prev: map[string]int64{"session:s1": 4_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 6_500)}, + }, + wantDelta: egressDelta{Total: 2_500}, + wantNext: map[string]int64{"session:s1": 6_500}, + }, + { + name: "a pruned session leaves the map and adds nothing", + prev: map[string]int64{"session:s1": 4_000, "session:s2": 1_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s2", 1_000)}, + }, + wantDelta: egressDelta{}, + wantNext: map[string]int64{"session:s2": 1_000}, + }, + { + name: "a counter regression clamps at zero instead of going negative", + prev: map[string]int64{"session:s1": 9_000}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 500)}, + }, + wantDelta: egressDelta{}, + wantNext: map[string]int64{"session:s1": 500}, + }, + { + name: "a regressing session does not cancel out a growing one", + prev: map[string]int64{"session:s1": 9_000, "session:s2": 100}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{ + viewerSession("s1", 500), + viewerSession("s2", 900), + }, + }, + wantDelta: egressDelta{Total: 800}, + wantNext: map[string]int64{"session:s1": 500, "session:s2": 900}, + }, + { + name: "relay routes are excluded so relayed bytes are not counted twice", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{{ + SessionID: "s1", + Routes: []streamtelemetry.RouteActivityView{ + {Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 700}, + {Role: streamtelemetry.RoleInternalRelay, BytesAccepted: 50_000}, + {Role: streamtelemetry.RoleProducer, BytesAccepted: 900}, + }, + }}, + }, + wantDelta: egressDelta{Total: 700}, + wantNext: map[string]int64{"session:s1": 700}, + }, + { + name: "viewer transfers count as download egress and other transfer roles do not", + prev: map[string]int64{"transfer:t1": 200}, + snapshot: streamtelemetry.Snapshot{ + Transfers: []streamtelemetry.TransferView{ + {ID: "t1", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 1_200}, + {ID: "t2", Role: streamtelemetry.RoleInternalRelay, BytesAccepted: 8_000}, + }, + }, + wantDelta: egressDelta{Total: 1_000, Download: 1_000}, + wantNext: map[string]int64{"transfer:t1": 1_200}, + }, + { + name: "sessions and transfers with the same id stay separate", + prev: map[string]int64{}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("x", 10)}, + Transfers: []streamtelemetry.TransferView{{ID: "x", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 20}}, + }, + wantDelta: egressDelta{Total: 30, Download: 20}, + wantNext: map[string]int64{"session:x": 10, "transfer:x": 20}, + }, + { + name: "session growth stays out of the download subset", + prev: map[string]int64{"session:s1": 100, "transfer:t1": 100}, + snapshot: streamtelemetry.Snapshot{ + Sessions: []streamtelemetry.SessionView{viewerSession("s1", 700)}, + Transfers: []streamtelemetry.TransferView{{ID: "t1", Role: streamtelemetry.RoleViewerEgress, BytesAccepted: 350}}, + }, + wantDelta: egressDelta{Total: 850, Download: 250}, + wantNext: map[string]int64{"session:s1": 700, "transfer:t1": 350}, + }, + { + name: "a nil previous map behaves like an empty one", + prev: nil, + snapshot: streamtelemetry.Snapshot{Sessions: []streamtelemetry.SessionView{viewerSession("s1", 42)}}, + wantDelta: egressDelta{Total: 42}, + wantNext: map[string]int64{"session:s1": 42}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + delta, next := computeEgressDelta(tt.prev, tt.snapshot) + if delta != tt.wantDelta { + t.Fatalf("delta = %+v, want %+v", delta, tt.wantDelta) + } + if len(next) != len(tt.wantNext) { + t.Fatalf("next = %v, want %v", next, tt.wantNext) + } + for key, want := range tt.wantNext { + if got, ok := next[key]; !ok || got != want { + t.Fatalf("next[%q] = %d (present %t), want %d", key, got, ok, want) + } + } + }) + } +} + +func TestSampleBucket(t *testing.T) { + t.Parallel() + + newYork, err := time.LoadLocation("America/New_York") + if err != nil { + t.Skipf("timezone database unavailable: %v", err) + } + + tests := []struct { + name string + at time.Time + want time.Time + }{ + { + name: "seconds and nanoseconds are dropped", + at: time.Date(2026, 8, 26, 11, 58, 43, 987_654_321, time.UTC), + want: time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC), + }, + { + name: "a minute boundary is already its own bucket", + at: time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC), + want: time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC), + }, + { + name: "local times truncate on the UTC minute", + at: time.Date(2026, 8, 26, 7, 58, 43, 0, newYork), + want: time.Date(2026, 8, 26, 11, 58, 0, 0, time.UTC), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := sampleBucket(tt.at) + if !got.Equal(tt.want) { + t.Fatalf("sampleBucket(%s) = %s, want %s", tt.at, got, tt.want) + } + if got.Location() != time.UTC { + t.Fatalf("bucket location = %s, want UTC", got.Location()) + } + }) + } +} + +func TestSampleBucketDetectsRepeatedMinutes(t *testing.T) { + t.Parallel() + + first := time.Date(2026, 8, 26, 11, 58, 1, 0, time.UTC) + again := time.Date(2026, 8, 26, 11, 58, 59, 0, time.UTC) + later := time.Date(2026, 8, 26, 11, 59, 0, 0, time.UTC) + + if !sampleBucket(first).Equal(sampleBucket(again)) { + t.Fatal("two ticks inside one minute produced different buckets") + } + if sampleBucket(first).Equal(sampleBucket(later)) { + t.Fatal("ticks in different minutes produced the same bucket") + } +} + +func TestEgressKbps(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + deltaBytes int64 + elapsed time.Duration + want int64 + }{ + {name: "no bytes is zero", deltaBytes: 0, elapsed: time.Minute, want: 0}, + {name: "a negative delta never reports a negative rate", deltaBytes: -5, elapsed: time.Minute, want: 0}, + {name: "a zero elapsed period cannot divide", deltaBytes: 1_000, elapsed: 0, want: 0}, + {name: "a backwards clock cannot divide", deltaBytes: 1_000, elapsed: -time.Second, want: 0}, + {name: "one megabyte a second is 8000 kbps", deltaBytes: 1_000_000, elapsed: time.Second, want: 8_000}, + {name: "a minute of bytes spreads over the minute", deltaBytes: 60_000_000, elapsed: time.Minute, want: 8_000}, + {name: "sub-kilobit rates round rather than truncate to zero", deltaBytes: 100, elapsed: time.Second, want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := egressKbps(tt.deltaBytes, tt.elapsed); got != tt.want { + t.Fatalf("egressKbps(%d, %s) = %d, want %d", tt.deltaBytes, tt.elapsed, got, tt.want) + } + }) + } +} + +func TestNewSamplerSourceKey(t *testing.T) { + t.Parallel() + + if got := NewSampler(nil, nil, "api-2").source; got != "proc:api-2" { + t.Fatalf("source = %q, want %q", got, "proc:api-2") + } + if got := NewSampler(nil, nil, "").source; got == "proc:" { + t.Fatal("an empty node id must fall back to a host identity, not an empty source key") + } +} + +func TestSamplerStopIsIdempotent(t *testing.T) { + t.Parallel() + + sampler := NewSampler(nil, nil, "api-1") + sampler.Stop() + sampler.Stop() +} diff --git a/internal/database/postgres_tune.go b/internal/database/postgres_tune.go index fa6951654..62dca0efa 100644 --- a/internal/database/postgres_tune.go +++ b/internal/database/postgres_tune.go @@ -1,7 +1,6 @@ package database import ( - "bufio" "context" "fmt" "math" @@ -10,6 +9,7 @@ import ( "strconv" "strings" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/jackc/pgx/v5/pgxpool" ) @@ -468,20 +468,17 @@ func normalizePostgresTuneDBSize(raw string) (string, error) { } func detectTotalMemoryBytes() (int64, string, error) { - for _, path := range []string{ - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - } { - if mem, err := readCgroupMemoryLimit(path); err == nil && mem > 0 { + for _, path := range nodemetrics.CgroupMemoryLimitPaths() { + if mem, err := nodemetrics.ReadCgroupMemoryLimit(path); err == nil && mem > 0 { return mem, path, nil } } - if mem, err := readMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 { + if mem, err := nodemetrics.ReadMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 { return mem, "/host/proc/meminfo", nil } - mem, err := readMeminfoTotalBytes("/proc/meminfo") + mem, err := nodemetrics.ReadMeminfoTotalBytes("/proc/meminfo") if err != nil { return 0, "", err } @@ -491,35 +488,6 @@ func detectTotalMemoryBytes() (int64, string, error) { return mem, "/proc/meminfo", nil } -func readMeminfoTotalBytes(path string) (int64, error) { - file, err := os.Open(path) - if err != nil { - return 0, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "MemTotal:") { - continue - } - fields := strings.Fields(line) - if len(fields) < 2 { - return 0, fmt.Errorf("malformed MemTotal line") - } - kb, err := strconv.ParseInt(fields[1], 10, 64) - if err != nil { - return 0, err - } - return kb * sizeKB, nil - } - if err := scanner.Err(); err != nil { - return 0, err - } - return 0, fmt.Errorf("MemTotal not found") -} - func runningInContainer() bool { if _, err := os.Stat("/.dockerenv"); err == nil { return true @@ -528,26 +496,6 @@ func runningInContainer() bool { return err == nil && (strings.Contains(string(raw), "docker") || strings.Contains(string(raw), "kubepods")) } -func readCgroupMemoryLimit(path string) (int64, error) { - raw, err := os.ReadFile(path) - if err != nil { - return 0, err - } - value := strings.TrimSpace(string(raw)) - if value == "" || value == "max" { - return 0, fmt.Errorf("no cgroup memory limit") - } - mem, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return 0, err - } - // Docker may expose a huge sentinel when no concrete memory limit is set. - if mem <= 0 || mem > 1<<60 { - return 0, fmt.Errorf("no concrete cgroup memory limit") - } - return mem, nil -} - func parsePostgresTuneByteSize(raw string) (int64, error) { normalized := strings.ToUpper(strings.TrimSpace(raw)) normalized = strings.ReplaceAll(normalized, " ", "") diff --git a/internal/downloads/artifacts.go b/internal/downloads/artifacts.go index 67c040faf..edb76a836 100644 --- a/internal/downloads/artifacts.go +++ b/internal/downloads/artifacts.go @@ -503,7 +503,7 @@ func (m *ArtifactManager) localToneMapCapabilities(ctx context.Context) (tonemap if cfg == nil { return nil, nil } - backend := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.Playback.HWAccel, cfg.Playback.FFmpegPath) + backend := playback.ResolveHWAccelWithFFmpegContext(ctx, cfg.Playback.HWAccel, cfg.Playback.FFmpegPath, cfg.Playback.HWDevice) if err := ctx.Err(); err != nil { return nil, err } diff --git a/internal/downloads/remote_preparer.go b/internal/downloads/remote_preparer.go index 0a5fd0d57..2c7a5744d 100644 --- a/internal/downloads/remote_preparer.go +++ b/internal/downloads/remote_preparer.go @@ -35,6 +35,12 @@ type NodeAwarePreparer struct { capabilityMu sync.Mutex capabilities map[string]remoteToneMapCapabilities capabilityFlight singleflight.Group + // capabilityInvalidations counts how many times each node's inventory has + // been dropped. A fetch snapshots it before asking the node and refuses to + // install its answer if it has moved since — otherwise a probe already in + // flight when an operator changed the node's policy writes the report it + // was sent to collect, restoring the pre-edit inventory for a full TTL. + capabilityInvalidations map[string]uint64 } // remoteToneMapCapabilities caches one node's validated inventory; an empty @@ -276,7 +282,7 @@ func (p *NodeAwarePreparer) ToneMapModeAvailable(ctx context.Context, mode tonem capable := make(map[string]struct{}) for nodeURL, capabilities := range byNode { if capabilities.Supports(mode, kind) { - capable[strings.TrimRight(nodeURL, "/")] = struct{}{} + capable[nodepool.NormalizeNodeURL(nodeURL)] = struct{}{} } } available := selector.TranscodeWorkAvailableWith(func(candidate *nodepool.Node) bool { @@ -334,7 +340,7 @@ func (p *NodeAwarePreparer) audioBoostCapableNodeURLs(ctx context.Context) map[s } func (p *NodeAwarePreparer) audioBoostCapabilityForNode(ctx context.Context, nodeURL string) (bool, error) { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) if entry, ok := p.cachedRemoteCapabilitiesForNode(nodeURL, time.Now()); ok { return supportsAudioBoostTransformation(entry.transformations), entry.err } @@ -397,7 +403,7 @@ func (p *NodeAwarePreparer) toneMapCapabilitiesByNode(ctx context.Context) (map[ // toneMapCapabilitiesForNode returns a defensive copy of a fresh cached // inventory or retrieves the node's authenticated hardware capabilities. func (p *NodeAwarePreparer) toneMapCapabilitiesForNode(ctx context.Context, nodeURL string) (tonemap.Capabilities, error) { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) if capabilities, err, ok := p.cachedToneMapCapabilitiesForNode(nodeURL, time.Now()); ok { return capabilities, err } @@ -446,22 +452,25 @@ func (p *NodeAwarePreparer) cachedRemoteCapabilitiesForNode(nodeURL string, now } func (p *NodeAwarePreparer) fetchToneMapCapabilitiesForNode(ctx context.Context, nodeURL string) (tonemap.Capabilities, error) { + // Snapshotted before the node is asked anything, so an invalidation landing + // during the request is visible at install time. + generation := p.capabilityInvalidationsFor(nodeURL) cfg := p.config() if cfg == nil || strings.TrimSpace(cfg.Auth.JWTSecret) == "" { err := errors.New("transcode node credentials unavailable") - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } requestCtx, cancel := context.WithTimeout(ctx, p.remoteToneMapProbeTimeout(nodeURL)) defer cancel() info, status, err := transcodenode.FetchHWCapabilities(requestCtx, p.probeClient, nodeURL, cfg.Auth.JWTSecret) if err != nil { - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } if status != http.StatusOK { err := fmt.Errorf("transcode node returned %d", status) - p.cacheToneMapCapabilityFailure(nodeURL, err) + p.cacheToneMapCapabilityFailure(nodeURL, generation, err) return nil, err } entry := remoteToneMapCapabilities{ @@ -471,14 +480,30 @@ func (p *NodeAwarePreparer) fetchToneMapCapabilitiesForNode(ctx context.Context, probeRequestTimeout: normalizeRemoteToneMapProbeTimeout(info.ProbeRequestTimeoutMillis), } p.capabilityMu.Lock() - if p.capabilities == nil { - p.capabilities = make(map[string]remoteToneMapCapabilities) + if p.capabilityInvalidations[nodeURL] == generation { + if p.capabilities == nil { + p.capabilities = make(map[string]remoteToneMapCapabilities) + } + p.capabilities[nodeURL] = entry } - p.capabilities[nodeURL] = entry p.capabilityMu.Unlock() + // The answer still goes back to the caller that is waiting on it, overtaken + // or not. Its request is already in flight, most policy edits do not remove + // the executor it is about to pick, and refusing would fail a download over + // a change that probably does not affect it. What must not happen is the + // durable part: nothing is written, so the next caller asks the node again + // rather than reading this answer for a minute. return append(tonemap.Capabilities(nil), entry.capabilities...), nil } +// capabilityInvalidationsFor reports how many times a node's inventory has been +// dropped. +func (p *NodeAwarePreparer) capabilityInvalidationsFor(nodeURL string) uint64 { + p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + return p.capabilityInvalidations[nodeURL] +} + // ToneMapCapabilityTimeout returns the complete cold-node capability budget // used when pooled nodes are the only eligible tone-map executors. func (p *NodeAwarePreparer) ToneMapCapabilityTimeout() time.Duration { @@ -486,25 +511,110 @@ func (p *NodeAwarePreparer) ToneMapCapabilityTimeout() time.Duration { } func (p *NodeAwarePreparer) remoteToneMapProbeTimeout(nodeURL string) time.Duration { - nodeURL = strings.TrimRight(nodeURL, "/") + nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() timeout := p.capabilities[nodeURL].probeRequestTimeout p.capabilityMu.Unlock() - if timeout > 0 { - return timeout + // The larger of what was learned from this node and what it currently + // describes; neither dominates. + // + // A learned budget is preserved across failures on purpose, so a cold retry + // is not cut short by a fallback — but it describes the node as it was, and + // an operator who widens hw_device_override leaves one behind that prices a + // smaller device set than the node now walks. Every retry would be canceled + // at that deadline, and since a budget is only ever learned from a read that + // completes, nothing would replace it. + // + // What the node currently describes is its own stored report and its own + // override — not the cluster setting, which says nothing about a node + // overridden onto four devices. Pricing four at the cluster's one cancels the + // matrix mid-walk, which drops the node from the capability map and sends the + // download local, or fails it outright where local fallback is off. + var node *nodepool.Node + if lookup, ok := p.planner.(transcodeNodeLookup); ok { + if found, ok := lookup.TranscodeNodeByURL(nodeURL); ok { + node = found + } } - cfg := p.config() - if cfg == nil { - return tonemap.ProbeRequestTimeout("", "") + hwAccel, hwDevice := "", "" + if cfg := p.config(); cfg != nil { + hwAccel, hwDevice = cfg.Playback.HWAccel, cfg.Playback.HWDevice + } + // The whole capability read, not just its tone-map half: the node runs a + // hardware walk first, and that walk scales with the device set it walks. + cold := playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(hwAccel), + node.EffectiveHWDevice(hwDevice), + playback.CapabilityRequestTimeout(hwAccel, hwDevice), + ) + if cold > timeout { + return cold + } + return timeout +} + +// transcodeNodeLookup resolves the pooled record behind a transcode node URL, +// which carries that node's stored capability report and its acceleration +// override. Optional, like the planner's other capabilities: without it this +// path falls back to the cluster-wide setting. *nodepool.Planner implements it. +type transcodeNodeLookup interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) +} + +// InvalidateNodeCapabilities drops one node's cached inventory so the next +// prepared download reads it again. +// +// It exists for the same reason the playback-v3 cache has one: an operator +// changing a node's acceleration policy, or the health sweep noticing the node's +// capability hash move, makes this cache wrong the moment it lands — and a +// download planned from it selects the node for a tone-map executor it no longer +// has, so the reconfigured worker rejects the recipe or the download falls back +// locally for no reason. A minute of TTL is a minute of that. +// +// The learned probe budget survives, exactly as it does across a failure: how +// long this node takes to answer has not changed, and the read the invalidation +// triggers is the cold one that most needs the real number. +func (p *NodeAwarePreparer) InvalidateNodeCapabilities(nodeURL string) { + if p == nil || nodeURL == "" { + return + } + nodeURL = nodepool.NormalizeNodeURL(nodeURL) + p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + // Counted whether or not anything is cached. A cold cache is the case where + // dropping an entry does nothing and a fetch is most likely to be in flight: + // the invalidation that follows a policy edit arrives while planning is + // already asking the node, and without a mark that fetch's answer would be + // installed after the edit as though it described the node afterwards. + if p.capabilityInvalidations == nil { + p.capabilityInvalidations = make(map[string]uint64) + } + p.capabilityInvalidations[nodeURL]++ + entry, ok := p.capabilities[nodeURL] + if !ok { + return } - return tonemap.ProbeRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice) + p.capabilities[nodeURL] = remoteToneMapCapabilities{probeRequestTimeout: entry.probeRequestTimeout} } // cacheToneMapCapabilityFailure negatively caches an unreachable or invalid // node briefly so repeated artifact planning does not amplify the failure. -func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, err error) { - nodeURL = strings.TrimRight(nodeURL, "/") +// +// Fenced on the same invalidation count a successful result is, and for a +// sharper reason: a negative entry does not merely go stale, it takes the node +// out of planning entirely for its TTL. A fetch that failed because the node +// was mid-reload — which is exactly what a policy edit causes — would otherwise +// keep downloads off the node it was just reconfigured for, falling back +// locally or failing outright where local fallback is off, after the change +// that would have fixed it had already landed. +func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, generation uint64, err error) { + nodeURL = nodepool.NormalizeNodeURL(nodeURL) p.capabilityMu.Lock() + defer p.capabilityMu.Unlock() + if p.capabilityInvalidations[nodeURL] != generation { + return + } if p.capabilities == nil { p.capabilities = make(map[string]remoteToneMapCapabilities) } @@ -515,7 +625,6 @@ func (p *NodeAwarePreparer) cacheToneMapCapabilityFailure(nodeURL string, err er expiresAt: time.Now().Add(remoteToneMapCapabilityErrorTTL), probeRequestTimeout: probeRequestTimeout, } - p.capabilityMu.Unlock() } func remotePreparedArtifact(node *nodepool.Node, result downloadprepare.Result) PreparedArtifact { diff --git a/internal/downloads/remote_preparer_test.go b/internal/downloads/remote_preparer_test.go index ea32555aa..e0be3aaf6 100644 --- a/internal/downloads/remote_preparer_test.go +++ b/internal/downloads/remote_preparer_test.go @@ -759,7 +759,7 @@ func TestNodeAwarePreparerUsesTargetNodeProbeBudget(t *testing.T) { preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) preparer.probeClient = &http.Client{Transport: transport} - if got, want := preparer.remoteToneMapProbeTimeout(remote.URL), tonemap.ProbeRequestTimeout(tonemap.BackendQSV, "/central/device"); got != want { + if got, want := preparer.remoteToneMapProbeTimeout(remote.URL), playback.CapabilityRequestTimeout(tonemap.BackendQSV, "/central/device"); got != want { t.Fatalf("unknown-node probe timeout = %s, want configured probe budget %s", got, want) } if _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL); err != nil { @@ -808,7 +808,14 @@ func TestNormalizeRemoteToneMapProbeTimeout(t *testing.T) { {name: "missing", want: 5 * time.Second}, {name: "too small", millis: time.Second.Milliseconds(), want: 5 * time.Second}, {name: "node specific", millis: (161 * time.Second).Milliseconds(), want: 161 * time.Second}, - {name: "too large", millis: (10 * time.Minute).Milliseconds(), want: 5 * time.Minute}, + { + // The ceiling is derived from the probe formula, not picked, so the + // expectation is too — a round number was already below what a + // nine-device node legitimately advertises. + name: "too large", + millis: (24 * time.Hour).Milliseconds(), + want: playback.MaxCapabilityRequestTimeout(), + }, } { t.Run(test.name, func(t *testing.T) { if got := normalizeRemoteToneMapProbeTimeout(test.millis); got != test.want { @@ -826,7 +833,7 @@ func TestNodeAwarePreparerCapabilityFailurePreservesNodeProbeBudget(t *testing.T expiresAt: time.Now().Add(-time.Second), } - preparer.cacheToneMapCapabilityFailure(nodeURL, context.DeadlineExceeded) + preparer.cacheToneMapCapabilityFailure(nodeURL, preparer.capabilityInvalidationsFor(nodeURL), context.DeadlineExceeded) if got, want := preparer.remoteToneMapProbeTimeout(nodeURL), 161*time.Second; got != want { t.Fatalf("probe timeout after transient failure = %s, want preserved node budget %s", got, want) @@ -836,7 +843,7 @@ func TestNodeAwarePreparerCapabilityFailurePreservesNodeProbeBudget(t *testing.T func TestNodeAwarePreparerDerivesProbeBudgetWithoutCachedAdvertisement(t *testing.T) { preparer := NewNodeAwarePreparer(nil, nil, nil) - if got, want := preparer.remoteToneMapProbeTimeout("https://node.example"), tonemap.ProbeRequestTimeout("", ""); got != want { + if got, want := preparer.remoteToneMapProbeTimeout("https://node.example"), playback.CapabilityRequestTimeout("", ""); got != want { t.Fatalf("uncached probe timeout = %s, want derived budget %s", got, want) } } @@ -1020,3 +1027,251 @@ func TestNodeAwarePreparerDoesNotFallBackWhenRecoveryProbeIsIndeterminate(t *tes t.Fatalf("local calls = %d, want 0", local.calls) } } + +// nodeLookupPlanner is a planner that can resolve a node by URL, as the real +// one does. It reserves nothing: these tests only price probes. +type nodeLookupPlanner struct { + node *nodepool.Node +} + +func (p *nodeLookupPlanner) ReserveTranscodeWork(string) (*nodepool.Node, func()) { + return nil, func() {} +} + +func (p *nodeLookupPlanner) TranscodeNode(int) (*nodepool.Node, bool) { return nil, false } + +func (p *nodeLookupPlanner) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if p.node == nil || p.node.URL != nodeURL { + return nil, false + } + return p.node, true +} + +// The cluster setting describes the cluster. A node overridden onto four render +// devices walks four of them cold, and pricing that walk at the cluster's single +// device cancels it partway — which drops the node from the capability map and +// sends the download local, or fails it where local fallback is off. +func TestNodeAwarePreparerColdProbeBudgetFollowsTheNodeOverride(t *testing.T) { + const nodeURL = "https://node.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + cfg := &config.Config{} + cfg.Playback.HWAccel = tonemap.BackendQSV + cfg.Playback.HWDevice = "/dev/dri/renderD128" + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != want { + t.Fatalf("cold probe timeout = %s, want the node's four-device budget %s", got, want) + } + if cluster := playback.CapabilityRequestTimeout(cfg.Playback.HWAccel, cfg.Playback.HWDevice); want <= cluster { + t.Fatalf("fixture is inert: the override budget %s must exceed the cluster's %s", want, cluster) + } +} + +// What the node last advertised is its own measurement of its own matrix, and it +// survives an API restart because it is stored with the report — so where it +// exceeds what this replica can price, it is the answer. +func TestNodeAwarePreparerColdProbeBudgetTakesTheStoredAdvertisement(t *testing.T) { + const nodeURL = "https://node.example" + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, + Capabilities: json.RawMessage(`{"resolved":"qsv","probe_request_timeout_ms":161000}`), + }} + cfg := &config.Config{} + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + + if got, want := preparer.remoteToneMapProbeTimeout(nodeURL), 161*time.Second; got != want { + t.Fatalf("cold probe timeout = %s, want the advertised %s", got, want) + } +} + +// A budget learned before an operator widened the node's device set describes +// the node as it was. Keeping it would cancel every cold retry at the old +// one-device deadline — and a budget is only ever learned from a read that +// completes, so nothing would replace it. +func TestNodeAwarePreparerRepricesALearnedBudgetAfterTheDeviceSetGrows(t *testing.T) { + const nodeURL = "https://node.example" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + backend := tonemap.BackendQSV + planner := &nodeLookupPlanner{node: &nodepool.Node{ + ID: 1, URL: nodeURL, HWAccelOverride: &backend, HWDeviceOverride: &devices, + }} + cfg := &config.Config{} + cfg.Playback.HWAccel = tonemap.BackendQSV + cfg.Playback.HWDevice = "/dev/dri/renderD128" + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + // What the node advertised while it was still on one device. + learned := playback.CapabilityRequestTimeout(backend, "/dev/dri/renderD128") + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + probeRequestTimeout: learned, + expiresAt: time.Now().Add(-time.Second), + } + + want := playback.CapabilityRequestTimeout(backend, devices) + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != want { + t.Fatalf("probe timeout after the override grew = %s, want the four-device %s", got, want) + } + if want <= learned { + t.Fatalf("fixture is inert: the four-device budget %s must exceed the learned %s", want, learned) + } +} + +// The other direction: a node whose own measurement exceeds what this replica +// can price for it keeps that measurement. An API replica has none of the node's +// cards, so its pricing is a floor rather than the truth. +func TestNodeAwarePreparerKeepsALearnedBudgetLargerThanThePolicyPrice(t *testing.T) { + const nodeURL = "https://node.example" + planner := &nodeLookupPlanner{node: &nodepool.Node{ID: 1, URL: nodeURL}} + cfg := &config.Config{} + preparer := NewNodeAwarePreparer(nil, planner, func() *config.Config { return cfg }) + learned := playback.MaxCapabilityRequestTimeout() + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + probeRequestTimeout: learned, + expiresAt: time.Now().Add(-time.Second), + } + + if got := preparer.remoteToneMapProbeTimeout(nodeURL); got != learned { + t.Fatalf("probe timeout = %s, want the node's own larger measurement %s", got, learned) + } +} + +// A policy edit or a capability-hash change makes this cache wrong the moment it +// lands. Left for its TTL, a download planned from it selects the node for a +// tone-map executor it no longer has, and the reconfigured worker rejects the +// recipe or the download falls back locally for no reason. +func TestNodeAwarePreparerInvalidateNodeCapabilitiesDropsTheInventory(t *testing.T) { + const nodeURL = "https://node.example" + preparer := NewNodeAwarePreparer(nil, nil, nil) + preparer.capabilities[nodeURL] = remoteToneMapCapabilities{ + capabilities: tonemap.Capabilities{{Mode: tonemap.ModeHardware, Backend: tonemap.BackendQSV}}, + probeRequestTimeout: 161 * time.Second, + expiresAt: time.Now().Add(time.Minute), + } + + // The stored URL carries a trailing slash the pools have already dropped; it + // still has to reach the entry planning reads. + preparer.InvalidateNodeCapabilities(nodeURL + "/") + + entry := preparer.capabilities[nodeURL] + if len(entry.capabilities) != 0 || !entry.expiresAt.IsZero() { + t.Fatalf("inventory survived the invalidation: %+v", entry) + } + // The budget describes how long the node takes to answer, which a policy + // change does not alter — and the read this invalidation triggers is the + // cold one that most needs the real number. + if entry.probeRequestTimeout != 161*time.Second { + t.Fatalf("probe budget = %s, want the learned 161s preserved", entry.probeRequestTimeout) + } +} + +// A policy edit invalidates while planning is already asking the node. The +// answer in flight describes the node before the edit, so installing it would +// restore the pre-edit inventory for a full TTL — and downloads would keep +// selecting a tone-map executor the reconfigured worker no longer has. +func TestNodeAwarePreparerDoesNotCacheAnOvertakenCapabilityFetch(t *testing.T) { + var hits atomic.Int32 + invalidated := make(chan struct{}) + released := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + // The edit lands while this request is still open. + close(invalidated) + <-released + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + ToneMapCapabilities: tonemap.Capabilities{{ + Mode: tonemap.ModeSoftware, Backend: tonemap.BackendSoftware, + Filter: tonemap.SoftwareFilterBT2390, SourceKinds: []tonemap.SourceKind{tonemap.SourcePQ}, + }}, + }) + })) + defer remote.Close() + cfg := &config.Config{} + cfg.Auth.JWTSecret = "secret" + preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) + + fetched := make(chan error, 1) + go func() { + _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + fetched <- err + }() + <-invalidated + preparer.InvalidateNodeCapabilities(remote.URL) + close(released) + if err := <-fetched; err != nil { + t.Fatalf("the caller waiting on the fetch got an error: %v", err) + } + + // Nothing durable was written, so the next planning pass asks the node again + // rather than reading the overtaken answer. + key := nodepool.NormalizeNodeURL(remote.URL) + preparer.capabilityMu.Lock() + entry, cached := preparer.capabilities[key] + preparer.capabilityMu.Unlock() + if cached && len(entry.capabilities) > 0 { + t.Fatalf("an overtaken fetch repopulated the cache: %+v", entry) + } + if _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL); err != nil { + t.Fatalf("second lookup: %v", err) + } + if hits.Load() != 2 { + t.Fatalf("node was asked %d times, want a second read after the invalidation", hits.Load()) + } +} + +// A negative entry does not merely go stale, it takes the node out of planning +// for its TTL. A fetch failing because the node was mid-reload — exactly what a +// policy edit causes — would otherwise keep downloads off the node that edit had +// just reconfigured, after the change that would have fixed it already landed. +func TestNodeAwarePreparerDoesNotCacheAnOvertakenCapabilityFailure(t *testing.T) { + var hits atomic.Int32 + invalidated := make(chan struct{}) + released := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + close(invalidated) + <-released + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _ = json.NewEncoder(w).Encode(playback.HWAccelInfo{ + ToneMapCapabilities: tonemap.Capabilities{{ + Mode: tonemap.ModeSoftware, Backend: tonemap.BackendSoftware, + Filter: tonemap.SoftwareFilterBT2390, SourceKinds: []tonemap.SourceKind{tonemap.SourcePQ}, + }}, + }) + })) + defer remote.Close() + cfg := &config.Config{} + cfg.Auth.JWTSecret = "secret" + preparer := NewNodeAwarePreparer(nil, nil, func() *config.Config { return cfg }) + + failed := make(chan error, 1) + go func() { + _, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + failed <- err + }() + <-invalidated + preparer.InvalidateNodeCapabilities(remote.URL) + close(released) + if err := <-failed; err == nil { + t.Fatal("the node answered 503 and the caller saw no error") + } + + // The reconfigured node is asked again rather than sitting behind a negative + // entry the invalidation should have outranked. + capabilities, err := preparer.toneMapCapabilitiesForNode(context.Background(), remote.URL) + if err != nil { + t.Fatalf("second lookup: %v", err) + } + if len(capabilities) != 1 { + t.Fatalf("capabilities = %#v, want the reconfigured node's answer", capabilities) + } + if hits.Load() != 2 { + t.Fatalf("node was asked %d times, want a second read after the invalidation", hits.Load()) + } +} diff --git a/internal/httpstream/readfrom_deadline_test.go b/internal/httpstream/readfrom_deadline_test.go index b5cc0beed..dd834fb45 100644 --- a/internal/httpstream/readfrom_deadline_test.go +++ b/internal/httpstream/readfrom_deadline_test.go @@ -8,21 +8,22 @@ import ( "time" ) -// pacedReader delivers src in fixed-size pieces with a pause between each, so a -// transfer takes a predictable wall-clock time regardless of socket buffering. -// It models a slow disk or a rate-limited upstream, which is what makes a single +// pacedReader delivers src at a fixed byte rate, so a transfer takes a +// predictable wall-clock time regardless of the caller's read-buffer size. It +// models a slow disk or a rate-limited upstream, which is what makes a single // zero-copy slice long-lived. type pacedReader struct { remaining int64 piece int64 pause time.Duration + started time.Time + delivered int64 } func (r *pacedReader) Read(p []byte) (int, error) { if r.remaining <= 0 { return 0, io.EOF } - time.Sleep(r.pause) n := r.piece if n > int64(len(p)) { n = int64(len(p)) @@ -30,6 +31,15 @@ func (r *pacedReader) Read(p []byte) (int, error) { if n > r.remaining { n = r.remaining } + // ReaderFrom implementations choose their own buffer size. Scale the pause + // to the bytes actually returned instead of assuming every call accepts a + // full piece; otherwise a smaller platform buffer silently slows the reader + // and consumes the deadline margin this test is meant to control. + if r.started.IsZero() { + r.started = time.Now() + } + r.delivered += n + time.Sleep(time.Until(r.started.Add(time.Duration(r.delivered) * r.pause / time.Duration(r.piece)))) r.remaining -= n return int(n), nil } @@ -55,10 +65,11 @@ func sliceDuration() time.Duration { // despite making continuous progress. func TestReadFromRollsDeadlineBetweenSlices(t *testing.T) { slice := sliceDuration() - // Window comfortably exceeds one slice but is far shorter than the whole - // transfer, so only per-slice bumping can carry it to completion. - window := slice * 3 - total := readFromChunk * 3 + // Leave enough headroom for scheduler jitter while keeping the whole + // transfer longer than the window, so only per-slice bumping can carry it to + // completion. + window := slice * 6 + total := readFromChunk * 8 done := make(chan error, 1) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -96,8 +107,8 @@ func TestReadFromRollsDeadlineBetweenSlices(t *testing.T) { // stay reproducible so nobody restores a large slice without noticing. func TestOversizedReadFromSliceIsReaped(t *testing.T) { slice := sliceDuration() - window := slice * 3 - oversized := readFromChunk * 8 // one slice ≈ 8x slice duration >> window + window := slice * 6 + oversized := readFromChunk * 12 // one slice ≈ 12x slice duration >> window done := make(chan error, 1) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -159,8 +170,8 @@ func TestReadFromChunkAllowsSlowClients(t *testing.T) { // sustaining the documented floor rate was reaped despite never stalling. func TestReadFromRollsDeadlineUnderProductionStep(t *testing.T) { slice := sliceDuration() - window := slice * 3 - total := readFromChunk * 3 + window := slice * 6 + total := readFromChunk * 8 done := make(chan error, 1) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -197,7 +208,7 @@ func TestReadFromRollsDeadlineUnderProductionStep(t *testing.T) { // spend that wait against the window set at construction. func TestReadFromBumpsBeforeTheFirstSlice(t *testing.T) { slice := sliceDuration() - window := slice * 3 + window := slice * 6 total := readFromChunk done := make(chan error, 1) diff --git a/internal/imageutil/imageutil.go b/internal/imageutil/imageutil.go index 96a30fbcc..52eac27f7 100644 --- a/internal/imageutil/imageutil.go +++ b/internal/imageutil/imageutil.go @@ -157,24 +157,42 @@ func GenerateSquareVariants(data []byte, sizes []int) (*VariantResult, error) { // Thumbhash computes a base64-encoded thumbhash from raw image bytes. // The image is scaled to max 100x100 before hashing. +// +// The downscale happens in libvips before the Go-side decode, not after: +// decoding a full provider original in Go materializes the whole raster on +// the heap — well over a hundred MiB for a large poster — per concurrent +// caller, while vips shrinks it to thumbhashSourceDimension with +// shrink-on-load and hands Go a raster of a few KiB. The pure-Go decode of +// the raw bytes remains as the fallback for anything vips cannot parse. +// +// Changing this pipeline changes the emitted hash bytes for a given image. +// Stored thumbhashes remain valid placeholders, and the one site that +// compares hashes for equality (ebook scan cover change detection) stores +// the freshly computed hash whenever it re-caches, so a pipeline change +// costs one re-cache per scan-covered ebook and then converges. func Thumbhash(data []byte) (string, error) { - img, _, err := image.Decode(bytes.NewReader(data)) + img, err := decodeThumbhashSource(data) if err != nil { - normalized, normalizeErr := normalizeThumbhashSource(data) - if normalizeErr != nil { - return "", fmt.Errorf("imageutil: decode for thumbhash: %w", err) - } - img, _, err = image.Decode(bytes.NewReader(normalized)) - if err != nil { - return "", fmt.Errorf("imageutil: decode normalized thumbhash source: %w", err) - } + return "", err } - - scaled := scaleImage(img, 100) + scaled := scaleImage(img, thumbhashSourceDimension) hashBytes := thumbhash.EncodeImage(scaled) return base64.StdEncoding.EncodeToString(hashBytes), nil } +func decodeThumbhashSource(data []byte) (image.Image, error) { + if normalized, err := normalizeThumbhashSource(data); err == nil { + if img, _, err := image.Decode(bytes.NewReader(normalized)); err == nil { + return img, nil + } + } + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("imageutil: decode for thumbhash: %w", err) + } + return img, nil +} + func normalizeThumbhashSource(data []byte) ([]byte, error) { img := bimg.NewImage(data) size, err := img.Size() diff --git a/internal/imageutil/imageutil_test.go b/internal/imageutil/imageutil_test.go new file mode 100644 index 000000000..049eec264 --- /dev/null +++ b/internal/imageutil/imageutil_test.go @@ -0,0 +1,91 @@ +package imageutil + +import ( + "bytes" + "image" + "image/color" + "image/jpeg" + "runtime" + "testing" +) + +// largeTestJPEG encodes a width×height gradient so the bytes are a real, +// decodable JPEG of meaningful dimensions rather than a fixture file. +func largeTestJPEG(t testing.TB, width, height int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.SetNRGBA(x, y, color.NRGBA{ + R: uint8(x * 255 / width), + G: uint8(y * 255 / height), + B: uint8((x + y) * 255 / (width + height)), + A: 255, + }) + } + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil { + t.Fatalf("encode test jpeg: %v", err) + } + return buf.Bytes() +} + +func TestThumbhashDeterministic(t *testing.T) { + data := largeTestJPEG(t, 1200, 800) + first, err := Thumbhash(data) + if err != nil { + t.Fatalf("Thumbhash: %v", err) + } + if first == "" { + t.Fatal("Thumbhash returned empty hash") + } + second, err := Thumbhash(data) + if err != nil { + t.Fatalf("Thumbhash (second call): %v", err) + } + if first != second { + t.Fatalf("Thumbhash not deterministic: %q vs %q", first, second) + } +} + +func TestThumbhashRejectsGarbage(t *testing.T) { + if _, err := Thumbhash([]byte("not an image at all")); err == nil { + t.Fatal("Thumbhash accepted garbage input") + } +} + +// TestThumbhashDoesNotDecodeFullRasterInGo pins the reason the vips downscale +// runs before the Go decode: hashing must not materialize the original's full +// raster on the Go heap. A 6000×4000 JPEG decodes to ≥36 MiB in pure Go, and +// under tens of concurrent image-cache workers that is an OOM risk; through +// the vips path the Go side only ever decodes a ≤100px PNG. The 15 MiB bound +// is far above the new path's real footprint and far below the old one's. +func TestThumbhashDoesNotDecodeFullRasterInGo(t *testing.T) { + data := largeTestJPEG(t, 6000, 4000) + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + if _, err := Thumbhash(data); err != nil { + t.Fatalf("Thumbhash: %v", err) + } + var after runtime.MemStats + runtime.ReadMemStats(&after) + + allocated := after.TotalAlloc - before.TotalAlloc + if allocated > 15<<20 { + t.Fatalf("Thumbhash allocated %d bytes on the Go heap; the full raster is being decoded in Go", allocated) + } +} + +func BenchmarkThumbhashLargeJPEG(b *testing.B) { + data := largeTestJPEG(b, 6000, 4000) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := Thumbhash(data); err != nil { + b.Fatalf("Thumbhash: %v", err) + } + } +} diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 2dfdcc164..bba0a1a33 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -145,6 +145,24 @@ type compatTranscodeNodeHealth interface { TranscodeNodeHealthy(nodeURL string) bool } +// compatTranscodeNodeLookup resolves the pooled record behind a transcode node +// URL, which carries that node's own acceleration override. Optional, like the +// enumerators above: without it dispatch falls back to the cluster-wide +// acceleration setting. *nodepool.Planner implements it. +type compatTranscodeNodeLookup interface { + TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) +} + +// compatProxyNodeLookup resolves the pooled record behind a proxy node URL, so +// capability-budget pricing can read a proxy's stored report and overrides — +// the transcode lookup above answers nothing for a proxy URL, and pricing +// proxies from the cluster policy alone undersizes a sweep whose slowest +// member is a cold proxy. Optional for the same reason. *nodepool.Planner +// implements it. +type compatProxyNodeLookup interface { + ProxyNodeByURL(nodeURL string) (*nodepool.Node, bool) +} + // transcodeStreamDetailsSetter is implemented by the native SessionManager. // Optional (like sessionStarterContext) so lightweight test fakes don't have // to; without it the session keeps transport-level defaults only. @@ -443,14 +461,14 @@ func resolveCompatToneMapRecipeWithPolicy(file *models.MediaFile, capabilities t // localToneMapCapabilities probes the API host's live FFmpeg backend and device. func (h *PlaybackHandler) localToneMapCapabilities(ctx context.Context) (tonemap.Capabilities, error) { - backend := playback.ResolveHWAccelWithFFmpegContext(ctx, h.HWAccel, h.FFmpegPath) - if err := ctx.Err(); err != nil { - return nil, err - } hwDevice := "" if h.cfg != nil { hwDevice = h.cfg.Playback.HWDevice } + backend := playback.ResolveHWAccelWithFFmpegContext(ctx, h.HWAccel, h.FFmpegPath, hwDevice) + if err := ctx.Err(); err != nil { + return nil, err + } probe := tonemap.Probe if h.compatToneMapProbe != nil { probe = h.compatToneMapProbe @@ -565,8 +583,65 @@ func compatSupportsAudioBoost(transformations []playback.TransformationV3) bool return false } +// toneMapCapabilityTimeout bounds one capability sweep. Every caller wraps a +// single deadline around concurrent per-node fetches (plus the local probe), +// so the budget has to cover the slowest node in the fan-out, not a typical +// one: each pooled node is priced the way the v3 and download paths price a +// cold read — ColdCapabilityRequestTimeout over its stored report and its +// effective override — and the sweep takes the maximum, with the fixed +// fallback as the floor for the local probe and for nodes this process cannot +// resolve. Without the derivation, a node with enough configured devices to +// out-price two minutes was canceled while still inside its own advertised +// probe budget, and HDR or audio-boost planning excluded it for no fault. func (h *PlaybackHandler) toneMapCapabilityTimeout() time.Duration { - return compatRemoteNodeProbeFallbackTimeout + hwDevice := "" + if h.cfg != nil { + hwDevice = h.cfg.Playback.HWDevice + } + // The local probe runs under the same deadline, priced by the cluster + // policy; this is also the whole answer when no pool is reachable. + budget := playback.ColdCapabilityRequestTimeout(nil, h.HWAccel, hwDevice, compatRemoteNodeProbeFallbackTimeout) + + // A URL its lookup cannot resolve — a record that left the pool, or a + // planner without the lookup at all — prices as a nil node: cluster policy + // over the fallback, which is the pre-derivation behavior. + price := func(node *nodepool.Node) { + cold := playback.ColdCapabilityRequestTimeout( + node.StoredCapabilities(), + node.EffectiveHWAccel(h.HWAccel), + node.EffectiveHWDevice(hwDevice), + compatRemoteNodeProbeFallbackTimeout, + ) + if cold > budget { + budget = cold + } + } + if enumerator, ok := h.NodePlanner.(compatTranscodeNodeEnumerator); ok { + lookup, canLookup := h.NodePlanner.(compatTranscodeNodeLookup) + for _, nodeURL := range enumerator.TranscodeNodeURLs() { + var node *nodepool.Node + if canLookup { + node, _ = lookup.TranscodeNodeByURL(nodeURL) + } + price(node) + } + } + // Proxy nodes answer the audio-boost recipe sweep under this same + // deadline, resolved through their own pool: the transcode lookup answers + // nothing for a proxy URL, and a cold proxy whose report or override + // out-prices the fallback would otherwise be canceled inside its own + // budget and dropped from planning. + if enumerator, ok := h.NodePlanner.(compatProxyNodeEnumerator); ok { + lookup, canLookup := h.NodePlanner.(compatProxyNodeLookup) + for _, nodeURL := range enumerator.ProxyNodeURLs() { + var node *nodepool.Node + if canLookup { + node, _ = lookup.ProxyNodeByURL(nodeURL) + } + price(node) + } + } + return budget } func (h *PlaybackHandler) remoteTranscodeStartTimeout(request transcodenode.TranscodeStartRequest, nodeProbeTimeoutMillis int64) time.Duration { @@ -1037,19 +1112,20 @@ func (h *PlaybackHandler) buildProxyRedirectURL( switch method { case string(playback.PlayDirect): - return proxyNode.URL + "/stream/direct/" + token, nil + return nodepool.NodeEndpoint(proxyNode.ClientURL(), "/stream/direct/"+token), nil case string(playback.PlayRemux): remuxPath := "/stream/remux/" if claims.PlayMethod == streamtoken.PlayMethodAudioDownmixRemux { remuxPath = "/stream/remux/audio-v2/" } - redirectURL := proxyNode.URL + remuxPath + token + redirectURL := nodepool.NodeEndpoint(proxyNode.ClientURL(), remuxPath+token) if seekSeconds > 0 { redirectURL += "?seek=" + strconv.FormatFloat(seekSeconds, 'f', -1, 64) } return redirectURL, nil case string(playback.PlayTranscode): - return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8?" + playback.SourceTimelineQueryParam + "=1", nil + return nodepool.NodeEndpoint(proxyNode.ClientURL(), + "/stream/transcode/"+token+"/master.m3u8?"+playback.SourceTimelineQueryParam+"=1"), nil default: return "", fmt.Errorf("unsupported proxy method %q", method) } @@ -1076,6 +1152,26 @@ func clampSeekSeconds(seekSeconds float64, sources []PlaybackMediaSource) float6 return seekSeconds } +// remoteDispatchHWAccel picks the acceleration backend to name in a start +// request to one node: that node's own hw_accel_override when it carries one, +// and otherwise the cluster-wide setting this host runs under. A node under an +// override resolves to its own answer regardless, so naming it keeps the +// request and what runs in agreement. The cluster value passes through +// untouched otherwise — "auto" included, because the node honors a named +// backend verbatim and must be left to resolve it against live hardware. This +// mirrors the v1 dispatch path in internal/api/handlers/playback_v3.go. +func (h *PlaybackHandler) remoteDispatchHWAccel(nodeURL string) string { + lookup, ok := h.NodePlanner.(compatTranscodeNodeLookup) + if !ok { + return h.HWAccel + } + node, found := lookup.TranscodeNodeByURL(nodeURL) + if !found { + return h.HWAccel + } + return node.EffectiveHWAccel(h.HWAccel) +} + // startRemoteTranscode submits a frozen compatibility recipe to a selected node. func (h *PlaybackHandler) startRemoteTranscode( ctx context.Context, @@ -1221,7 +1317,7 @@ func (h *PlaybackHandler) startRemoteTranscodeWithToneMapMode( TargetCodecVideo: compatTargetVideoCodec, TargetCodecAudio: compatTargetAudioCodec, SegmentDuration: segmentDuration, - HWAccel: h.HWAccel, + HWAccel: h.remoteDispatchHWAccel(transcodeNodeURL), AudioTrackIndex: compatAudioTrackIndexOrDefault(source), SourceAudioChannels: compatSourceAudioChannels(source), TotalDuration: float64(source.Version.Duration), @@ -1263,7 +1359,7 @@ func (h *PlaybackHandler) startRemoteTranscodeWithToneMapMode( } requestCtx, cancel := context.WithTimeout(ctx, h.remoteTranscodeStartTimeout(request, nodeProbeTimeoutMillis)) defer cancel() - httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, transcodeNodeURL+"/transcode/start", strings.NewReader(string(body))) + httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodepool.NodeEndpoint(transcodeNodeURL, "/transcode/start"), strings.NewReader(string(body))) if err != nil { return transcodenode.TranscodeStartResponse{}, 0, false, fmt.Errorf("build transcode request: %w", logredact.SanitizeURLError(err)) } diff --git a/internal/jellycompat/playback_scrobble_test.go b/internal/jellycompat/playback_scrobble_test.go index e9e923772..67dcf9006 100644 --- a/internal/jellycompat/playback_scrobble_test.go +++ b/internal/jellycompat/playback_scrobble_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "runtime" "strconv" "strings" "sync" @@ -671,6 +672,10 @@ func TestPositionlessLateStopPreservesAndDeliversPendingFallback(t *testing.T) { scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} h.WatchScrobbler = scrobbler h.terminalFallbackDelay = time.Hour + // Receiving the scrobble event is not enough to read the store on: the + // release that sets TerminalFallbackSent runs after the dispatch returns. + releases := make(chan struct{}, 2) + h.playbackStore = &terminalReleaseObservingStore{CompatPlaybackStore: store, releases: releases} store.Put(PlaybackSession{ ID: "play-1", CompatToken: "token-1", @@ -711,6 +716,11 @@ func TestPositionlessLateStopPreservesAndDeliversPendingFallback(t *testing.T) { case <-time.After(time.Second): t.Fatal("timed out waiting for preserved terminal fallback") } + select { + case <-releases: + case <-time.After(time.Second): + t.Fatal("timed out waiting for fallback delivery lease release") + } terminal, ok = store.GetFinalizable("play-1", "token-1") if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { t.Fatalf("delivered fallback state = ok=%v session=%+v", ok, terminal) @@ -749,9 +759,8 @@ func TestStoppedScrobbleQueueFailureRetainsAndRetriesTerminalEvent(t *testing.T) case <-time.After(2 * time.Second): t.Fatal("timed out waiting for terminal queue retry") } - if _, ok := handler.playbackStore.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("authoritative terminal event remained after successful retry") - } + awaitTerminalCompleted(t, handler.playbackStore, "play-1", "token-1", + "authoritative terminal event remained after successful retry") } func TestStoppedScrobbleRestagesAfterTerminalPersistenceFailure(t *testing.T) { @@ -795,9 +804,8 @@ func TestStoppedScrobbleRestagesAfterTerminalPersistenceFailure(t *testing.T) { if calls := flakyStore.calls(); calls < 2 { t.Fatalf("stage calls = %d, want persistence retry", calls) } - if _, ok := flakyStore.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("restaged authoritative event remained after delivery") - } + awaitTerminalCompleted(t, flakyStore, "play-1", "token-1", + "restaged authoritative event remained after delivery") } func TestStoppedScrobblePreservesExplicitZeroPosition(t *testing.T) { @@ -853,9 +861,7 @@ func TestTerminalScrobbleRecoveryDeliversPersistedEventAfterRestart(t *testing.T case <-time.After(time.Second): t.Fatal("timed out waiting for recovered terminal event") } - if _, ok := store.GetFinalizable("play-1", "token-1"); ok { - t.Fatal("recovered authoritative event remained pending") - } + awaitTerminalCompleted(t, store, "play-1", "token-1", "recovered authoritative event remained pending") } func TestTerminalScrobbleRecoveryWaitsForConfirmedProviderStop(t *testing.T) { @@ -1178,3 +1184,29 @@ func TestTeardownStillCleansLocalPlaybackAfterAnotherCallerClaimsStop(t *testing t.Fatalf("losing teardown emitted provider event: %+v", scrobbler.calls) } } + +// awaitTerminalCompleted waits for the delivery path to retire a terminal event. +// +// The mirror of awaitTerminalFallbackSent, and racy for the same reason: +// CompleteTerminal runs after the dispatch that puts the event on the scrobbler +// channel, so a test that reads the store the instant it receives can see the +// entry still present. Waiting for its absence removes the ordering dependence. +func awaitTerminalCompleted(t *testing.T, store terminalFinalizableStore, id, token, message string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + if _, ok := store.GetFinalizable(id, token); !ok { + return + } + if time.Now().After(deadline) { + t.Fatal(message) + } + runtime.Gosched() + } +} + +// terminalFinalizableStore is the one method these waits need, so they work on +// the concrete store and on the handler's interface field alike. +type terminalFinalizableStore interface { + GetFinalizable(id, compatToken string) (*PlaybackSession, bool) +} diff --git a/internal/jellycompat/process_token_darwin.go b/internal/jellycompat/process_token_darwin.go new file mode 100644 index 000000000..6f3c665d9 --- /dev/null +++ b/internal/jellycompat/process_token_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package jellycompat + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func processToken(pid int) string { + if pid <= 0 { + return "" + } + process, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil || process.Proc.P_pid != int32(pid) { + return "" + } + startedAt := process.Proc.P_starttime + return fmt.Sprintf("%d:%d", startedAt.Sec, startedAt.Usec) +} diff --git a/internal/jellycompat/process_token_linux.go b/internal/jellycompat/process_token_linux.go new file mode 100644 index 000000000..75daf5272 --- /dev/null +++ b/internal/jellycompat/process_token_linux.go @@ -0,0 +1,30 @@ +//go:build linux + +package jellycompat + +import ( + "os" + "path/filepath" + "strconv" + "strings" +) + +func processToken(pid int) string { + if pid <= 0 { + return "" + } + data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + return "" + } + stat := string(data) + commEnd := strings.LastIndex(stat, ") ") + if commEnd == -1 || commEnd+2 >= len(stat) { + return "" + } + fields := strings.Fields(stat[commEnd+2:]) + if len(fields) < 20 { + return "" + } + return fields[19] +} diff --git a/internal/jellycompat/remote_dispatch_hwaccel_test.go b/internal/jellycompat/remote_dispatch_hwaccel_test.go new file mode 100644 index 000000000..ba8478bad --- /dev/null +++ b/internal/jellycompat/remote_dispatch_hwaccel_test.go @@ -0,0 +1,105 @@ +package jellycompat + +import ( + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/nodepool" +) + +// nodeLookupPlannerStub is a planner that can also resolve a pooled node from +// its URL, which is what carries the node's own acceleration override. +type nodeLookupPlannerStub struct { + node *nodepool.Node +} + +func (s *nodeLookupPlannerStub) PlanSession(string, string, bool, int) nodepool.Plan { + return nodepool.Plan{TranscodeNode: s.node} +} + +func (s *nodeLookupPlannerStub) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + if s.node == nil || s.node.URL != nodeURL { + return nil, false + } + return s.node, true +} + +// planner without a lookup, standing in for a test fake or a future planner +// that cannot resolve nodes. +type plainPlannerStub struct{} + +func (plainPlannerStub) PlanSession(string, string, bool, int) nodepool.Plan { return nodepool.Plan{} } + +// The Jellyfin surface dispatches to the same nodes as the native API, so it +// has to name the same backend: the node's own override when it has one, and +// the cluster value untouched otherwise. +func TestRemoteDispatchHWAccelPrefersTheNodesOverride(t *testing.T) { + override := func(value string) *string { return &value } + overriddenNode := func(cluster string, node *nodepool.Node) *PlaybackHandler { + return &PlaybackHandler{HWAccel: cluster, NodePlanner: &nodeLookupPlannerStub{node: node}} + } + tests := []struct { + name string + handler *PlaybackHandler + nodeURL string + want string + }{ + { + name: "node overridden to software wins over a qsv cluster", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", HWAccelOverride: override("none")}), + nodeURL: "http://node-1", + want: "none", + }, + { + name: "an override beats the stale report it contradicts", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", + HWAccelOverride: override("none"), + Capabilities: json.RawMessage(`{"resolved":"qsv"}`)}), + nodeURL: "http://node-1", + want: "none", + }, + { + name: "a node with no override keeps the cluster value", + handler: overriddenNode("qsv", &nodepool.Node{URL: "http://node-1"}), + nodeURL: "http://node-1", + want: "qsv", + }, + { + // The node re-resolves auto against live hardware at session start; + // a report from its last snapshot must not stand in for that. + name: "auto reaches the node even when the last report says otherwise", + handler: overriddenNode("auto", &nodepool.Node{ + URL: "http://node-1", Capabilities: json.RawMessage(`{"resolved":"none"}`)}), + nodeURL: "http://node-1", + want: "auto", + }, + { + name: "unknown node keeps the cluster value", + handler: overriddenNode("qsv", &nodepool.Node{ + URL: "http://node-1", HWAccelOverride: override("none")}), + nodeURL: "http://node-2", + want: "qsv", + }, + { + name: "planner without a lookup keeps the cluster value", + handler: &PlaybackHandler{HWAccel: "qsv", NodePlanner: plainPlannerStub{}}, + nodeURL: "http://node-1", + want: "qsv", + }, + { + name: "no planner at all keeps the cluster value", + handler: &PlaybackHandler{HWAccel: "qsv"}, + nodeURL: "http://node-1", + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.handler.remoteDispatchHWAccel(test.nodeURL); got != test.want { + t.Fatalf("remoteDispatchHWAccel(%q) = %q, want %q", test.nodeURL, got, test.want) + } + }) + } +} diff --git a/internal/jellycompat/remote_transcode_reconstruct_test.go b/internal/jellycompat/remote_transcode_reconstruct_test.go index 9e1f602d4..200d10593 100644 --- a/internal/jellycompat/remote_transcode_reconstruct_test.go +++ b/internal/jellycompat/remote_transcode_reconstruct_test.go @@ -1060,8 +1060,11 @@ func TestRemoteTranscodeStartTimeoutCoversColdProbePreflightAndReadiness(t *test if got := handler.remoteTranscodeStartTimeout(request, (137 * time.Second).Milliseconds()); got != want { t.Fatalf("remote transcode start timeout = %v, want %v", got, want) } - maxWant := 5*time.Minute + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout - if got := handler.remoteTranscodeStartTimeout(request, (10 * time.Minute).Milliseconds()); got != maxWant { + // Still bounded — the advertisement comes off the wire from a worker — but + // at the ceiling the probe formula produces, not a round number a real + // nine-device node already exceeds. + maxWant := playback.MaxCapabilityRequestTimeout() + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout + if got := handler.remoteTranscodeStartTimeout(request, (24 * time.Hour).Milliseconds()); got != maxWant { t.Fatalf("bounded remote transcode start timeout = %v, want %v", got, maxWant) } fallbackWant := compatRemoteNodeProbeFallbackTimeout + playback.ManifestStartupTimeout + tonemap.SourcePreflightTimeout(100) + transcodenode.TranscodeStartReadinessTimeout diff --git a/internal/jellycompat/tone_map_capability_timeout_test.go b/internal/jellycompat/tone_map_capability_timeout_test.go new file mode 100644 index 000000000..8c173e9cb --- /dev/null +++ b/internal/jellycompat/tone_map_capability_timeout_test.go @@ -0,0 +1,147 @@ +package jellycompat + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// compatTimeoutPlanner is the smallest planner that lets +// toneMapCapabilityTimeout enumerate and resolve pooled nodes. +type compatTimeoutPlanner struct { + transcodes []string + proxies []string + nodes map[string]*nodepool.Node + proxyNodes map[string]*nodepool.Node +} + +func (p compatTimeoutPlanner) PlanSession(string, string, bool, int) nodepool.Plan { + return nodepool.Plan{} +} + +func (p compatTimeoutPlanner) TranscodeNodeURLs() []string { return p.transcodes } + +func (p compatTimeoutPlanner) ProxyNodeURLs() []string { return p.proxies } + +func (p compatTimeoutPlanner) TranscodeNodeByURL(nodeURL string) (*nodepool.Node, bool) { + node, ok := p.nodes[nodeURL] + return node, ok +} + +func (p compatTimeoutPlanner) ProxyNodeByURL(nodeURL string) (*nodepool.Node, bool) { + node, ok := p.proxyNodes[nodeURL] + return node, ok +} + +// The capability sweep wraps one deadline around concurrent fetches of every +// pooled node, so the budget must cover the slowest node in the fan-out. A +// node whose stored report advertises a probe budget past the fixed fallback +// used to be canceled mid-probe while still inside that budget, and HDR or +// audio-boost planning then excluded it. +func TestToneMapCapabilityTimeoutCoversTheSlowestPooledNode(t *testing.T) { + advertisedMillis := (4 * time.Minute).Milliseconds() + report := json.RawMessage(fmt.Sprintf(`{"probe_request_timeout_ms":%d}`, advertisedMillis)) + + // Derivation-guard rather than a constant: the expected budget is what the + // shared pricing rule answers for this report, asserted to actually exceed + // the fallback so the test cannot pass vacuously if the fixture stops + // out-pricing it. + want := playback.ColdCapabilityRequestTimeout(report, "", "", compatRemoteNodeProbeFallbackTimeout) + if want <= compatRemoteNodeProbeFallbackTimeout { + t.Fatalf("fixture no longer out-prices the fallback: got %v, fallback %v", want, compatRemoteNodeProbeFallbackTimeout) + } + + handler := &PlaybackHandler{ + NodePlanner: compatTimeoutPlanner{ + transcodes: []string{"http://cheap:8082", "http://slow:8082"}, + nodes: map[string]*nodepool.Node{ + // The cheap node advertises nothing; only the slow one raises + // the sweep, which is what makes the answer a maximum. + "http://cheap:8082": {URL: "http://cheap:8082"}, + "http://slow:8082": {URL: "http://slow:8082", Capabilities: report}, + }, + }, + } + + if got := handler.toneMapCapabilityTimeout(); got != want { + t.Fatalf("toneMapCapabilityTimeout() = %v, want the slowest node's cold budget %v", got, want) + } +} + +// A node's own acceleration override prices its probe walk even before any +// refetch stores a report for it — the moment the override is saved is exactly +// when the stored figure is most wrong. +func TestToneMapCapabilityTimeoutPricesANodeOverride(t *testing.T) { + node := &nodepool.Node{URL: "http://wide:8082"} + accel := "qsv" + devices := "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131" + node.HWAccelOverride = &accel + node.HWDeviceOverride = &devices + + want := playback.ColdCapabilityRequestTimeout(nil, accel, devices, compatRemoteNodeProbeFallbackTimeout) + + handler := &PlaybackHandler{ + NodePlanner: compatTimeoutPlanner{ + transcodes: []string{node.URL}, + nodes: map[string]*nodepool.Node{node.URL: node}, + }, + } + + if got := handler.toneMapCapabilityTimeout(); got != want { + t.Fatalf("toneMapCapabilityTimeout() = %v, want the override-priced budget %v", got, want) + } +} + +// A planner that cannot enumerate nodes — or none at all — keeps the +// pre-derivation behavior: the cluster policy over the fixed fallback. +func TestToneMapCapabilityTimeoutFallsBackWithoutAPool(t *testing.T) { + want := playback.ColdCapabilityRequestTimeout(nil, "", "", compatRemoteNodeProbeFallbackTimeout) + + handler := &PlaybackHandler{cfg: &config.Config{}} + if got := handler.toneMapCapabilityTimeout(); got != want { + t.Fatalf("toneMapCapabilityTimeout() with no planner = %v, want %v", got, want) + } +} + +// Proxy nodes answer the audio-boost sweep under the same deadline, resolved +// through their own pool: a proxy whose stored report out-prices the fallback +// raises the sweep exactly as a transcode node's would. +func TestToneMapCapabilityTimeoutPricesProxyNodesFromTheirRecords(t *testing.T) { + advertisedMillis := (3 * time.Minute).Milliseconds() + report := json.RawMessage(fmt.Sprintf(`{"probe_request_timeout_ms":%d}`, advertisedMillis)) + + want := playback.ColdCapabilityRequestTimeout(report, "", "", compatRemoteNodeProbeFallbackTimeout) + if want <= compatRemoteNodeProbeFallbackTimeout { + t.Fatalf("fixture no longer out-prices the fallback: got %v, fallback %v", want, compatRemoteNodeProbeFallbackTimeout) + } + + handler := &PlaybackHandler{ + NodePlanner: compatTimeoutPlanner{ + proxies: []string{"http://proxy:8083"}, + proxyNodes: map[string]*nodepool.Node{ + "http://proxy:8083": {URL: "http://proxy:8083", Capabilities: report}, + }, + }, + } + if got := handler.toneMapCapabilityTimeout(); got != want { + t.Fatalf("toneMapCapabilityTimeout() = %v, want the proxy's cold budget %v", got, want) + } +} + +// A pooled proxy the lookup cannot resolve still prices at the floor instead +// of shrinking or failing the sweep. +func TestToneMapCapabilityTimeoutCountsUnresolvedProxyNodesAtTheFloor(t *testing.T) { + want := playback.ColdCapabilityRequestTimeout(nil, "", "", compatRemoteNodeProbeFallbackTimeout) + + handler := &PlaybackHandler{ + NodePlanner: compatTimeoutPlanner{proxies: []string{"http://proxy:8083"}}, + } + if got := handler.toneMapCapabilityTimeout(); got != want { + t.Fatalf("toneMapCapabilityTimeout() with an unresolved proxy = %v, want %v", got, want) + } +} diff --git a/internal/jellycompat/web_component.go b/internal/jellycompat/web_component.go index d3de02d28..74e596760 100644 --- a/internal/jellycompat/web_component.go +++ b/internal/jellycompat/web_component.go @@ -1103,26 +1103,6 @@ func currentProcessToken() string { return processToken(os.Getpid()) } -func processToken(pid int) string { - if pid <= 0 { - return "" - } - data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) - if err != nil { - return "" - } - stat := string(data) - commEnd := strings.LastIndex(stat, ") ") - if commEnd == -1 || commEnd+2 >= len(stat) { - return "" - } - fields := strings.Fields(stat[commEnd+2:]) - if len(fields) < 20 { - return "" - } - return fields[19] -} - func finishWebOperation(root, id string, err error) *WebComponentOperationStatus { now := time.Now().UTC().Format(time.RFC3339) diff --git a/internal/metadata/artwork_reconcile.go b/internal/metadata/artwork_reconcile.go index 4cd2bd4d7..8afe24e5d 100644 --- a/internal/metadata/artwork_reconcile.go +++ b/internal/metadata/artwork_reconcile.go @@ -3,6 +3,7 @@ package metadata import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "strconv" @@ -11,11 +12,68 @@ import ( "time" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/Silo-Server/silo-server/internal/catalog" ) +// Retry parameters for the bulk-reset UPDATEs below. They run concurrently +// with per-item writes from ordinary metadata refresh jobs touching the same +// tables (media_items, media_files) in a different row order, which is a real, +// observed deadlock source (SQLSTATE 40P01) — not a hypothetical one. +var ( + artworkReconcileDeadlockMaxAttempts = 5 + artworkReconcileDeadlockBaseBackoff = 100 * time.Millisecond +) + +// artworkReconcileBulkBatchSize bounds how many rows one bulk-reset statement +// touches. +// +// These resets were originally single full-table UPDATEs. On a large library +// that is one statement holding row locks on every matching row for as long as +// it runs — an observed 1h51m on media_items, during which 12 of 16 pool +// connections sat blocked behind it and ordinary playback/metadata writes +// stalled. Retrying on deadlock (above) treats the symptom; the lock footprint +// is the cause. +// +// Batching bounds that footprint: each statement locks at most this many rows +// and commits, so concurrent writers interleave instead of queueing behind a +// table-wide writer. The loop is self-terminating because both SET clauses +// falsify the predicate that selected the row — resetSet writes the provider +// URL into pathCol (which cachedPredicate excludes via NOT LIKE '%://%'), and +// clearSet empties it. +// +// A var, not a const, so DB tests can shrink it to exercise the multi-batch +// path without seeding thousands of rows. +var artworkReconcileBulkBatchSize = 5000 + +// retryOnDeadlock runs op, retrying when Postgres reports a deadlock (40P01) +// or serialization failure (40001), with exponential backoff. It returns +// immediately for any other error, and honors context cancellation between +// attempts. +func retryOnDeadlock(ctx context.Context, op func() error) error { + backoff := artworkReconcileDeadlockBaseBackoff + for attempt := 1; ; attempt++ { + err := op() + if err == nil { + return nil + } + var pgErr *pgconn.PgError + if attempt < artworkReconcileDeadlockMaxAttempts && errors.As(err, &pgErr) && + (pgErr.Code == "40P01" || pgErr.Code == "40001") { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff): + } + backoff *= 2 + continue + } + return err + } +} + // ArtworkObjectChecker is the S3 surface the reconciler needs: existence // checks against the public asset bucket. Satisfied by *s3client.Client. type ArtworkObjectChecker interface { @@ -286,6 +344,10 @@ func artworkSweepSurfaces() []artworkSweepSurface { } } +type collectionPosterMutationLocker interface { + AcquirePosterMutationLock(ctx context.Context, collectionID string) (func(), error) +} + // ArtworkCacheReconciler verifies cached artwork keys against the public S3 // bucket and resets rows whose objects are missing, so the existing pipelines // (image cache queue, book enrichment, chapter thumbnail backfill, collection @@ -293,7 +355,7 @@ func artworkSweepSurfaces() []artworkSweepSurface { type ArtworkCacheReconciler struct { pool *pgxpool.Pool s3 ArtworkObjectChecker - collectionRepo *catalog.LibraryCollectionRepository + collectionRepo collectionPosterMutationLocker } func NewArtworkCacheReconciler(pool *pgxpool.Pool, s3 ArtworkObjectChecker) *ArtworkCacheReconciler { @@ -741,32 +803,112 @@ func (r *ArtworkCacheReconciler) objectExistsWithRetry(ctx context.Context, buck // manual backfill can cache them again. Rows without one are cleared so their // owning pipeline can refill them. func (r *ArtworkCacheReconciler) bulkResetSurface(ctx context.Context, s artworkSweepSurface, stats *ArtworkReconcileStats) error { + // Counts are recorded before the error check: batches commit as they go, + // and the task serializes stats on failure, so an interrupted reset must + // still report the rows it durably changed. if s.sourceCol != "" { - requeue := fmt.Sprintf( - `UPDATE %s SET %s WHERE %s AND %s`, - s.table, s.resetSet(), s.cachedPredicate(), s.remoteSourcePredicate(), - ) - tag, err := r.pool.Exec(ctx, requeue) + requeued, err := r.bulkUpdateInBatches(ctx, s, s.resetSet(), + fmt.Sprintf(`%s AND %s`, s.cachedPredicate(), s.remoteSourcePredicate())) + stats.Requeued += requeued + stats.Checked += requeued if err != nil { return fmt.Errorf("artwork reconcile: bulk reset %s: %w", s.name, err) } - stats.Requeued += int(tag.RowsAffected()) - stats.Checked += int(tag.RowsAffected()) } - clearSQL := fmt.Sprintf( - `UPDATE %s SET %s WHERE %s AND NOT (%s)`, - s.table, s.clearSet, s.cachedPredicate(), s.remoteSourcePredicate(), - ) - tag, err := r.pool.Exec(ctx, clearSQL) + cleared, err := r.bulkUpdateInBatches(ctx, s, s.clearSet, + fmt.Sprintf(`%s AND NOT (%s)`, s.cachedPredicate(), s.remoteSourcePredicate())) + stats.Cleared += cleared + stats.Checked += cleared if err != nil { return fmt.Errorf("artwork reconcile: bulk clear %s: %w", s.name, err) } - stats.Cleared += int(tag.RowsAffected()) - stats.Checked += int(tag.RowsAffected()) return nil } +// bulkUpdateInBatches applies setClause to every row of s matching where, in +// batches of artworkReconcileBulkBatchSize, and returns the total row count. +// +// Each batch selects its slice through the surface's unique key order and takes +// row locks with FOR UPDATE before updating. The consistent ordering is what +// keeps concurrent batches from deadlocking against each other; retryOnDeadlock +// still covers deadlocks against unrelated writers using a different order. +// SKIP LOCKED is deliberately NOT used — skipping a contended row would end the +// loop early and silently leave rows unreset. +// +// A keyset cursor carries each batch's last key into the next batch's WHERE. +// Without it every iteration restarts the ordered scan at the smallest key and +// rechecks all previously updated rows — still in the key index but no longer +// matching — making the sweep O(N²/batchSize) on a large surface. The cursor +// also makes termination unconditional (keys strictly increase), though both +// callers additionally pass a where that stops matching once setClause is +// applied; see the comment on artworkReconcileBulkBatchSize. A row re-cached +// by a concurrent writer behind the cursor is skipped by this sweep and picked +// up by the next reconcile, which is the semantics a point-in-time reset wants. +func (r *ArtworkCacheReconciler) bulkUpdateInBatches(ctx context.Context, s artworkSweepSurface, setClause, where string) (int, error) { + keyCols := strings.Join(s.keyColumnNames(), ", ") + cursorParams := make([]string, len(s.keyCols)) + for i := range cursorParams { + cursorParams[i] = fmt.Sprintf("$%d", i+1) + } + stmtFor := func(withCursor bool) string { + cursorCond := "" + if withCursor { + cursorCond = fmt.Sprintf(` AND (%s) > (%s)`, keyCols, strings.Join(cursorParams, ", ")) + } + // The batch CTE both locks the slice and reports its last key; the + // updated CTE counts what actually changed. Selecting the last key + // from batch rather than from RETURNING keeps the cursor moving even + // if a row version no longer matches at update time. + return fmt.Sprintf(` + WITH batch AS ( + SELECT %[1]s FROM %[2]s + WHERE %[3]s%[4]s + ORDER BY %[1]s + LIMIT %[5]d + FOR UPDATE + ), updated AS ( + UPDATE %[2]s SET %[6]s + WHERE (%[1]s) IN (SELECT %[1]s FROM batch) + RETURNING 1 + ) + SELECT (SELECT count(*) FROM updated), + (SELECT ARRAY[%[7]s] FROM batch ORDER BY %[1]s DESC LIMIT 1)`, + keyCols, s.table, where, cursorCond, artworkReconcileBulkBatchSize, + setClause, strings.Join(s.keySelectExpressions(), ", "), + ) + } + firstStmt, nextStmt := stmtFor(false), stmtFor(true) + + total := 0 + var cursorArgs []any + for { + if err := ctx.Err(); err != nil { + return total, err + } + stmt := nextStmt + if cursorArgs == nil { + stmt = firstStmt + } + var rows int64 + var lastKeys []string + if err := retryOnDeadlock(ctx, func() error { + return r.pool.QueryRow(ctx, stmt, cursorArgs...).Scan(&rows, &lastKeys) + }); err != nil { + return total, err + } + total += int(rows) + if lastKeys == nil { + return total, nil + } + parsed, err := s.parseKeys(lastKeys) + if err != nil { + return total, fmt.Errorf("parsing bulk update cursor: %w", err) + } + cursorArgs = parsed + } +} + // sweptRow is one candidate row in the per-row verification sweep. type sweptRow struct { keys []string @@ -775,9 +917,10 @@ type sweptRow struct { } type coordinatedPosterResetResult struct { - present bool - reset bool - storageErr error + present bool + reset bool + storageErr error + coordinationErr error } func (r *ArtworkCacheReconciler) resetCollectionPosterIfStillMissing( @@ -793,7 +936,12 @@ func (r *ArtworkCacheReconciler) resetCollectionPosterIfStillMissing( unlockDatabase, err := r.collectionRepo.AcquirePosterMutationLock(ctx, collectionID) if err != nil { unlockLocal() - return coordinatedPosterResetResult{}, fmt.Errorf("artwork reconcile: locking collection poster %s: %w", collectionID, err) + if ctxErr := ctx.Err(); ctxErr != nil { + return coordinatedPosterResetResult{}, ctxErr + } + return coordinatedPosterResetResult{ + coordinationErr: fmt.Errorf("artwork reconcile: locking collection poster %s: %w", collectionID, err), + }, nil } defer func() { unlockDatabase() @@ -947,6 +1095,11 @@ func (r *ArtworkCacheReconciler) verifyAndReset(ctx context.Context, s artworkSw return err } switch { + case result.coordinationErr != nil: + stats.Errors++ + stats.SweepErrors++ + slog.WarnContext(ctx, "artwork reconcile: collection poster lock failed; leaving row untouched", + "surface", s.name, "key", row.path, "row", strings.Join(row.keys, "/"), "error", result.coordinationErr) case result.storageErr != nil: stats.Errors++ stats.SweepErrors++ @@ -1036,27 +1189,57 @@ func (r *ArtworkCacheReconciler) countChapterThumbnailFiles(ctx context.Context) } func (r *ArtworkCacheReconciler) bulkResetChapterThumbnails(ctx context.Context, stats *ArtworkReconcileStats) error { - tag, err := r.pool.Exec(ctx, ` - UPDATE media_files - SET chapters = ( - SELECT jsonb_agg( - CASE WHEN coalesce(e->>'thumbnail_path', '') <> '' - THEN (e - 'thumbnail_retry_after' - 'thumbnail_failed_at' - 'thumbnail_last_error') - || '{"thumbnail_path": "", "thumbnail_thumbhash": ""}'::jsonb - ELSE e - END - ORDER BY ord - ) - FROM jsonb_array_elements(chapters) WITH ORDINALITY AS t(e, ord) - ), - chapter_thumbnail_retry_after = NULL - WHERE `+chapterThumbnailFilesPredicate) - if err != nil { - return fmt.Errorf("artwork reconcile: bulk clearing chapter thumbnails: %w", err) + // Batched for the same reason as bulkUpdateInBatches: unbatched, this locks + // every media_files row with chapter thumbnails for the whole run, and the + // per-row jsonb_agg rebuild makes it slow. The id cursor keeps each batch's + // scan from re-evaluating the JSONB predicate over every previously reset + // row, and guarantees termination outright; the SET also empties every + // thumbnail_path the predicate looks for. + stmt := ` + WITH batch AS ( + SELECT id FROM media_files + WHERE ` + chapterThumbnailFilesPredicate + ` AND id > $1 + ORDER BY id + LIMIT ` + strconv.Itoa(artworkReconcileBulkBatchSize) + ` + FOR UPDATE + ), updated AS ( + UPDATE media_files + SET chapters = ( + SELECT jsonb_agg( + CASE WHEN coalesce(e->>'thumbnail_path', '') <> '' + THEN (e - 'thumbnail_retry_after' - 'thumbnail_failed_at' - 'thumbnail_last_error') + || '{"thumbnail_path": "", "thumbnail_thumbhash": ""}'::jsonb + ELSE e + END + ORDER BY ord + ) + FROM jsonb_array_elements(chapters) WITH ORDINALITY AS t(e, ord) + ), + chapter_thumbnail_retry_after = NULL + WHERE id IN (SELECT id FROM batch) + RETURNING 1 + ) + SELECT (SELECT count(*) FROM updated), (SELECT max(id) FROM batch)` + + cursor := int64(0) + for { + if err := ctx.Err(); err != nil { + return err + } + var rows int64 + var lastID *int64 + if err := retryOnDeadlock(ctx, func() error { + return r.pool.QueryRow(ctx, stmt, cursor).Scan(&rows, &lastID) + }); err != nil { + return fmt.Errorf("artwork reconcile: bulk clearing chapter thumbnails: %w", err) + } + stats.Cleared += int(rows) + stats.Checked += int(rows) + if lastID == nil { + return nil + } + cursor = *lastID } - stats.Cleared += int(tag.RowsAffected()) - stats.Checked += int(tag.RowsAffected()) - return nil } // chapterFileRow is one media_files row in the chapter thumbnail sweep. diff --git a/internal/metadata/artwork_reconcile_bulk_db_test.go b/internal/metadata/artwork_reconcile_bulk_db_test.go new file mode 100644 index 000000000..4a69f6b55 --- /dev/null +++ b/internal/metadata/artwork_reconcile_bulk_db_test.go @@ -0,0 +1,492 @@ +package metadata + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +// shrinkBulkBatchSize forces the bulk-reset loops through multiple batches with +// a handful of seeded rows instead of thousands. +func shrinkBulkBatchSize(t *testing.T, size int) { + t.Helper() + prev := artworkReconcileBulkBatchSize + artworkReconcileBulkBatchSize = size + t.Cleanup(func() { artworkReconcileBulkBatchSize = prev }) +} + +func itemPosterSurface(t *testing.T) artworkSweepSurface { + t.Helper() + for _, s := range artworkSweepSurfaces() { + if s.name == "item posters" { + return s + } + } + t.Fatal("item posters surface not found") + return artworkSweepSurface{} +} + +// The bulk resets under test sweep their whole table, not just this test's +// fixtures. The test database may be shared and populated, so each test +// snapshots every pre-existing row its reset would touch and restores those +// rows on cleanup; only the seeded fixtures are asserted on. + +// restoreImageLadderState snapshots the image-ladder backfill singleton and +// restores it on cleanup. Inserting rows with local cached poster paths fires +// the reopen_image_ladder_backfill_v2 trigger, which on a database that has +// completed ladder v2 lowers backfilled_version — durable state a test +// fixture must not leave behind on a shared database. +func restoreImageLadderState(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + ctx := context.Background() + var version int + var lastAttempt *time.Time + err := pool.QueryRow(ctx, + `SELECT backfilled_version, last_attempt_at FROM image_ladder_backfill_state WHERE id = 1`). + Scan(&version, &lastAttempt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return + } + t.Fatalf("snapshot image ladder state: %v", err) + } + t.Cleanup(func() { + if _, err := pool.Exec(ctx, + `UPDATE image_ladder_backfill_state SET backfilled_version = $1, last_attempt_at = $2, updated_at = NOW() WHERE id = 1`, + version, lastAttempt); err != nil { + t.Errorf("restore image ladder state: %v", err) + } + }) +} + +// restoreDisplacedGCCandidates snapshots the artwork-GC candidate rows the +// bulk reset can touch and restores them on cleanup. Displacing a cached +// poster path ending in /original. fires queue_displaced_artwork_revision, +// which inserts a candidate for the path or resets an existing candidate's +// schedule, attempts, lease, and error state — auxiliary shared-database state +// the media_items row restore alone does not undo. Candidates created for the +// test's own fixture paths are deleted outright. +func restoreDisplacedGCCandidates(t *testing.T, pool *pgxpool.Pool, seededPathPattern string) { + t.Helper() + ctx := context.Background() + surface := itemPosterSurface(t) + + var displaced []string + rows, err := pool.Query(ctx, fmt.Sprintf( + `SELECT poster_path FROM media_items WHERE %s`, surface.cachedPredicate())) + if err != nil { + t.Fatalf("list displaceable poster paths: %v", err) + } + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + t.Fatalf("scan displaceable path: %v", err) + } + displaced = append(displaced, path) + } + if err := rows.Err(); err != nil { + t.Fatalf("read displaceable paths: %v", err) + } + + type candidate struct { + originalPath string + imageType string + objectKeys []string + notBefore time.Time + nextAttemptAt *time.Time + deletedAt *time.Time + attemptCount int + lockedAt *time.Time + lockedBy string + lastError string + } + var snapshot []candidate + snapshotPaths := []string{} + rows, err = pool.Query(ctx, ` + SELECT original_path, image_type, object_keys, not_before, next_attempt_at, + deleted_at, attempt_count, locked_at, locked_by, last_error + FROM artwork_revision_gc_candidates WHERE original_path = ANY($1)`, displaced) + if err != nil { + t.Fatalf("snapshot gc candidates: %v", err) + } + for rows.Next() { + var c candidate + if err := rows.Scan(&c.originalPath, &c.imageType, &c.objectKeys, &c.notBefore, + &c.nextAttemptAt, &c.deletedAt, &c.attemptCount, &c.lockedAt, &c.lockedBy, &c.lastError); err != nil { + t.Fatalf("scan gc candidate: %v", err) + } + snapshot = append(snapshot, c) + snapshotPaths = append(snapshotPaths, c.originalPath) + } + if err := rows.Err(); err != nil { + t.Fatalf("read gc candidates: %v", err) + } + + t.Cleanup(func() { + if _, err := pool.Exec(ctx, + `DELETE FROM artwork_revision_gc_candidates WHERE original_path LIKE $1`, seededPathPattern); err != nil { + t.Errorf("delete fixture gc candidates: %v", err) + } + if _, err := pool.Exec(ctx, ` + DELETE FROM artwork_revision_gc_candidates + WHERE original_path = ANY($1) AND original_path <> ALL($2)`, displaced, snapshotPaths); err != nil { + t.Errorf("delete reset-created gc candidates: %v", err) + } + for _, c := range snapshot { + if _, err := pool.Exec(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at, + deleted_at, attempt_count, locked_at, locked_by, last_error, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW()) + ON CONFLICT (original_path) DO UPDATE SET + image_type = EXCLUDED.image_type, + object_keys = EXCLUDED.object_keys, + not_before = EXCLUDED.not_before, + next_attempt_at = EXCLUDED.next_attempt_at, + deleted_at = EXCLUDED.deleted_at, + attempt_count = EXCLUDED.attempt_count, + locked_at = EXCLUDED.locked_at, + locked_by = EXCLUDED.locked_by, + last_error = EXCLUDED.last_error, + updated_at = NOW()`, + c.originalPath, c.imageType, c.objectKeys, c.notBefore, c.nextAttemptAt, + c.deletedAt, c.attemptCount, c.lockedAt, c.lockedBy, c.lastError); err != nil { + t.Errorf("restore gc candidate %s: %v", c.originalPath, err) + } + } + }) +} + +func restorePreexistingPosterRows(t *testing.T, pool *pgxpool.Pool, seededPrefix string) { + t.Helper() + ctx := context.Background() + surface := itemPosterSurface(t) + rows, err := pool.Query(ctx, fmt.Sprintf( + `SELECT content_id, poster_path, last_refreshed, updated_at FROM media_items + WHERE %s AND content_id NOT LIKE $1`, surface.cachedPredicate()), seededPrefix+"%") + if err != nil { + t.Fatalf("snapshot pre-existing poster rows: %v", err) + } + type itemState struct { + contentID string + posterPath string + lastRefreshed *time.Time + updatedAt *time.Time + } + var snapshot []itemState + for rows.Next() { + var s itemState + if err := rows.Scan(&s.contentID, &s.posterPath, &s.lastRefreshed, &s.updatedAt); err != nil { + t.Fatalf("scan poster snapshot: %v", err) + } + snapshot = append(snapshot, s) + } + if err := rows.Err(); err != nil { + t.Fatalf("read poster snapshot: %v", err) + } + t.Cleanup(func() { + for _, s := range snapshot { + if _, err := pool.Exec(ctx, + `UPDATE media_items SET poster_path = $2, last_refreshed = $3, updated_at = $4 WHERE content_id = $1`, + s.contentID, s.posterPath, s.lastRefreshed, s.updatedAt); err != nil { + t.Errorf("restore poster row %s: %v", s.contentID, err) + } + } + }) +} + +func restorePreexistingChapterRows(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + ctx := context.Background() + rows, err := pool.Query(ctx, + `SELECT id, chapters, chapter_thumbnail_retry_after FROM media_files WHERE `+chapterThumbnailFilesPredicate) + if err != nil { + t.Fatalf("snapshot pre-existing chapter rows: %v", err) + } + type fileState struct { + id int64 + chapters []byte + retryAfter *time.Time + } + var snapshot []fileState + for rows.Next() { + var s fileState + if err := rows.Scan(&s.id, &s.chapters, &s.retryAfter); err != nil { + t.Fatalf("scan chapter snapshot: %v", err) + } + snapshot = append(snapshot, s) + } + if err := rows.Err(); err != nil { + t.Fatalf("read chapter snapshot: %v", err) + } + t.Cleanup(func() { + for _, s := range snapshot { + if _, err := pool.Exec(ctx, + `UPDATE media_files SET chapters = $2::jsonb, chapter_thumbnail_retry_after = $3 WHERE id = $1`, + s.id, s.chapters, s.retryAfter); err != nil { + t.Errorf("restore chapter row %d: %v", s.id, err) + } + } + }) +} + +// TestBulkResetSurfaceBatches drives bulkResetSurface across several batches +// and verifies both halves: rows with a remote source are requeued (path reset +// to the source URL), rows without one are cleared. Assertions are scoped to +// the seeded rows; pre-existing rows the sweep touches are snapshot-restored. +func TestBulkResetSurfaceBatches(t *testing.T) { + pool := localArtworkTestPool(t) + ctx := context.Background() + shrinkBulkBatchSize(t, 3) + + prefix := fmt.Sprintf("bulkreset-%d", time.Now().UnixNano()) + restoreImageLadderState(t, pool) + restoreDisplacedGCCandidates(t, pool, "metadata/movie/"+prefix+"-%") + restorePreexistingPosterRows(t, pool, prefix) + const requeueRows, clearRows = 7, 5 + sourceURL := func(i int) string { + return fmt.Sprintf("https://image.example.org/t/p/original/%s-%d.jpg", prefix, i) + } + + var seeded []string + for i := 0; i < requeueRows+clearRows; i++ { + contentID := fmt.Sprintf("%s-%d", prefix, i) + source := "" + if i < requeueRows { + source = sourceURL(i) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path, poster_source_path, last_refreshed) + VALUES ($1, 'movie', 'Bulk Reset Test', 'matched', '{}'::text[], $2, $3, NOW()) + `, contentID, fmt.Sprintf("metadata/movie/%s/poster/original.webp", contentID), source); err != nil { + t.Fatalf("seed item %d: %v", i, err) + } + seeded = append(seeded, contentID) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id LIKE $1`, prefix+"-%") + }) + + r := &ArtworkCacheReconciler{pool: pool} + var stats ArtworkReconcileStats + if err := r.bulkResetSurface(ctx, itemPosterSurface(t), &stats); err != nil { + t.Fatalf("bulkResetSurface: %v", err) + } + if stats.Requeued < requeueRows { + t.Errorf("stats.Requeued = %d, want >= %d", stats.Requeued, requeueRows) + } + if stats.Cleared < clearRows { + t.Errorf("stats.Cleared = %d, want >= %d", stats.Cleared, clearRows) + } + + for i, contentID := range seeded { + var posterPath string + var lastRefreshed *time.Time + if err := pool.QueryRow(ctx, + `SELECT poster_path, last_refreshed FROM media_items WHERE content_id = $1`, + contentID).Scan(&posterPath, &lastRefreshed); err != nil { + t.Fatalf("read back item %d: %v", i, err) + } + if i < requeueRows { + if posterPath != sourceURL(i) { + t.Errorf("row %d: poster_path = %q, want requeued source %q", i, posterPath, sourceURL(i)) + } + } else { + if posterPath != "" { + t.Errorf("row %d: poster_path = %q, want cleared", i, posterPath) + } + if lastRefreshed != nil { + t.Errorf("row %d: last_refreshed = %v, want NULL after clear", i, lastRefreshed) + } + } + } +} + +// TestBulkResetChapterThumbnailsBatches drives the chapter-thumbnail reset +// across several batches and verifies the JSONB rewrite: cached thumbnail +// elements are emptied and stripped of retry state, elements without a +// thumbnail survive untouched, and the loop terminates once no row matches. +func TestBulkResetChapterThumbnailsBatches(t *testing.T) { + pool := localArtworkTestPool(t) + ctx := context.Background() + shrinkBulkBatchSize(t, 2) + restorePreexistingChapterRows(t, pool) + + suffix := time.Now().UnixNano() + var folderID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_folders (type, name) + VALUES ('movies', $1) + RETURNING id`, fmt.Sprintf("bulk-chapter-test-%d", suffix)).Scan(&folderID); err != nil { + t.Fatalf("seed media folder: %v", err) + } + const fileRows = 5 + fileIDs := make([]int, fileRows) + for i := range fileIDs { + chapters := fmt.Sprintf(`[ + {"title": "c1", "thumbnail_path": "chapters/%d-%d/1.webp", "thumbnail_thumbhash": "aa", "thumbnail_retry_after": "2026-01-01T00:00:00Z", "thumbnail_failed_at": "2026-01-01T00:00:00Z", "thumbnail_last_error": "boom"}, + {"title": "c2"} + ]`, suffix, i) + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, chapters, chapter_thumbnail_retry_after) + VALUES ($1, $2, $3::jsonb, NOW()) + RETURNING id`, folderID, fmt.Sprintf("/bulk-chapter-test/%d-%d.mkv", suffix, i), chapters).Scan(&fileIDs[i]); err != nil { + t.Fatalf("seed media file %d: %v", i, err) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_files WHERE id = ANY($1)`, fileIDs) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + + r := &ArtworkCacheReconciler{pool: pool} + var stats ArtworkReconcileStats + if err := r.bulkResetChapterThumbnails(ctx, &stats); err != nil { + t.Fatalf("bulkResetChapterThumbnails: %v", err) + } + if stats.Cleared < fileRows { + t.Errorf("stats.Cleared = %d, want >= %d", stats.Cleared, fileRows) + } + + for i, id := range fileIDs { + var first, second map[string]any + var retryAfter *time.Time + if err := pool.QueryRow(ctx, + `SELECT chapters->0, chapters->1, chapter_thumbnail_retry_after FROM media_files WHERE id = $1`, + id).Scan(&first, &second, &retryAfter); err != nil { + t.Fatalf("read back file %d: %v", i, err) + } + if got := first["thumbnail_path"]; got != "" { + t.Errorf("file %d: thumbnail_path = %v, want emptied", i, got) + } + if got := first["thumbnail_thumbhash"]; got != "" { + t.Errorf("file %d: thumbnail_thumbhash = %v, want emptied", i, got) + } + for _, key := range []string{"thumbnail_retry_after", "thumbnail_failed_at", "thumbnail_last_error"} { + if _, ok := first[key]; ok { + t.Errorf("file %d: %s survived the reset", i, key) + } + } + if got := second["title"]; got != "c2" { + t.Errorf("file %d: untouched element title = %v, want c2", i, got) + } + if _, ok := second["thumbnail_path"]; ok { + t.Errorf("file %d: element without a thumbnail gained thumbnail_path", i) + } + if retryAfter != nil { + t.Errorf("file %d: chapter_thumbnail_retry_after = %v, want NULL", i, retryAfter) + } + } +} + +// TestBulkResetSurfacePartialFailureKeepsCounts interrupts bulkResetSurface +// after its requeue phase by giving the clear phase a broken SET clause. +// Batches commit as they go and the task serializes stats on failure, so the +// rows the requeue phase durably changed must be counted despite the error. +func TestBulkResetSurfacePartialFailureKeepsCounts(t *testing.T) { + pool := localArtworkTestPool(t) + ctx := context.Background() + shrinkBulkBatchSize(t, 2) + + prefix := fmt.Sprintf("bulkpartial-%d", time.Now().UnixNano()) + restoreImageLadderState(t, pool) + restoreDisplacedGCCandidates(t, pool, "metadata/movie/"+prefix+"-%") + restorePreexistingPosterRows(t, pool, prefix) + + const requeueRows = 5 + for i := 0; i < requeueRows; i++ { + contentID := fmt.Sprintf("%s-%d", prefix, i) + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path, poster_source_path) + VALUES ($1, 'movie', 'Bulk Partial Test', 'matched', '{}'::text[], $2, $3) + `, contentID, fmt.Sprintf("metadata/movie/%s/poster/original.webp", contentID), + fmt.Sprintf("https://image.example.org/t/p/original/%s-%d.jpg", prefix, i)); err != nil { + t.Fatalf("seed item %d: %v", i, err) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id LIKE $1`, prefix+"-%") + }) + + surface := itemPosterSurface(t) + surface.clearSet = `nonexistent_bulk_partial_column = ''` + + r := &ArtworkCacheReconciler{pool: pool} + var stats ArtworkReconcileStats + err := r.bulkResetSurface(ctx, surface, &stats) + if err == nil { + t.Fatal("bulkResetSurface succeeded, want the clear phase to fail") + } + if stats.Requeued < requeueRows { + t.Errorf("stats.Requeued = %d after clear-phase failure, want >= %d committed requeues", stats.Requeued, requeueRows) + } +} + +func TestRetryOnDeadlock(t *testing.T) { + prevBackoff := artworkReconcileDeadlockBaseBackoff + artworkReconcileDeadlockBaseBackoff = time.Millisecond + t.Cleanup(func() { artworkReconcileDeadlockBaseBackoff = prevBackoff }) + + deadlock := &pgconn.PgError{Code: "40P01"} + ctx := context.Background() + + t.Run("retries deadlocks until success", func(t *testing.T) { + calls := 0 + err := retryOnDeadlock(ctx, func() error { + calls++ + if calls < 3 { + return fmt.Errorf("exec: %w", deadlock) + } + return nil + }) + if err != nil { + t.Fatalf("retryOnDeadlock: %v", err) + } + if calls != 3 { + t.Errorf("calls = %d, want 3", calls) + } + }) + + t.Run("gives up after max attempts", func(t *testing.T) { + calls := 0 + err := retryOnDeadlock(ctx, func() error { + calls++ + return deadlock + }) + if !errors.Is(err, deadlock) { + t.Fatalf("err = %v, want the deadlock error", err) + } + if calls != artworkReconcileDeadlockMaxAttempts { + t.Errorf("calls = %d, want %d", calls, artworkReconcileDeadlockMaxAttempts) + } + }) + + t.Run("returns other errors immediately", func(t *testing.T) { + calls := 0 + boom := errors.New("boom") + if err := retryOnDeadlock(ctx, func() error { + calls++ + return boom + }); !errors.Is(err, boom) { + t.Fatalf("err = %v, want boom", err) + } + if calls != 1 { + t.Errorf("calls = %d, want 1", calls) + } + }) + + t.Run("honors cancellation between attempts", func(t *testing.T) { + canceled, cancel := context.WithCancel(context.Background()) + cancel() + err := retryOnDeadlock(canceled, func() error { return deadlock }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + }) +} diff --git a/internal/metadata/artwork_reconcile_test.go b/internal/metadata/artwork_reconcile_test.go index 5e52041e1..858b4d3d9 100644 --- a/internal/metadata/artwork_reconcile_test.go +++ b/internal/metadata/artwork_reconcile_test.go @@ -24,6 +24,14 @@ type fakeObjectChecker struct { checked map[string]int } +type failingCollectionPosterMutationLocker struct { + err error +} + +func (f failingCollectionPosterMutationLocker) AcquirePosterMutationLock(context.Context, string) (func(), error) { + return nil, f.err +} + func (f *fakeObjectChecker) Bucket() string { return "test-bucket" } func (f *fakeObjectChecker) ObjectExists(_ context.Context, _ string, key string) (bool, error) { @@ -95,6 +103,40 @@ func TestCollectionPosterSurfaceUsesCoordinatedRevalidation(t *testing.T) { } } +func TestVerifyAndResetContinuesAfterCollectionPosterLockFailure(t *testing.T) { + var surface artworkSweepSurface + for _, candidate := range artworkSweepSurfaces() { + if candidate.name == artworkCollectionPostersName { + surface = candidate + break + } + } + if surface.name == "" { + t.Fatal("collection poster surface is missing") + } + + const posterPath = "collection-images/collection-1/poster/original.webp" + const presentPath = "collection-images/collection-2/poster/original.webp" + lockErr := errors.New("simulated lock failure") + reconciler := &ArtworkCacheReconciler{ + s3: &fakeObjectChecker{missing: map[string]bool{posterPath: true}}, + collectionRepo: failingCollectionPosterMutationLocker{ + err: lockErr, + }, + } + stats := ArtworkReconcileStats{} + err := reconciler.verifyAndReset(context.Background(), surface, []sweptRow{ + {keys: []string{"collection-1"}, path: posterPath}, + {keys: []string{"collection-2"}, path: presentPath}, + }, &stats) + if err != nil { + t.Fatalf("verifyAndReset returned a batch error: %v", err) + } + if stats.Checked != 2 || stats.Verified != 1 || stats.Errors != 1 || stats.SweepErrors != 1 { + t.Fatalf("stats = %+v, want one verified row and one row-level sweep error", stats) + } +} + func TestResetCollectionPosterIfStillMissingRevalidates(t *testing.T) { dsn := os.Getenv("SILO_TEST_DATABASE_URL") if dsn == "" { diff --git a/internal/metadata/image_cache_job_repo.go b/internal/metadata/image_cache_job_repo.go index 0d56a1a77..0100b99ea 100644 --- a/internal/metadata/image_cache_job_repo.go +++ b/internal/metadata/image_cache_job_repo.go @@ -33,7 +33,10 @@ const ( ImageCacheStatusSucceeded = "succeeded" ImageCacheStatusFailed = "failed" - imageCacheLeaseDuration = 15 * time.Minute + // ImageCacheLeaseDuration is stamped on every claimed row up front. + // Exported so worker/claim-page sizing can assert a claimed page always + // drains inside it (see internal/taskmanager/tasks). + ImageCacheLeaseDuration = 15 * time.Minute imageCacheMaxAttempts = 8 imageCacheDeferredRetry = 7 * 24 * time.Hour @@ -427,7 +430,7 @@ func (r *ImageCacheJobRepository) recoverExpiredRunning(ctx context.Context, tar AND locked_at < NOW() - $1::interval ` args := []any{ - intervalLiteral(imageCacheLeaseDuration), + intervalLiteral(ImageCacheLeaseDuration), imageCacheMaxAttempts, intervalLiteral(imageCacheFailedCooldown), } @@ -831,7 +834,11 @@ func (r *ImageCacheJobRepository) EnqueueExistingProviderArtwork(ctx context.Con in.ProviderID = imageCacheProviderIDFromSource(in.SourcePath, fallbackProvider) in.ProviderContentID = imageCacheProviderContentID(in.ProviderID, tmdbID, tvdbID, imdbID, firstNonEmpty(in.SeriesID, in.TargetContentID)) in.ContentType = imageCacheContentType(in.ContentType) - inputs = append(inputs, in) + normalized, ok := normalizeImageCacheJobInput(in) + if !ok { + continue + } + inputs = append(inputs, normalized) page.Discovered++ } } diff --git a/internal/metadata/image_cache_job_repo_test.go b/internal/metadata/image_cache_job_repo_test.go index 34ed0151b..19713ea93 100644 --- a/internal/metadata/image_cache_job_repo_test.go +++ b/internal/metadata/image_cache_job_repo_test.go @@ -96,6 +96,7 @@ func TestNormalizeImageCacheJobInputSkipsNonProviderArtwork(t *testing.T) { "", "tmdb/series/1396/poster/original.webp", "s3://media/tmdb/series/1396/poster/original.webp", + " s3://media/tmdb/series/1396/poster/original.webp ", "local://poster.jpg", "generated://collections/1/poster.jpg", } { @@ -149,6 +150,18 @@ func TestImageCacheDiscoverySurfacesUseBoundedNativeKeyPages(t *testing.T) { "e.still_source_path", "p.photo_source_path", } + uncachedColumns := [...]string{ + "mi.poster_path", + "mi.backdrop_path", + "mi.logo_path", + "loc.poster_path", + "loc.backdrop_path", + "loc.logo_path", + "s.poster_path", + "loc.poster_path", + "e.still_path", + "p.photo_path", + } for surface := 0; surface < imageCacheDiscoverySurfaceCount; surface++ { cursor := imageCacheDiscoveryCursor{Surface: surface, Key: "content-10", Subkey: "fr", NumericKey: 10} query, args := imageCacheDiscoveryQuery(cursor, 1000) @@ -168,7 +181,24 @@ func TestImageCacheDiscoverySurfacesUseBoundedNativeKeyPages(t *testing.T) { if strings.Contains(query, providerColumns[surface]+" "+providerColumns[surface]) { t.Fatalf("surface %d query duplicates provider column %q:\n%s", surface, providerColumns[surface], query) } - if len(args) < 2 || args[0] != 1000 { + uncachedPredicate := "AND (" + uncachedColumns[surface] + " LIKE '%://%' OR coalesce(" + uncachedColumns[surface] + ", '') = '')" + if !strings.Contains(query, uncachedPredicate) { + t.Fatalf("surface %d query missing exact uncached-target predicate %q:\n%s", surface, uncachedPredicate, query) + } + if strings.Contains(query, "@nonProviderSchemes") || !strings.Contains(query, nonProviderImageSchemesSQL) { + t.Fatalf("surface %d query did not expand the non-provider scheme filter:\n%s", surface, query) + } + if !strings.Contains(query, "LEFT JOIN LATERAL") || + !strings.Contains(query, "FROM metadata_image_cache_jobs j") || + !strings.Contains(query, `j.target_content_id = c.target_content_id COLLATE "default"`) || + !strings.Contains(query, "j.source_path IS DISTINCT FROM c.source_path") { + t.Fatalf("surface %d query is missing the indexed eligibility lookup:\n%s", surface, query) + } + wantArgs := 2 + if (surface >= 3 && surface <= 5) || surface == 7 { + wantArgs = 3 + } + if len(args) != wantArgs || args[0] != 1000 { t.Fatalf("surface %d args = %#v, want page limit and native cursor", surface, args) } } diff --git a/internal/metadata/image_cache_processor.go b/internal/metadata/image_cache_processor.go index 2e26896a6..1f8469005 100644 --- a/internal/metadata/image_cache_processor.go +++ b/internal/metadata/image_cache_processor.go @@ -34,11 +34,21 @@ const ( imageCacheDiscoveryBatchSize = 1000 // Waiting for a background worker to release a job polls with backoff and // gives up after immediateImageCacheIdleTimeout without progress. The - // worker's own lease runs for imageCacheLeaseDuration, and its pod can die + // worker's own lease runs for ImageCacheLeaseDuration, and its pod can die // while holding it, so an interactive refresh must not wait that long. immediateImageCacheMinPoll = 100 * time.Millisecond immediateImageCacheMaxPoll = 2 * time.Second immediateImageCacheIdleTimeout = 30 * time.Second + + // ImageCacheJobTimeout bounds one job end to end: source check, download + // (which also has its own 30-second cap in imagecache), variant encode, and + // uploads. Only the download had a deadline before; a hung upload or encode + // could hold a job past its claim lease, letting another worker reclaim and + // duplicate it. Two minutes is generous for the slowest realistic job, and + // worker/claim-page sizing (internal/taskmanager/tasks) relies on it to + // prove a claimed page always drains inside ImageCacheLeaseDuration. A job + // that hits it is marked failed and retried on the normal backoff. + ImageCacheJobTimeout = 2 * time.Minute ) // ErrTargetArtworkPending reports that some of a refreshed target's artwork was @@ -452,7 +462,9 @@ loop: go func(job *models.MetadataImageCacheJob) { defer wg.Done() defer func() { <-sem }() - result := p.processOne(ctx, job) + jobCtx, cancelJob := context.WithTimeout(ctx, ImageCacheJobTimeout) + result := p.processOne(jobCtx, job) + cancelJob() mu.Lock() switch result.outcome { case "succeeded": @@ -628,6 +640,8 @@ func (p *ImageCacheProcessor) runUntilIdle(ctx context.Context, workerID string, } var discoveryCursor imageCacheDiscoveryCursor discoveredThisSweep := false + enqueuedThisSweep := false + confirmationSweep := false for { if err := ctx.Err(); err != nil { return total, err @@ -689,6 +703,7 @@ func (p *ImageCacheProcessor) runUntilIdle(ctx context.Context, workerID string, total.EnqueuedExisting += page.Enqueued reportImageCacheRunProgress(reportProgress, total) if page.Enqueued > 0 { + enqueuedThisSweep = true break } if !page.Complete { @@ -697,11 +712,20 @@ func (p *ImageCacheProcessor) runUntilIdle(ctx context.Context, workerID string, if !discoveredThisSweep { return total, nil } + if confirmationSweep && !enqueuedThisSweep { + // A second complete sweep reported candidates but could not enqueue + // any of them. Repeating the same non-actionable catalog state cannot + // make progress; a later explicit backfill starts fresh from durable + // queue and catalog state. + return total, nil + } // Queue rows and cached paths now reflect the completed sweep. Resetting // catches concurrent changes that sorted behind the cursor. A retry or // process restart also starts here and safely reuses those durable rows. discoveryCursor = imageCacheDiscoveryCursor{} discoveredThisSweep = false + enqueuedThisSweep = false + confirmationSweep = true } } } diff --git a/internal/metadata/image_cache_processor_test.go b/internal/metadata/image_cache_processor_test.go index 7be2fdac3..867675207 100644 --- a/internal/metadata/image_cache_processor_test.go +++ b/internal/metadata/image_cache_processor_test.go @@ -475,7 +475,7 @@ func TestImageCacheProcessorRetriesTargetAfterConcurrentWorkerFinishes(t *testin func TestImageCacheProcessorStopsWaitingOnStuckBackgroundWorker(t *testing.T) { // A worker that claimed the job and died holds its lease for - // imageCacheLeaseDuration. The interactive refresh must not block that + // ImageCacheLeaseDuration. The interactive refresh must not block that // long: it gives up and reports the artwork as still pending. jobs := &targetImageCacheJobs{alwaysRunning: true} processor := NewImageCacheProcessorWithTargets( @@ -900,6 +900,33 @@ func TestImageCacheProcessorDiscoveryWrapsForConcurrentUpdates(t *testing.T) { } } +func TestImageCacheProcessorDiscoveryStopsAfterRepeatedNonEnqueueableCandidates(t *testing.T) { + unexpectedSweep := errors.New("unexpected third discovery sweep") + jobs := &loopingImageCacheJobs{ + enqueuePages: []imageCacheDiscoveryPage{ + {Scanned: 1, Discovered: 1, Next: imageCacheDiscoveryCursor{Surface: imageCacheDiscoverySurfaceCount}, Complete: true}, + {Scanned: 1, Discovered: 1, Next: imageCacheDiscoveryCursor{Surface: imageCacheDiscoverySurfaceCount}, Complete: true}, + {}, + }, + enqueueErrors: []error{nil, nil, unexpectedSweep}, + claimedResults: [][]*models.MetadataImageCacheJob{{}}, + } + processor := NewImageCacheProcessor(jobs, &fakeImageCacher{}, &fakeImageResolver{}, nil, nil) + + stats, err := processor.RunUntilIdle(context.Background(), "test-worker", 1000, 2, 0, nil) + if err != nil { + t.Fatalf("RunUntilIdle() error = %v", err) + } + if stats.EnqueuedExisting != 0 || jobs.enqueueCalls != 2 { + t.Fatalf("stats = %+v, discovery calls = %d; want no enqueues and one confirmation sweep", stats, jobs.enqueueCalls) + } + for call, cursor := range jobs.enqueueCursors { + if cursor != (imageCacheDiscoveryCursor{}) { + t.Fatalf("discovery call %d cursor = %+v, want reset cursor", call+1, cursor) + } + } +} + func TestImageCacheProcessorDiscoveryRetryStartsFromDurableState(t *testing.T) { discoveryErr := errors.New("temporary discovery failure") jobs := &loopingImageCacheJobs{ diff --git a/internal/metadata/trakt/client.go b/internal/metadata/trakt/client.go index 242e242bb..d5a5c5c4e 100644 --- a/internal/metadata/trakt/client.go +++ b/internal/metadata/trakt/client.go @@ -10,6 +10,7 @@ import ( "net/url" "strconv" "strings" + "sync/atomic" "time" "golang.org/x/time/rate" @@ -28,22 +29,44 @@ const ( // Client is an HTTP client for Trakt collection/discovery feeds. type Client struct { httpClient *http.Client - clientID string - baseURL string - limiter *rate.Limiter + // clientID is swapped atomically so a saved credential change reaches a + // long-lived client without rebuilding it — and therefore without a + // server restart. See SetClientID. + clientID atomic.Pointer[string] + baseURL string + limiter *rate.Limiter } -// NewClient creates a Trakt client. clientID is required by Trakt for API calls. +// NewClient creates a Trakt client. clientID is required by Trakt for API +// calls, but may be empty here when the caller keeps it current through +// SetClientID. func NewClient(clientID string, rateLimit int) *Client { if rateLimit <= 0 { rateLimit = defaultCollectionRateLimit } - return &Client{ + c := &Client{ httpClient: &http.Client{Timeout: 20 * time.Second}, - clientID: strings.TrimSpace(clientID), baseURL: defaultBaseURL, limiter: rate.NewLimiter(rate.Limit(rateLimit), rateLimit), } + c.SetClientID(clientID) + return c +} + +// SetClientID replaces the app client ID sent on subsequent requests. Safe to +// call while requests are in flight, which is what lets a new +// watchsync.trakt.client_id apply without restarting the server. +func (c *Client) SetClientID(clientID string) { + trimmed := strings.TrimSpace(clientID) + c.clientID.Store(&trimmed) +} + +// ClientID returns the app client ID currently in use. +func (c *Client) ClientID() string { + if current := c.clientID.Load(); current != nil { + return *current + } + return "" } // SetBaseURL overrides the API base URL. Used by tests. @@ -223,7 +246,10 @@ func (c *Client) getMediaList(ctx context.Context, path, mediaType, accessToken } func (c *Client) doGet(ctx context.Context, path string, accessToken string, dest any) error { - if strings.TrimSpace(c.clientID) == "" { + // Read once so every retry of this request uses one client ID even if a + // credential change lands mid-flight. + clientID := c.ClientID() + if clientID == "" { return errors.New("trakt: client id is required") } if err := c.limiter.Wait(ctx); err != nil { @@ -239,7 +265,7 @@ func (c *Client) doGet(ctx context.Context, path string, accessToken string, des req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("trakt-api-version", traktAPIVersion) - req.Header.Set("trakt-api-key", c.clientID) + req.Header.Set("trakt-api-key", clientID) if strings.TrimSpace(accessToken) != "" { req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(accessToken)) } diff --git a/internal/metadata/trakt/client_test.go b/internal/metadata/trakt/client_test.go index b055bcbca..fc3198358 100644 --- a/internal/metadata/trakt/client_test.go +++ b/internal/metadata/trakt/client_test.go @@ -118,6 +118,35 @@ func TestGetCollectionPresetRetriesRateLimit(t *testing.T) { } } +// A saved credential change has to reach a client that was built at startup; +// this is what lets watchsync.trakt.client_id stay out of the restart registry. +func TestSetClientIDAppliesToLaterRequests(t *testing.T) { + var gotAPIKey string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAPIKey = r.Header.Get("trakt-api-key") + writeJSON(t, w, []map[string]any{}) + })) + defer server.Close() + + client := NewClient("", 1000) + client.SetBaseURL(server.URL) + + if _, err := client.GetCollectionPreset(context.Background(), "popular", "movie", 1, ""); err == nil { + t.Fatal("expected an error while no client id is configured") + } + + client.SetClientID(" rotated-client-id ") + if got := client.ClientID(); got != "rotated-client-id" { + t.Fatalf("ClientID() = %q, want trimmed rotated-client-id", got) + } + if _, err := client.GetCollectionPreset(context.Background(), "popular", "movie", 1, ""); err != nil { + t.Fatalf("GetCollectionPreset after SetClientID: %v", err) + } + if gotAPIKey != "rotated-client-id" { + t.Fatalf("trakt-api-key = %q, want rotated-client-id", gotAPIKey) + } +} + func writeJSON(t *testing.T, w http.ResponseWriter, v any) { t.Helper() w.Header().Set("Content-Type", "application/json") diff --git a/internal/nodeconfig/watcher.go b/internal/nodeconfig/watcher.go index 8a13692c3..61ef4abea 100644 --- a/internal/nodeconfig/watcher.go +++ b/internal/nodeconfig/watcher.go @@ -2,6 +2,7 @@ package nodeconfig import ( "context" + "errors" "fmt" "log/slog" "reflect" @@ -24,8 +25,32 @@ type BootstrapOverrides struct { DatabaseURL string // from DATABASE_URL env JFListen string // from JF_PORT env RedisURL string // from REDIS_URL env + // NodeURL is this process's own stream_nodes identity (from NODE_URL env), + // set only in proxy/transcode mode. It is what lets a node find its own row + // and overlay that row's acceleration overrides onto the cluster-wide + // playback settings. Empty on an API host, which has no stream_nodes row. + NodeURL string + // NodeName is this process's own registered node name (from NODE_NAME + // env). It is the override-row fallback identity when the registered url + // and NODE_URL differ, as they do on split-horizon topologies. + NodeName string } +// nodeHWOverrides is one node's own acceleration policy, as stored on its +// stream_nodes row. A nil field means the node inherits the cluster-wide +// playback setting. +type nodeHWOverrides struct { + HWAccel *string + HWDevice *string +} + +// loadNodeHWOverrides reads one node's overrides, matching its registered row +// by URL first and by NODE_NAME as the fallback. found is false when the node +// has no stream_nodes row at all — a legitimate deployment (a node nobody +// registered yet), not an error. It is a field on the Watcher so the overlay +// can be exercised without a database. +type loadNodeHWOverrides func(ctx context.Context, nodeURL, nodeName string) (overrides nodeHWOverrides, found bool, err error) + // Watcher watches for configuration changes in the database and // automatically reloads the Config when changes are detected. type Watcher struct { @@ -41,6 +66,54 @@ type Watcher struct { // (e.g. resolving the seeded ffmpeg path) survive hot reloads. normalizers []func(*config.Config) reloadCh chan struct{} // buffered(1), event bus writes here + + // loadOverrides reads this node's own acceleration overrides; see the + // overlay in applySettings. + loadOverrides loadNodeHWOverrides + // overrides is the last successfully read overlay, kept so a database + // hiccup during a reload cannot silently flip a node back onto the + // cluster-wide backend. overridesLoaded distinguishes "read, and there is + // nothing to overlay" from "never read". + overrides nodeHWOverrides + overridesLoaded bool + missingRowLogged bool + duplicateRowLogged bool + ambiguousNameLogged bool + // nodeRowID is the row this worker has resolved to, kept so a later rename + // or repoint cannot sever the association. Zero until a lookup succeeds. + nodeRowID int + + // reloadMu makes one reload atomic from the read of server_settings through + // the config swap and its callbacks; see reload. + reloadMu sync.Mutex + // fetchSettings, when set, replaces the read of server_settings. Only a + // test sets it — nothing in production needs to read settings from + // anywhere but the database. + fetchSettingsFn func(context.Context) (map[string]string, error) +} + +// rememberNodeRowID records the row a lookup resolved to. +func (w *Watcher) rememberNodeRowID(id int) { + if id <= 0 { + return + } + w.mu.Lock() + w.nodeRowID = id + w.mu.Unlock() +} + +// rememberedNodeRowID returns the row this worker previously resolved to. +func (w *Watcher) rememberedNodeRowID() (int, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + return w.nodeRowID, w.nodeRowID > 0 +} + +// forgetNodeRowID drops a remembered row that no longer exists. +func (w *Watcher) forgetNodeRowID() { + w.mu.Lock() + w.nodeRowID = 0 + w.mu.Unlock() } // NewWatcher creates a new config watcher. Call Start to begin watching. The @@ -48,13 +121,15 @@ type Watcher struct { // before they reach config.LoadFromDB, so a hot reload never feeds ciphertext // into the live config (which would, e.g., break JWT validation). func NewWatcher(pool *pgxpool.Pool, cipher *secret.Cipher, eventBus cache.EventBus, bootstrap BootstrapOverrides) *Watcher { - return &Watcher{ + w := &Watcher{ pool: pool, cipher: cipher, eventBus: eventBus, bootstrap: bootstrap, reloadCh: make(chan struct{}, 1), } + w.loadOverrides = w.queryNodeHWOverrides + return w } // OnLoad registers a normalization applied to every config this watcher @@ -108,13 +183,19 @@ func (w *Watcher) Start(ctx context.Context) error { // ForceReload triggers an immediate config reload from the database. func (w *Watcher) ForceReload(ctx context.Context) error { + if w == nil || (w.pool == nil && w.fetchSettingsFn == nil) { + // A watcher with no database is one a test constructed, or a mode that + // runs entirely off bootstrap overrides. Either way there is nothing to + // re-read, and an error is a far better answer than the nil dereference + // this used to be for a route that an operator can reach. + return errors.New("no database pool") + } return w.reload(ctx) } // RequestReload asks the poll goroutine to reload soon. Non-blocking and -// coalescing — safe to call from request handlers. Unlike ForceReload, the -// reload runs on the poll goroutine, so concurrent requests can never swap a -// stale snapshot over a newer one. +// coalescing — safe to call from request handlers. Unlike ForceReload it does +// not wait for the reload, and does not report whether it succeeded. func (w *Watcher) RequestReload() { select { case w.reloadCh <- struct{}{}: @@ -133,12 +214,36 @@ func (w *Watcher) SetConfigForTest(cfg *config.Config) { // reload fetches all settings from the database, builds a new Config, // applies bootstrap overrides, and atomically swaps the config pointer. +// +// Read and swap are one critical section. Reloads arrive from three places — +// the 60s poll, the settings-changed event, and ForceReload on a request +// goroutine — and only the first two share a goroutine. Letting them overlap +// would make the winner the one that finished last rather than the one that +// read last: a poll that sampled server_settings before an operator's edit can +// return after ForceReload has already applied the edit, and put its pre-edit +// snapshot back. The node would then answer 204 to the endpoint, the API would +// reload its pool believing the node adopted the new backend and device, and +// the node would go on transcoding with the old ones until something reloaded +// it again. +// +// Serializing the whole operation, rather than just the swap, is what fixes +// that: a reload's fetch cannot begin until the previous reload's swap is +// done, so a later swap is always built on a later read. The callbacks run +// inside the section too, so an older reload can't announce its config as the +// current one after a newer swap. func (w *Watcher) reload(ctx context.Context) error { - m, err := w.fetchSettings(ctx) + w.reloadMu.Lock() + defer w.reloadMu.Unlock() + + fetch := w.fetchSettings + if w.fetchSettingsFn != nil { + fetch = w.fetchSettingsFn + } + m, err := fetch(ctx) if err != nil { return err } - return w.applySettings(m) + return w.applySettings(ctx, m) } // fetchSettings reads all server_settings rows and decrypts sensitive values. @@ -171,9 +276,10 @@ func (w *Watcher) fetchSettings(ctx context.Context) (map[string]string, error) } // applySettings builds a Config from a plaintext settings map, re-applies -// bootstrap overrides, swaps the config pointer, and notifies OnChange -// callbacks when the config actually changed. -func (w *Watcher) applySettings(m map[string]string) error { +// bootstrap overrides, overlays this node's own acceleration policy, swaps the +// config pointer, and notifies OnChange callbacks when the config actually +// changed. +func (w *Watcher) applySettings(ctx context.Context, m map[string]string) error { newCfg, err := config.LoadFromDB(m) if err != nil { return fmt.Errorf("parse config: %w", err) @@ -200,6 +306,11 @@ func (w *Watcher) applySettings(m map[string]string) error { normalize(newCfg) } + // Last word, after the bootstrap re-apply and the normalizers: the node's + // own row decides its acceleration policy, and nothing above may put the + // cluster value back. + w.applyNodeHWOverrides(ctx, newCfg) + w.mu.Lock() old := w.cfg w.cfg = newCfg @@ -220,6 +331,221 @@ func (w *Watcher) applySettings(m map[string]string) error { return nil } +// applyNodeHWOverrides overlays this node's stored acceleration policy onto a +// freshly built config, so everything downstream — probes, warmup, the node's +// own fallback when a start request omits a backend — reads one effective +// value rather than consulting the row separately. +// +// It is a no-op on a host with no node identity (the API server, which has no +// stream_nodes row). Failure is deliberately conservative: a node that cannot +// read its row keeps the overlay it last read, because a database hiccup is +// not evidence that an operator cleared the override. +func (w *Watcher) applyNodeHWOverrides(ctx context.Context, cfg *config.Config) { + if (w.bootstrap.NodeURL == "" && w.bootstrap.NodeName == "") || w.loadOverrides == nil || cfg == nil { + return + } + + overrides, found, err := w.loadOverrides(ctx, w.bootstrap.NodeURL, w.bootstrap.NodeName) + switch { + case err != nil: + slog.WarnContext(ctx, "node acceleration override lookup failed; keeping the previous effective policy", + "component", "nodeconfig", "error", err) + var loaded bool + w.mu.RLock() + overrides, loaded = w.overrides, w.overridesLoaded + w.mu.RUnlock() + if !loaded { + // Never read one: the cluster-wide settings stand as they are. + return + } + case !found: + // Logged once rather than on every 60s reload: an unregistered node is + // a standing condition, not an event. + w.mu.Lock() + first := !w.missingRowLogged + w.missingRowLogged = true + previous, hadOverrides := w.overrides, w.overridesLoaded + if !hadOverrides { + w.overrides, w.overridesLoaded = nodeHWOverrides{}, true + } + w.mu.Unlock() + if first { + slog.InfoContext(ctx, "no stream_nodes row for this node; inheriting the cluster acceleration settings", + "component", "nodeconfig", "node_url", w.bootstrap.NodeURL, "node_name", w.bootstrap.NodeName) + } + if !hadOverrides { + return + } + // The row was there and now is not. On a split-horizon deployment the + // match is by NODE_NAME, and renaming a node through the admin form + // leaves this worker's environment pointing at a name nothing carries — + // so "no row" arrives while the API is still dispatching that row's + // overridden backend. Reverting to the cluster device here would pair + // the two wrongly for as long as the names disagree. A row that has + // gone is not evidence an operator cleared the override, so the last + // one read stands, exactly as it does when the lookup errors. + slog.WarnContext(ctx, "this node's stream_nodes row is no longer matchable; keeping the acceleration policy last read from it", + "component", "nodeconfig", "node_url", w.bootstrap.NodeURL, "node_name", w.bootstrap.NodeName) + overrides = previous + default: + w.mu.Lock() + w.overrides, w.overridesLoaded = overrides, true + w.mu.Unlock() + } + + if overrides.HWAccel != nil { + cfg.Playback.HWAccel = *overrides.HWAccel + } + if overrides.HWDevice != nil { + cfg.Playback.HWDevice = *overrides.HWDevice + } +} + +// queryNodeHWOverrides reads this node's own stream_nodes row. The URL is +// matched with trailing slashes ignored on both sides, because NODE_URL and +// the registered URL are typed by different people; the scan this costs is +// irrelevant next to getting the match wrong and silently inheriting the +// cluster policy. +// +// That tolerance can match two rows, though: stream_nodes.url is unique on the +// exact string, so "http://n1" and "http://n1/" are two legal registrations +// that rtrim collapses into one key. Ordering by id makes the winner the same +// on every reload — without it the seq scan returns whichever row it reached +// first, and the 30-second health sweep rewriting those rows would silently +// flip a node between two policies. The duplicate is reported rather than +// quietly resolved, because only an operator can fix it. +func (w *Watcher) queryNodeHWOverrides(ctx context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { + if w.pool == nil { + return nodeHWOverrides{}, false, errors.New("no database pool") + } + // The row this worker already resolved to, by its immutable id. + // + // Neither of the identities below survives an edit: a repoint changes the + // url, and a rename changes the name — and on a split-horizon deployment the + // name is the only match there was, so renaming a node severs the + // association permanently. The worker then keeps whatever policy it last + // read while the API dispatches the row's current one, which is the exact + // backend/device mismatch this overlay exists to prevent. Once the row has + // been identified, its id is what identifies it. + if id, ok := w.rememberedNodeRowID(); ok { + overrides, _, matched, err := w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE id = $1`, id) + if err != nil { + return nodeHWOverrides{}, false, err + } + if len(matched) > 0 { + return overrides, true, nil + } + // The row was deleted. Forget it and fall back to the identities, which + // is how a re-registered node finds its new row. + w.forgetNodeRowID() + } + + overrides, id, matched, err := w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE rtrim(url, '/') = rtrim($1, '/') ORDER BY id LIMIT 2`, nodeURL) + if err != nil { + return nodeHWOverrides{}, false, err + } + if len(matched) > 1 { + w.logDuplicateNodeRows(ctx, nodeURL, matched) + } + if len(matched) > 0 { + w.rememberNodeRowID(id) + return overrides, true, nil + } + + // The registered url is how the API reaches the node, which on a + // split-horizon topology (public CDN url registered, internal NODE_URL on + // the node) never equals the node's own address. The name is the identity + // an operator controls on both sides, so it is the fallback — but names + // carry no unique constraint, so an ambiguous match identifies nothing. + if nodeName == "" { + return nodeHWOverrides{}, false, nil + } + overrides, id, matched, err = w.queryOverrideRows(ctx, + `SELECT id, url, hw_accel_override, hw_device_override FROM stream_nodes + WHERE name = $1 ORDER BY id LIMIT 2`, nodeName) + if err != nil { + return nodeHWOverrides{}, false, err + } + switch len(matched) { + case 0: + return nodeHWOverrides{}, false, nil + case 1: + w.rememberNodeRowID(id) + return overrides, true, nil + default: + w.logAmbiguousNodeName(ctx, nodeName, matched) + return nodeHWOverrides{}, false, nil + } +} + +// queryOverrideRows runs one identity query and returns the first row plus the +// urls of every row it matched, so callers can act on ambiguity. +func (w *Watcher) queryOverrideRows(ctx context.Context, query string, arg any) (nodeHWOverrides, int, []string, error) { + rows, err := w.pool.Query(ctx, query, arg) + if err != nil { + return nodeHWOverrides{}, 0, nil, fmt.Errorf("query node acceleration overrides: %w", err) + } + defer rows.Close() + + var ( + overrides nodeHWOverrides + firstID int + matched []string + ) + for rows.Next() { + var ( + id int + url string + row nodeHWOverrides + ) + if err := rows.Scan(&id, &url, &row.HWAccel, &row.HWDevice); err != nil { + return nodeHWOverrides{}, 0, nil, fmt.Errorf("scan node acceleration overrides: %w", err) + } + if len(matched) == 0 { + overrides, firstID = row, id + } + matched = append(matched, url) + } + if err := rows.Err(); err != nil { + return nodeHWOverrides{}, 0, nil, fmt.Errorf("read node acceleration overrides: %w", err) + } + return overrides, firstID, matched, nil +} + +// logAmbiguousNodeName warns once per process: several registered nodes share +// this node's NODE_NAME, so the name identifies nothing and the overrides of +// none of them are adopted. +func (w *Watcher) logAmbiguousNodeName(ctx context.Context, nodeName string, matched []string) { + w.mu.Lock() + first := !w.ambiguousNameLogged + w.ambiguousNameLogged = true + w.mu.Unlock() + if first { + slog.WarnContext(ctx, "several stream_nodes rows share this node's name; ignoring their acceleration overrides", + "component", "nodeconfig", "node_name", nodeName, "matched_urls", matched) + } +} + +// logDuplicateNodeRows warns, once per process, that more than one +// stream_nodes row claims this node's URL. Once rather than every 60s: the +// duplicate is a standing misconfiguration, and the lowest id keeps winning +// until an operator removes the other row. +func (w *Watcher) logDuplicateNodeRows(ctx context.Context, nodeURL string, matched []string) { + w.mu.Lock() + first := !w.duplicateRowLogged + w.duplicateRowLogged = true + w.mu.Unlock() + if !first { + return + } + slog.WarnContext(ctx, "several stream_nodes rows match this node's URL; using the lowest id", + "component", "nodeconfig", "node_url", nodeURL, "matched_urls", matched) +} + // poll runs the background loop that reloads config on timer or event. func (w *Watcher) poll(ctx context.Context) { ticker := time.NewTicker(60 * time.Second) diff --git a/internal/nodeconfig/watcher_overrides_db_test.go b/internal/nodeconfig/watcher_overrides_db_test.go new file mode 100644 index 000000000..4c3d5e0b0 --- /dev/null +++ b/internal/nodeconfig/watcher_overrides_db_test.go @@ -0,0 +1,172 @@ +package nodeconfig + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// newOverrideTestPool follows the repository-wide convention for tests that +// need a real database: skip unless one is configured, and skip again if it +// predates the migration under test rather than failing on a missing column. +func newOverrideTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + var columns int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM information_schema.columns + WHERE table_name = 'stream_nodes' AND column_name = 'hw_accel_override'`).Scan(&columns); err != nil || columns < 1 { + t.Skip("test database has not applied the stream_nodes override migration") + } + return pool +} + +func insertOverrideNode(t *testing.T, pool *pgxpool.Pool, url string, accel *string) int { + t.Helper() + return insertOverrideNodeNamed(t, pool, fmt.Sprintf("override-%d", time.Now().UnixNano()), url, accel) +} + +// insertOverrideNodeNamed is insertOverrideNode with an explicit registered +// name, for the name-fallback tests below where the name (not the +// auto-generated one) is the thing under test. +func insertOverrideNodeNamed(t *testing.T, pool *pgxpool.Pool, name, url string, accel *string) int { + t.Helper() + ctx := context.Background() + var id int + if err := pool.QueryRow(ctx, + `INSERT INTO stream_nodes (name, type, url, hw_accel_override) + VALUES ($1, 'transcode', $2, $3) RETURNING id`, + name, url, accel).Scan(&id); err != nil { + t.Fatalf("insert node %q: %v", url, err) + } + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `DELETE FROM stream_nodes WHERE id = $1`, id); err != nil { + t.Errorf("cleanup node %d: %v", id, err) + } + }) + return id +} + +// stream_nodes.url is unique on the exact string, so a node registered twice — +// once with a trailing slash — is two legal rows that the lookup's rtrim +// tolerance collapses into one key. The winner has to be the same on every +// reload: without an explicit order the seq scan returns whichever row it +// reaches first, and the 30-second health sweep rewriting those rows would +// silently flip the node between two acceleration policies mid-deployment. +func TestQueryNodeHWOverridesPicksTheSameRowAcrossReloads(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + base := fmt.Sprintf("http://dup-node-%d:8082", time.Now().UnixNano()) + + pinned := "none" + firstID := insertOverrideNode(t, pool, base, &pinned) + insertOverrideNode(t, pool, base+"/", nil) + + w := &Watcher{pool: pool} + assertPinned := func(stage string) { + t.Helper() + overrides, found, err := w.queryNodeHWOverrides(ctx, base, "") + if err != nil { + t.Fatalf("%s: lookup: %v", stage, err) + } + if !found { + t.Fatalf("%s: node row not found", stage) + } + if overrides.HWAccel == nil || *overrides.HWAccel != pinned { + t.Fatalf("%s: hw_accel override = %v, want the lowest-id row's %q", stage, overrides.HWAccel, pinned) + } + } + + assertPinned("initial load") + + // What the health sweep does every 30 seconds. It moves the tuple, and with + // it the physical order an unordered scan would have followed. + if _, err := pool.Exec(ctx, + `UPDATE stream_nodes SET healthy = NOT healthy, last_health_check = NOW() WHERE id = $1`, firstID); err != nil { + t.Fatalf("health sweep update: %v", err) + } + + assertPinned("after a health sweep rewrote the row") +} + +// A split-horizon node — public url registered, internal NODE_URL on the box +// — has no url match at all, so the lookup falls back to the registered name. +func TestQueryNodeHWOverridesFallsBackToUniqueName(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + registeredURL := fmt.Sprintf("https://public-%d.example.com", time.Now().UnixNano()) + pinned := "vaapi" + insertOverrideNodeNamed(t, pool, name, registeredURL, &pinned) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, "http://10.0.4.7:8082", name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if !found { + t.Fatal("name-matched row not found") + } + if overrides.HWAccel == nil || *overrides.HWAccel != pinned { + t.Fatalf("hw_accel override = %v, want the name-matched row's %q", overrides.HWAccel, pinned) + } +} + +// Two rows sharing the fallback name identify nothing: names carry no unique +// constraint, so an ambiguous match must not silently adopt either row's +// overrides. +func TestQueryNodeHWOverridesAmbiguousNameIsNotFound(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + pinned := "none" + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://a-%d.example.com", time.Now().UnixNano()), &pinned) + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://b-%d.example.com", time.Now().UnixNano()), nil) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, "http://10.0.4.7:8082", name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if found { + t.Fatalf("ambiguous name matched, overrides = %+v", overrides) + } +} + +// The url match wins even when the name also identifies a different row: url +// is the primary identity, and name is only consulted when the url misses. +func TestQueryNodeHWOverridesURLWinsOverNameMatch(t *testing.T) { + pool := newOverrideTestPool(t) + ctx := context.Background() + url := fmt.Sprintf("http://url-match-%d:8082", time.Now().UnixNano()) + name := fmt.Sprintf("silo-name-%d", time.Now().UnixNano()) + byURL := "qsv" + byName := "nvenc" + insertOverrideNodeNamed(t, pool, fmt.Sprintf("other-%d", time.Now().UnixNano()), url, &byURL) + insertOverrideNodeNamed(t, pool, name, fmt.Sprintf("https://different-%d.example.com", time.Now().UnixNano()), &byName) + + w := &Watcher{pool: pool} + overrides, found, err := w.queryNodeHWOverrides(ctx, url, name) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if !found { + t.Fatal("url-matched row not found") + } + if overrides.HWAccel == nil || *overrides.HWAccel != byURL { + t.Fatalf("hw_accel override = %v, want the url-matched %q, not the name-matched row", overrides.HWAccel, byURL) + } +} diff --git a/internal/nodeconfig/watcher_overrides_test.go b/internal/nodeconfig/watcher_overrides_test.go new file mode 100644 index 000000000..70901a698 --- /dev/null +++ b/internal/nodeconfig/watcher_overrides_test.go @@ -0,0 +1,290 @@ +package nodeconfig + +import ( + "context" + "errors" + "testing" +) + +// clusterSettings is what every node in a QSV deployment loads before its own +// row is consulted. +func clusterSettings() map[string]string { + return map[string]string{ + "playback.hw_accel": "qsv", + "playback.hw_device": "/dev/dri/renderD128", + } +} + +func newOverrideWatcher(t *testing.T, nodeURL string, load loadNodeHWOverrides) *Watcher { + t.Helper() + w := NewWatcher(nil, nil, nil, BootstrapOverrides{NodeURL: nodeURL}) + w.loadOverrides = load + return w +} + +// A node's own row wins over the cluster-wide settings: this is the whole +// mechanism behind a CPU-only box living in a hardware-accelerated cluster. +func TestApplySettingsOverlaysNodeHWOverrides(t *testing.T) { + accel, device := "none", "/dev/dri/renderD129" + tests := []struct { + name string + overrides nodeHWOverrides + wantAccel string + wantDevice string + }{ + { + name: "both overridden", + overrides: nodeHWOverrides{HWAccel: &accel, HWDevice: &device}, + wantAccel: "none", + wantDevice: "/dev/dri/renderD129", + }, + { + name: "backend only, device still inherited", + overrides: nodeHWOverrides{HWAccel: &accel}, + wantAccel: "none", + wantDevice: "/dev/dri/renderD128", + }, + { + name: "device only, backend still inherited", + overrides: nodeHWOverrides{HWDevice: &device}, + wantAccel: "qsv", + wantDevice: "/dev/dri/renderD129", + }, + { + name: "row with no overrides inherits both", + wantAccel: "qsv", + wantDevice: "/dev/dri/renderD128", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + return test.overrides, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != test.wantAccel || cfg.Playback.HWDevice != test.wantDevice { + t.Fatalf("effective policy = %q / %q, want %q / %q", + cfg.Playback.HWAccel, cfg.Playback.HWDevice, test.wantAccel, test.wantDevice) + } + }) + } +} + +// The API host has no stream_nodes row and must never pay for a lookup. +func TestApplySettingsSkipsOverlayWithoutNodeIdentity(t *testing.T) { + looked := false + w := newOverrideWatcher(t, "", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + looked = true + return nodeHWOverrides{}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if looked { + t.Fatal("a host with no NodeURL or NodeName queried stream_nodes") + } + if got := w.Config().Playback.HWAccel; got != "qsv" { + t.Fatalf("HWAccel = %q, want the cluster value", got) + } +} + +// A split-horizon node whose registered url differs from its own NODE_URL +// still finds its row by NODE_NAME — the fallback identity an operator +// controls on both sides. The fake models the DB-level fallback behavior: +// it only reports a hit when the name matches, and this test asserts the +// overlay guard passes both bootstrap values through to the loader. +func TestApplySettingsOverlaysNodeHWOverridesByName(t *testing.T) { + accel := "vaapi" + var gotURL, gotName string + w := NewWatcher(nil, nil, nil, BootstrapOverrides{ + NodeURL: "http://10.0.4.7:8082", + NodeName: "silo-dev-transcode-ns17", + }) + w.loadOverrides = func(_ context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { + gotURL, gotName = nodeURL, nodeName + if nodeName == "silo-dev-transcode-ns17" { + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + return nodeHWOverrides{}, false, nil + } + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if gotURL != "http://10.0.4.7:8082" || gotName != "silo-dev-transcode-ns17" { + t.Fatalf("loader received url %q name %q, want both bootstrap values", gotURL, gotName) + } + if got := w.Config().Playback.HWAccel; got != "vaapi" { + t.Fatalf("HWAccel = %q, want the name-matched override", got) + } +} + +// A node with only NODE_NAME set (no NODE_URL at all) still runs the +// overlay: the guard fires when either identity is present, not just the url. +func TestApplySettingsOverlayRunsWithNodeNameOnly(t *testing.T) { + accel := "none" + looked := false + w := NewWatcher(nil, nil, nil, BootstrapOverrides{NodeName: "silo-dev-transcode-ns17"}) + w.loadOverrides = func(_ context.Context, nodeURL, nodeName string) (nodeHWOverrides, bool, error) { + looked = true + if nodeURL != "" { + t.Fatalf("nodeURL = %q, want empty", nodeURL) + } + if nodeName != "silo-dev-transcode-ns17" { + t.Fatalf("nodeName = %q, want the bootstrap value", nodeName) + } + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + if !looked { + t.Fatal("a host with NodeName set skipped the overlay lookup") + } + if got := w.Config().Playback.HWAccel; got != "none" { + t.Fatalf("HWAccel = %q, want the overlay applied", got) + } +} + +// An unregistered node keeps the cluster settings, and says so once rather +// than on every 60-second reload. +func TestApplySettingsMissingRowInheritsAndLogsOnce(t *testing.T) { + calls := 0 + w := newOverrideWatcher(t, "http://node-unregistered", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + calls++ + return nodeHWOverrides{}, false, nil + }) + for range 3 { + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } + if calls != 3 { + t.Fatalf("lookups = %d, want one per reload", calls) + } + w.mu.RLock() + logged := w.missingRowLogged + w.mu.RUnlock() + if !logged { + t.Fatal("missing row was never recorded, so it would be logged on every reload") + } +} + +// A database hiccup must not flip a node back onto the cluster-wide backend: +// an unreadable row is not evidence that an operator cleared the override. +func TestApplySettingsKeepsPreviousOverrideWhenTheLookupFails(t *testing.T) { + accel := "nvenc" + fail := false + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + if fail { + return nodeHWOverrides{}, false, errors.New("connection refused") + } + return nodeHWOverrides{HWAccel: &accel}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("first apply: %v", err) + } + if got := w.Config().Playback.HWAccel; got != "nvenc" { + t.Fatalf("HWAccel = %q, want the stored override", got) + } + + fail = true + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply during outage: %v", err) + } + if got := w.Config().Playback.HWAccel; got != "nvenc" { + t.Fatalf("HWAccel = %q after a failed lookup, want the last known override", got) + } +} + +// Before any successful read there is nothing to keep, so the cluster settings +// stand rather than an invented value. +func TestApplySettingsFallsBackToClusterWhenNoOverrideWasEverRead(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{}, false, errors.New("connection refused") + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} + +// The overlay is the last word: a bootstrap re-apply happens before it, and +// nothing may put the cluster value back afterwards. +func TestApplySettingsOverlayOutlivesBootstrapReapply(t *testing.T) { + accel := "none" + w := NewWatcher(nil, nil, nil, BootstrapOverrides{ + NodeURL: "http://node-1", + Listen: ":9999", + Mode: "transcode", + }) + w.loadOverrides = func(context.Context, string, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{HWAccel: &accel}, true, nil + } + settings := clusterSettings() + settings["server.listen"] = ":8080" + if err := w.applySettings(context.Background(), settings); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Server.Listen != ":9999" || cfg.Server.Mode != "transcode" { + t.Fatalf("bootstrap overrides lost: listen %q mode %q", cfg.Server.Listen, cfg.Server.Mode) + } + if cfg.Playback.HWAccel != "none" { + t.Fatalf("HWAccel = %q, want the node override", cfg.Playback.HWAccel) + } +} + +// On a split-horizon deployment the row is matched by NODE_NAME, and renaming +// the node through the admin form leaves this worker's environment pointing at a +// name nothing carries. "No row" then arrives while the API is still dispatching +// that row's overridden backend, so reverting to the cluster device here would +// pair the two wrongly for as long as the names disagree. A row that has gone is +// not evidence an operator cleared the override. +func TestApplySettingsKeepsOverrideWhenTheRowStopsMatching(t *testing.T) { + accel, device := "nvenc", "0" + found := true + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + if !found { + return nodeHWOverrides{}, false, nil + } + return nodeHWOverrides{HWAccel: &accel, HWDevice: &device}, true, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("first apply: %v", err) + } + + found = false + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply after the rename: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "nvenc" || cfg.Playback.HWDevice != "0" { + t.Fatalf("effective policy = %q / %q, want the last override read from the row", + cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} + +// A node that never had a row is a different case: there is nothing to keep, so +// the cluster settings stand rather than an invented value. +func TestApplySettingsInheritsClusterWhenNoRowWasEverFound(t *testing.T) { + w := newOverrideWatcher(t, "http://node-1", func(context.Context, string, string) (nodeHWOverrides, bool, error) { + return nodeHWOverrides{}, false, nil + }) + if err := w.applySettings(context.Background(), clusterSettings()); err != nil { + t.Fatalf("apply: %v", err) + } + cfg := w.Config() + if cfg.Playback.HWAccel != "qsv" || cfg.Playback.HWDevice != "/dev/dri/renderD128" { + t.Fatalf("effective policy = %q / %q, want the cluster values", cfg.Playback.HWAccel, cfg.Playback.HWDevice) + } +} diff --git a/internal/nodeconfig/watcher_test.go b/internal/nodeconfig/watcher_test.go index 46e9d197c..f382b9936 100644 --- a/internal/nodeconfig/watcher_test.go +++ b/internal/nodeconfig/watcher_test.go @@ -1,7 +1,12 @@ package nodeconfig import ( + "context" + "slices" + "sync" + "sync/atomic" "testing" + "time" "github.com/Silo-Server/silo-server/internal/config" ) @@ -20,7 +25,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { }) settings := map[string]string{"server.log_level": "debug"} - if err := w.applySettings(settings); err != nil { + if err := w.applySettings(context.Background(), settings); err != nil { t.Fatalf("first apply: %v", err) } if calls != 1 { @@ -28,7 +33,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { } // Same settings again — pointer swaps, but callbacks must not fire. - if err := w.applySettings(settings); err != nil { + if err := w.applySettings(context.Background(), settings); err != nil { t.Fatalf("second apply: %v", err) } if calls != 1 { @@ -36,7 +41,7 @@ func TestApplySettingsSkipsCallbacksOnNoopReload(t *testing.T) { } // A real change fires callbacks again. - if err := w.applySettings(map[string]string{"server.log_level": "warn"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "warn"}); err != nil { t.Fatalf("third apply: %v", err) } if calls != 2 { @@ -50,7 +55,7 @@ func TestApplySettingsReappliesBootstrapOverrides(t *testing.T) { RedisURL: "redis://env-host:6379", }) - err := w.applySettings(map[string]string{ + err := w.applySettings(context.Background(), map[string]string{ "server.listen": ":8080", "redis.url": "redis://db-host:6379", }) @@ -84,7 +89,7 @@ func TestRequestReloadCoalesces(t *testing.T) { func TestOnChangeAfterFirstApplySeesLaterChanges(t *testing.T) { w := newTestWatcher(t, BootstrapOverrides{}) - if err := w.applySettings(map[string]string{"server.log_level": "info"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "info"}); err != nil { t.Fatalf("initial apply: %v", err) } @@ -94,7 +99,7 @@ func TestOnChangeAfterFirstApplySeesLaterChanges(t *testing.T) { gotNew = updated.Server.LogLevel }) - if err := w.applySettings(map[string]string{"server.log_level": "error"}); err != nil { + if err := w.applySettings(context.Background(), map[string]string{"server.log_level": "error"}); err != nil { t.Fatalf("second apply: %v", err) } if gotOld != "info" || gotNew != "error" { @@ -108,7 +113,7 @@ func TestOnLoadNormalizersApplyOnEveryLoad(t *testing.T) { c.Playback.FFmpegPath = "/resolved/ffmpeg" }) - if err := w.applySettings(map[string]string{ + if err := w.applySettings(context.Background(), map[string]string{ "playback.ffmpeg_path": "/usr/lib/jellyfin-ffmpeg/ffmpeg", }); err != nil { t.Fatalf("applySettings() error = %v", err) @@ -118,7 +123,7 @@ func TestOnLoadNormalizersApplyOnEveryLoad(t *testing.T) { } // A reload constructs a fresh config; the normalizer must apply again. - if err := w.applySettings(map[string]string{ + if err := w.applySettings(context.Background(), map[string]string{ "playback.ffmpeg_path": "/usr/lib/jellyfin-ffmpeg/ffmpeg", }); err != nil { t.Fatalf("applySettings() reload error = %v", err) @@ -127,3 +132,71 @@ func TestOnLoadNormalizersApplyOnEveryLoad(t *testing.T) { t.Fatalf("after reload Playback.FFmpegPath = %q, want normalized value", got) } } + +// staleReloadWindow is how long the poll's fetch waits for a concurrent forced +// reload to overtake it. It is a bound on a violation, not a wait for +// completion: serialized, nothing can overtake and the wait expires; unserialized, +// the forced reload finishes far inside it and the poll's stale snapshot lands +// last. +const staleReloadWindow = 250 * time.Millisecond + +// The poll, the settings-changed event, and ForceReload all reload, and only +// the first two share a goroutine. A poll that read server_settings before an +// operator's edit must not be able to put that pre-edit snapshot back after +// ForceReload has already applied the edit — the node would answer the endpoint +// 204 while running the old policy. +func TestReloadDoesNotLetAnEarlierSnapshotSupersedeALaterOne(t *testing.T) { + w := &Watcher{} + + var mu sync.Mutex + var events []string + record := func(event string) { + mu.Lock() + defer mu.Unlock() + events = append(events, event) + } + w.OnChange(func(_, updated *config.Config) { record("apply " + updated.Server.LogLevel) }) + + polling := make(chan struct{}) + forced := make(chan struct{}) + var fetches atomic.Int32 + w.fetchSettingsFn = func(context.Context) (map[string]string, error) { + if fetches.Add(1) == 1 { + record("fetch info") + close(polling) + select { + case <-forced: + case <-time.After(staleReloadWindow): + } + return map[string]string{"server.log_level": "info"}, nil + } + record("fetch error") + return map[string]string{"server.log_level": "error"}, nil + } + + polled := make(chan error, 1) + go func() { polled <- w.reload(context.Background()) }() + <-polling + + go func() { + if err := w.ForceReload(context.Background()); err != nil { + t.Errorf("ForceReload() error = %v", err) + } + close(forced) + }() + + if err := <-polled; err != nil { + t.Fatalf("poll reload error = %v", err) + } + <-forced + + if got := w.Config().Server.LogLevel; got != "error" { + t.Errorf("Server.LogLevel = %q, want the value the later read saw", got) + } + mu.Lock() + defer mu.Unlock() + want := []string{"fetch info", "apply info", "fetch error", "apply error"} + if !slices.Equal(events, want) { + t.Errorf("reloads interleaved:\n got %v\nwant %v", events, want) + } +} diff --git a/internal/nodemetrics/cgroupcpu.go b/internal/nodemetrics/cgroupcpu.go new file mode 100644 index 000000000..aaf8b9bab --- /dev/null +++ b/internal/nodemetrics/cgroupcpu.go @@ -0,0 +1,357 @@ +package nodemetrics + +import ( + "fmt" + "math" + "os" + "strconv" + "strings" + "time" +) + +// cgroup CPU correction. +// +// /proc/stat describes the host even inside a container, exactly as +// /proc/meminfo does. A transcode node limited to two cores on a 64-core host +// would otherwise report the host's busyness against the host's core count — +// pinned at its quota and dropping segments while the dashboard shows a few +// percent idle. So CPU is corrected the same way memory is: the cgroup's own +// cumulative usage is the busy signal, and its quota is what that usage is +// normalized against. +// +// The correction applies only where something actually caps CPU. A cgroup +// imposes no limit far more often than it imposes one — every unconstrained +// container and systemd service has one — and its usage then describes Silo +// alone rather than the machine, so an uncapped deployment keeps reading +// /proc/stat and reports the load it is really competing with. + +// cgroupCPUPath locates one cgroup version's CPU accounting. +type cgroupCPUPath struct { + // usage is the file holding cumulative CPU time consumed by the cgroup. + usage string + // usageKey names the row to read when usage is a "key value" table; empty + // when the file holds a bare integer. + usageKey string + // usageUnit is how long one unit in that file is. + usageUnit time.Duration + // quota holds the CPU budget: cgroup v2's " " pair, or + // cgroup v1's quota alone. + quota string + // period is v1's separate period file; empty when quota carries both. + period string +} + +// cgroupCPUUsageKey names the cumulative-usage row of cgroup v2's cpu.stat. +const cgroupCPUUsageKey = "usage_usec" + +// cgroupCPUPaths lists where to read CPU accounting, v2 first. cgroup v1 mounts +// cpu and cpuacct together on most distributions and separately on some, so +// both layouts are tried. +var cgroupCPUPaths = []cgroupCPUPath{ + { + usage: "/sys/fs/cgroup/cpu.stat", + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: "/sys/fs/cgroup/cpu.max", + }, + { + usage: "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage", + usageUnit: time.Nanosecond, + quota: "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us", + period: "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us", + }, + { + usage: "/sys/fs/cgroup/cpuacct/cpuacct.usage", + usageUnit: time.Nanosecond, + quota: "/sys/fs/cgroup/cpu/cpu.cfs_quota_us", + period: "/sys/fs/cgroup/cpu/cpu.cfs_period_us", + }, +} + +// cgroupCPUSetPaths lists where a cgroup publishes the CPUs it may run on, v2 +// first. The "effective" file is what the kernel actually allows after +// intersecting with every ancestor, which is the number a process is really +// bounded by; the plain file is the request, and is read only where the +// effective one is absent. +var cgroupCPUSetPaths = []string{ + "/sys/fs/cgroup/cpuset.cpus.effective", + "/sys/fs/cgroup/cpuset/cpuset.effective_cpus", + "/sys/fs/cgroup/cpuset.cpus", + "/sys/fs/cgroup/cpuset/cpuset.cpus", +} + +// cgroupCPUSetCores counts the CPUs this cgroup may run on, or 0 when it is not +// restricted to a subset. +// +// A cpuset is the other way a deployment caps CPU, and unlike a CFS quota it +// leaves cpu.max saying "max". Without reading it, a process pinned to two CPUs +// on a sixty-four core host divides its own busy time by sixty-four and reports +// three percent while it is saturated — which defeats the whole point of the +// correction. +// +// The effective file already accounts for ancestors, so unlike the quota this +// needs no walk of its own; where only the pre-intersection file exists, the +// per-level candidates cover the same ground. +// +// A cpuset that spans the whole host is no cpuset at all — see cgroupCapBinds, +// which is the rule, and which applies to the CFS quota the same way. +func cgroupCPUSetCores(paths []string, hostCores int) int { + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + continue + } + count := countCPUSetEntries(string(raw)) + if count <= 0 { + continue + } + if !cgroupCapBinds(float64(count), hostCores) { + // The effective set is the intersection with every ancestor, so no + // file later in the list can narrow what this one just said was the + // whole machine. + return 0 + } + return count + } + return 0 +} + +// cgroupCapBinds reports whether a CPU cap of the given size in cores actually +// restricts a process on a host with hostCores CPUs. +// +// A cap as large as the machine restricts nothing, and it is not a rare +// misconfiguration: every unconstrained container and service publishes an +// effective cpuset holding every online CPU, because it inherits one from a root +// that does, and a deployment sized to "the whole box" writes a quota to match. +// +// Reading such a cap as binding is worse than ignoring it. The cap decides which +// cgroup's usage is measured, not just what it is divided by, so an idle Silo +// beside a saturated neighbor on a shared host would report its own few percent +// as the machine's load — the exact misreport this whole correction exists to +// prevent, arrived at from the other direction. +// +// A host size of 0 means /proc/stat could not be counted, and an unknown host is +// no reason to discard a cap that may well be real. +func cgroupCapBinds(cores float64, hostCores int) bool { + if cores <= 0 { + return false + } + return hostCores <= 0 || cores < float64(hostCores) +} + +// countCPUSetEntries counts the CPUs in a Linux cpu list ("0-3,8,12-13"). +// An unparseable or empty list counts nothing rather than guessing. +func countCPUSetEntries(list string) int { + total := 0 + for _, part := range strings.Split(strings.TrimSpace(list), ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + low, high, ranged := strings.Cut(part, "-") + first, err := strconv.Atoi(strings.TrimSpace(low)) + if err != nil || first < 0 { + continue + } + if !ranged { + total++ + continue + } + last, err := strconv.Atoi(strings.TrimSpace(high)) + if err != nil || last < first { + continue + } + total += last - first + 1 + } + return total +} + +// cgroupCPUSample is one cumulative CPU-time reading. Only differences between +// two readings mean anything. +type cgroupCPUSample struct { + usageNS int64 + at time.Time + valid bool +} + +// cgroupCPU returns this process's cgroup CPU reading for the given instant, +// and the CPU budget in cores that reading must be normalized against (0 when +// nothing caps it). +// +// pinned is the size of this process's cpuset, or 0 when it is not restricted +// to a subset. It is a second kind of cap and it belongs here rather than at the +// caller, because which constraint binds decides which cgroup's usage the +// reading has to come from — and the two answers are different populations. +func (s *Sampler) cgroupCPU(now time.Time, pinned int) (cgroupCPUSample, float64) { + for _, paths := range s.cgroupCPUPaths { + if _, err := readCgroupCPUUsage(paths); err != nil { + continue + } + // Usage comes from whichever level supplied the binding quota, not from + // the leaf. They are one measurement: a quota shared with sibling + // services throttles on their CPU time too, so dividing only this + // process's by it reports ten percent for a group that is saturated and + // being throttled — the same pairing error the memory path had. + binding, quota := effectiveCgroupCPUQuota(paths) + + // A cpuset applies to this cgroup, not to the ancestor that owns the + // quota. So when it is the tighter cap the measurement moves back down + // with it: the ancestor's usage counts siblings that do not share this + // cpuset, and dividing that by a smaller private budget pins the node at + // a hundred percent while Silo is idle. + if pinned > 0 && (quota <= 0 || float64(pinned) < quota) { + binding, quota = paths, float64(pinned) + } + + usage, err := readCgroupCPUUsage(binding) + if err != nil { + continue + } + return cgroupCPUSample{usageNS: usage, at: now, valid: true}, quota + } + return cgroupCPUSample{}, 0 +} + +// effectiveCgroupCPUQuota returns the tightest CPU budget in force on this +// cgroup, in cores, together with the level that imposes it. +// +// The quota is whatever the kernel will actually throttle against, which is the +// smallest limit anywhere between here and the mount root. A systemd unit inside +// a slice with CPUQuota=, or a container under a limited pod cgroup, reads "max" +// at its leaf and is nonetheless capped; taking the leaf's answer would +// normalize a two-core service against sixty-four. +// +// The level is returned because usage has to be read from it too. A quota on a +// shared ancestor is spent by every service under it, so this process's own CPU +// time over that quota describes nothing: Silo at ten percent beside a sibling +// at ninety reports ten, while the group is saturated and throttled. +// +// Everything within a level moves together: a quota from one cgroup divided by a +// period from another describes no real budget. +func effectiveCgroupCPUQuota(paths cgroupCPUPath) (cgroupCPUPath, float64) { + quotas := cgroupAncestorPaths(paths.quota) + periods := cgroupAncestorPaths(paths.period) + usages := cgroupAncestorPaths(paths.usage) + binding, tightest := paths, 0.0 + for i, quota := range quotas { + level := paths + level.quota = quota + if i < len(usages) { + level.usage = usages[i] + } + if paths.period != "" { + if i >= len(periods) { + break + } + level.period = periods[i] + } + cores, err := readCgroupCPUQuota(level) + if err != nil || cores <= 0 { + continue + } + // Ties go to the outer level, which the walk reaches later. Two cgroups + // publishing the same quota are not equivalent: the ancestor's is shared + // with siblings that can exhaust it, so it is the one whose usage + // describes what is being throttled. Silo at 0.2 cores beside a sibling + // at 1.8 under a shared two-core parent reads ten percent from the leaf + // while the parent is saturated. + if tightest == 0 || cores <= tightest { + tightest, binding = cores, level + } + } + return binding, tightest +} + +// readCgroupCPUUsage returns cumulative cgroup CPU time in nanoseconds. +func readCgroupCPUUsage(paths cgroupCPUPath) (int64, error) { + var value int64 + var err error + if paths.usageKey != "" { + value, err = readCgroupStatKey(paths.usage, paths.usageKey) + } else { + value, err = readCgroupSingleValue(paths.usage) + } + if err != nil { + return 0, err + } + if value < 0 { + return 0, fmt.Errorf("negative cgroup cpu usage") + } + return value * int64(paths.usageUnit), nil +} + +// readCgroupCPUQuota returns the cgroup's CPU budget in cores, or an error when +// it imposes none. +// +// "No quota" is spelled "max" in v2 and "-1" in v1, and both must read as "this +// cgroup may use the whole host", never as a budget of zero cores. +func readCgroupCPUQuota(paths cgroupCPUPath) (float64, error) { + raw, err := os.ReadFile(paths.quota) + if err != nil { + return 0, err + } + fields := strings.Fields(string(raw)) + if len(fields) == 0 { + return 0, fmt.Errorf("empty cgroup cpu quota") + } + if fields[0] == "max" { + return 0, fmt.Errorf("no cgroup cpu quota") + } + quota, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, err + } + if quota <= 0 { + return 0, fmt.Errorf("no cgroup cpu quota") + } + + period := int64(0) + if len(fields) > 1 { + // cgroup v2 prints the period beside the quota. + period, err = strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, err + } + } else if paths.period != "" { + period, err = readCgroupSingleValue(paths.period) + if err != nil { + return 0, err + } + } + if period <= 0 { + return 0, fmt.Errorf("no cgroup cpu period") + } + return float64(quota) / float64(period), nil +} + +// cgroupCPUPercent converts two cgroup CPU readings into a busy percentage of +// the given core budget. +// +// A usage counter that went backwards means the readings do not describe one +// continuous run (the container was restarted or migrated), so the pair is +// unusable rather than negative. +func cgroupCPUPercent(previous, current cgroupCPUSample, cores float64) (int, bool) { + if !previous.valid || !current.valid || cores <= 0 { + return 0, false + } + elapsedNS := current.at.Sub(previous.at).Nanoseconds() + if elapsedNS <= 0 || current.usageNS < previous.usageNS { + return 0, false + } + busy := float64(current.usageNS-previous.usageNS) * 100 / (float64(elapsedNS) * cores) + return clampPercent(int(busy + 0.5)), true +} + +// cgroupQuotaCores rounds a fractional CPU quota up to whole cores, which is +// how many CPUs the workload can actually be running on at one instant. +func cgroupQuotaCores(quota float64, hostCores int) int { + cores := int(math.Ceil(quota)) + if cores < 1 { + cores = 1 + } + if hostCores > 0 && cores > hostCores { + // A quota above the host's core count is not a limit worth reporting. + return hostCores + } + return cores +} diff --git a/internal/nodemetrics/cgrouppath.go b/internal/nodemetrics/cgrouppath.go new file mode 100644 index 000000000..825824328 --- /dev/null +++ b/internal/nodemetrics/cgrouppath.go @@ -0,0 +1,225 @@ +package nodemetrics + +import ( + "os" + "path" + "strings" +) + +// Resolving this process's own cgroup. +// +// Every cgroup file this package reads is named from the mount root — +// /sys/fs/cgroup/cpu.max and friends. Inside a container that is exactly right: +// the cgroup namespace makes the container's own cgroup appear as the root, so +// the root files describe the container. +// +// A process that is limited *without* being namespaced sees something else. A +// systemd unit with CPUQuota= or MemoryMax= lives at +// /sys/fs/cgroup/system.slice/silo.service/, and the root files there describe +// the whole machine. Reading them reports host-wide CPU busyness and the host's +// memory total, so a service pegged at its quota looks mostly idle with plenty +// of memory left — the exact failure the cgroup correction exists to prevent, +// reintroduced for anyone who runs Silo as a plain service rather than a +// container. +// +// So each path is tried at this process's own cgroup first and at the root +// second. The fallback is what keeps every container case working unchanged: +// a namespaced container reports "/" and rewrites to the root anyway, and a +// container without a cgroup namespace reports a host path +// (/docker/) that does not exist under its own mount, so the rewritten +// path simply fails to open and the root read happens as before. + +// cgroupMountRoot is where the cgroup hierarchy is mounted on Linux. It is a +// var so a test can point the ancestor walk at a temporary tree; nothing in +// production writes it. +var cgroupMountRoot = "/sys/fs/cgroup" + +// cgroupRelativePaths reports this process's path within each cgroup hierarchy, +// read from /self/cgroup. +// +// The v2 unified hierarchy has an empty controller field and is keyed by "". +// A v1 line names one or more controllers, and is keyed both by each controller +// individually and by the whole comma-joined field, because that joined form is +// what the mount directory is named ("cpu,cpuacct"). +// +// A missing or unreadable file yields no entries, which leaves every path at +// the root — the behavior this process had before. +func cgroupRelativePaths(procDir string) map[string]string { + raw, err := os.ReadFile(path.Join(procDir, "self", "cgroup")) + if err != nil { + return nil + } + paths := map[string]string{} + for line := range strings.Lines(string(raw)) { + // "::", and the path may itself + // contain colons, so only the first two separators are structural. + fields := strings.SplitN(strings.TrimSpace(line), ":", 3) + if len(fields) != 3 { + continue + } + controllers, relative := fields[1], strings.TrimPrefix(fields[2], "/") + if relative == "" { + // Already at the root of its hierarchy: a namespaced container, or + // an unconstrained process. Recording it would rewrite to the same + // place at extra cost. + continue + } + paths[controllers] = relative + for _, controller := range strings.Split(controllers, ",") { + if controller != "" { + paths[controller] = relative + } + } + } + if len(paths) == 0 { + return nil + } + return paths +} + +// cgroupSelfFile rewrites a root-relative cgroup file path to this process's own +// cgroup, or returns "" when it already reads the right file. +// +// The controller is taken from the path itself: a v1 file sits under a +// controller directory (/sys/fs/cgroup/memory/memory.stat), a v2 file sits +// directly at the root (/sys/fs/cgroup/memory.stat) and belongs to the unified +// hierarchy, which cgroupRelativePaths keys by "". +func cgroupSelfFile(relative map[string]string, file string) string { + if len(relative) == 0 { + return "" + } + rest, ok := strings.CutPrefix(file, cgroupMountRoot+"/") + if !ok { + return "" + } + controller, name := "", rest + if dir, base, found := strings.Cut(rest, "/"); found { + controller, name = dir, base + } + own, ok := relative[controller] + if !ok || name == "" { + return "" + } + return path.Join(cgroupMountRoot, controller, own, name) +} + +// cgroupAncestorPaths returns file and the same file name at every cgroup above +// it, nearest first, ending at the hierarchy mount root. +// +// A limit is not always written where the process sits. A systemd unit can +// inherit its quota from the slice that contains it, and a container can inherit +// one from its pod cgroup; in both cases the leaf reads "max" while the kernel +// throttles against an ancestor. Reading only the leaf would report the host's +// whole capacity for a process that has far less. +// +// The walk is by path, so every level it produces is a genuine ancestor of this +// process. Levels that hold no such file simply fail to read, which is how the +// v1 layouts skip the unified root they never had. +func cgroupAncestorPaths(file string) []string { + if file == "" { + return nil + } + // The file itself is always a level. Only the walk above it needs the file + // to live under the cgroup mount — a test harness pointing these at a temp + // directory has no hierarchy to climb, and must still read what it was given. + if !strings.HasPrefix(file, cgroupMountRoot+"/") { + return []string{file} + } + name := path.Base(file) + out := []string{file} + for dir := path.Dir(file); strings.HasPrefix(dir, cgroupMountRoot); dir = path.Dir(dir) { + if candidate := path.Join(dir, name); candidate != file { + out = append(out, candidate) + } + if dir == cgroupMountRoot { + break + } + } + return out +} + +// withCgroupSelfPaths returns files preceded by their this-process equivalents +// and every cgroup between the two, so a read sees each limit in force on this +// process rather than only the nearest and the root. +// +// The caller picks the tightest of what it can read, not the first: an ancestor +// with a finite limit binds a leaf that says "max", so stopping at the first +// readable file would report no limit for a process that has one. +func withCgroupSelfPaths(relative map[string]string, files []string) []string { + out := make([]string, 0, len(files)*2) + seen := make(map[string]bool, len(files)*2) + add := func(candidate string) { + if candidate == "" || seen[candidate] { + return + } + seen[candidate] = true + out = append(out, candidate) + } + for _, file := range files { + if own := cgroupSelfFile(relative, file); own != "" { + for _, candidate := range cgroupAncestorPaths(own) { + add(candidate) + } + } + add(file) + } + return out +} + +// withCgroupSelfCPUPaths is withCgroupSelfPaths for the CPU layouts, where one +// entry names several files that must be rewritten together — a usage file from +// this process's cgroup paired with the root's quota would normalize the +// service's own CPU time against the whole machine's budget. +func withCgroupSelfCPUPaths(relative map[string]string, layouts []cgroupCPUPath) []cgroupCPUPath { + out := make([]cgroupCPUPath, 0, len(layouts)*2) + for _, layout := range layouts { + own := layout + own.usage = cgroupSelfFile(relative, layout.usage) + own.quota = cgroupSelfFile(relative, layout.quota) + if layout.period != "" { + own.period = cgroupSelfFile(relative, layout.period) + } + if own.usage != "" && own.quota != "" && (layout.period == "" || own.period != "") { + out = append(out, own) + } + out = append(out, layout) + } + return out +} + +// withCgroupSelfUsagePaths is withCgroupSelfPaths for the memory layouts, which +// name three files that have to move together: memoryStats picks the level whose +// *limit* binds and then reads usage from that same level, so a tuple whose +// limit and usage come from different cgroups measures one population against +// another's capacity. +// +// Every level from this process's own cgroup up to the mount root is emitted, +// because the binding limit is often an ancestor's — a slice with MemoryMax=, a +// pod cgroup shared with sidecars — while the leaf says "max". +func withCgroupSelfUsagePaths(relative map[string]string, layouts []cgroupUsagePath) []cgroupUsagePath { + out := make([]cgroupUsagePath, 0, len(layouts)*2) + seen := make(map[string]bool, len(layouts)*2) + add := func(level cgroupUsagePath) { + if level.limit == "" || seen[level.limit] { + return + } + seen[level.limit] = true + out = append(out, level) + } + for _, layout := range layouts { + if own := cgroupSelfFile(relative, layout.limit); own != "" { + usageName, statName := path.Base(layout.usage), path.Base(layout.stat) + for _, ancestor := range cgroupAncestorPaths(own) { + dir := path.Dir(ancestor) + add(cgroupUsagePath{ + limit: ancestor, + usage: path.Join(dir, usageName), + stat: path.Join(dir, statName), + inactiveFile: layout.inactiveFile, + }) + } + } + add(layout) + } + return out +} diff --git a/internal/nodemetrics/cgrouppath_test.go b/internal/nodemetrics/cgrouppath_test.go new file mode 100644 index 000000000..0a801c6d4 --- /dev/null +++ b/internal/nodemetrics/cgrouppath_test.go @@ -0,0 +1,504 @@ +package nodemetrics + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +// writeSelfCgroup lays down a /self/cgroup with the given body. +func writeSelfCgroup(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "self"), 0o755); err != nil { + t.Fatalf("create self dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "self", "cgroup"), []byte(body), 0o644); err != nil { + t.Fatalf("write self/cgroup: %v", err) + } + return dir +} + +func TestCgroupRelativePaths(t *testing.T) { + tests := []struct { + name string + body string + want map[string]string + }{ + { + // A systemd unit with CPUQuota= or MemoryMax=. Its limits live + // below the mount root, so the root files describe the machine. + name: "systemd unit on cgroup v2", + body: "0::/system.slice/silo.service\n", + want: map[string]string{"": "system.slice/silo.service"}, + }, + { + // v1 names its controllers, and the mount directory is named with + // the whole comma-joined field, so both forms have to resolve. + name: "cgroup v1 controllers", + body: "9:cpu,cpuacct:/system.slice/silo.service\n5:memory:/system.slice/silo.service\n", + want: map[string]string{ + "cpu,cpuacct": "system.slice/silo.service", + "cpu": "system.slice/silo.service", + "cpuacct": "system.slice/silo.service", + "memory": "system.slice/silo.service", + }, + }, + { + // A namespaced container is already at its own root, which is + // exactly what the unrewritten paths read. + name: "namespaced container", + body: "0::/\n", + want: nil, + }, + { + name: "unreadable lines are skipped", + body: "garbage\n0::/system.slice/silo.service\n", + want: map[string]string{"": "system.slice/silo.service"}, + }, + { + // The path field may contain colons; only the first two separators + // are structural. + name: "path containing a colon", + body: "0::/system.slice/silo:one.service\n", + want: map[string]string{"": "system.slice/silo:one.service"}, + }, + {name: "empty file", body: "", want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cgroupRelativePaths(writeSelfCgroup(t, tt.body)) + if len(got) != len(tt.want) { + t.Fatalf("cgroupRelativePaths() = %v, want %v", got, tt.want) + } + for key, value := range tt.want { + if got[key] != value { + t.Fatalf("cgroupRelativePaths()[%q] = %q, want %q", key, got[key], value) + } + } + }) + } +} + +// A missing file is the ordinary case on a host that is not Linux, or one whose +// /proc is not where we looked. It must read as "no rewrite", never as an error +// that costs the root reading too. +func TestCgroupRelativePathsWithoutTheFile(t *testing.T) { + if got := cgroupRelativePaths(t.TempDir()); got != nil { + t.Fatalf("cgroupRelativePaths() = %v, want none when /self/cgroup is absent", got) + } +} + +func TestCgroupSelfFile(t *testing.T) { + v2 := map[string]string{"": "system.slice/silo.service"} + v1 := map[string]string{"cpu,cpuacct": "system.slice/silo.service", "memory": "system.slice/silo.service"} + + tests := []struct { + name string + relative map[string]string + file string + want string + }{ + { + name: "v2 file sits at the mount root", relative: v2, + file: cgroupCPUPaths[0].quota, + want: "/sys/fs/cgroup/system.slice/silo.service/cpu.max", + }, + { + name: "v1 file sits under its controller", relative: v1, + file: "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage", + want: "/sys/fs/cgroup/cpu,cpuacct/system.slice/silo.service/cpuacct.usage", + }, + { + // v1 memory under v2 membership: no unified entry names it, so the + // root path stands rather than being rewritten into nonsense. + name: "controller this process has no membership for", relative: v2, + file: "/sys/fs/cgroup/memory/memory.stat", + want: "", + }, + {name: "no membership at all", relative: nil, file: cgroupCPUPaths[0].quota, want: ""}, + {name: "path outside the cgroup mount", relative: v2, file: "/proc/stat", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := cgroupSelfFile(tt.relative, tt.file); got != tt.want { + t.Fatalf("cgroupSelfFile() = %q, want %q", got, tt.want) + } + }) + } +} + +// The rewritten path goes first and the root stays behind it, so a container +// whose /proc names a host path it cannot open still falls through to the read +// that has always worked. +func TestWithCgroupSelfPathsKeepsTheRootFallback(t *testing.T) { + relative := map[string]string{"": "system.slice/silo.service", "memory": "system.slice/silo.service"} + got := withCgroupSelfPaths(relative, CgroupMemoryLimitPaths()) + // Every cgroup between this process and the root, then the root path the + // list started with. The intermediate levels are the point: a leaf that says + // "max" inside a slice that does not is exactly the case being covered. + want := []string{ + "/sys/fs/cgroup/system.slice/silo.service/memory.max", + "/sys/fs/cgroup/system.slice/memory.max", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/system.slice/silo.service/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/system.slice/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory.limit_in_bytes", + } + if !slices.Equal(got, want) { + t.Fatalf("withCgroupSelfPaths() =\n%v\nwant\n%v", got, want) + } + + // With no membership the list is exactly what it was. + if got := withCgroupSelfPaths(nil, CgroupMemoryLimitPaths()); !slices.Equal(got, CgroupMemoryLimitPaths()) { + t.Fatalf("withCgroupSelfPaths(nil) = %v, want the original list", got) + } +} + +// A CPU layout names several files that have to move together: this process's +// usage normalized against the root's quota would divide the service's own CPU +// time by the whole machine's budget. +func TestWithCgroupSelfCPUPathsRewritesEveryFileOrNone(t *testing.T) { + relative := map[string]string{"": "system.slice/silo.service"} + got := withCgroupSelfCPUPaths(relative, cgroupCPUPaths) + + if len(got) != len(cgroupCPUPaths)+1 { + t.Fatalf("got %d layouts, want only the v2 one rewritten alongside the originals", len(got)) + } + own := got[0] + if own.usage != "/sys/fs/cgroup/system.slice/silo.service/cpu.stat" { + t.Fatalf("usage = %q, want this process's own cpu.stat", own.usage) + } + if own.quota != "/sys/fs/cgroup/system.slice/silo.service/cpu.max" { + t.Fatalf("quota = %q, want this process's own cpu.max", own.quota) + } + if got[1].usage != cgroupCPUPaths[0].usage { + t.Fatalf("got[1].usage = %q, want the root layout kept behind it", got[1].usage) + } + // The v1 layouts have no unified membership to resolve, so they are carried + // through unrewritten rather than half-rewritten. + for _, layout := range got[1:] { + if layout.usage != "" && layout.quota == "" { + t.Fatalf("layout %+v has a usage file with no quota file", layout) + } + } +} + +func TestWithCgroupSelfUsagePathsRewritesEveryFileOrNone(t *testing.T) { + relative := map[string]string{"memory": "system.slice/silo.service"} + got := withCgroupSelfUsagePaths(relative, cgroupMemoryUsagePaths) + + // The v1 layout resolves, so it contributes one tuple per cgroup from this + // process up to the mount root, followed by the root layouts the list + // started with. The v2 layout has no "" membership here and is carried + // through unrewritten. + want := []cgroupUsagePath{ + { + limit: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/system.slice/silo.service/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: "/sys/fs/cgroup/memory/system.slice/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory/system.slice/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/system.slice/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: cgroupMemoryUsagePaths[1].limit, + usage: cgroupMemoryUsagePaths[1].usage, + stat: cgroupMemoryUsagePaths[1].stat, + inactiveFile: cgroupInactiveFileKeyV1, + }, + { + limit: "/sys/fs/cgroup/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, + } + var v1 []cgroupUsagePath + for _, level := range got { + if strings.Contains(level.limit, "limit_in_bytes") { + v1 = append(v1, level) + } + } + if !slices.Equal(v1, want) { + t.Fatalf("v1 levels =\n%+v\nwant\n%+v", v1, want) + } + + // Every emitted level names all three files from one cgroup: memoryStats + // picks by limit and then reads usage from the same tuple, so a mismatch + // measures one population against another's capacity. + for _, level := range got { + if level.limit == "" { + continue + } + dir := filepath.Dir(level.limit) + if filepath.Dir(level.usage) != dir || filepath.Dir(level.stat) != dir { + t.Fatalf("level %+v mixes cgroups; limit, usage and stat must share one directory", level) + } + } + + // With no membership the list is exactly what it was. + if got := withCgroupSelfUsagePaths(nil, cgroupMemoryUsagePaths); !slices.Equal(got, cgroupMemoryUsagePaths) { + t.Fatalf("withCgroupSelfUsagePaths(nil) = %+v, want the original list", got) + } +} + +// A limit is not always written where the process sits: a systemd unit inherits +// its quota from the slice containing it, and a container from its pod cgroup. +// The leaf reads "max" and the kernel throttles anyway, so a walk that stops at +// the leaf reports the whole host to a process that has two cores. +func TestCgroupAncestorPaths(t *testing.T) { + got := cgroupAncestorPaths("/sys/fs/cgroup/kubepods/burstable/podabc/container1/cpu.max") + want := []string{ + "/sys/fs/cgroup/kubepods/burstable/podabc/container1/cpu.max", + "/sys/fs/cgroup/kubepods/burstable/podabc/cpu.max", + "/sys/fs/cgroup/kubepods/burstable/cpu.max", + "/sys/fs/cgroup/kubepods/cpu.max", + cgroupCPUPaths[0].quota, + } + if !slices.Equal(got, want) { + t.Fatalf("cgroupAncestorPaths() =\n%v\nwant\n%v", got, want) + } + + // The mount root itself has nowhere to climb to. + if got := cgroupAncestorPaths(cgroupCPUPaths[0].quota); !slices.Equal(got, []string{cgroupCPUPaths[0].quota}) { + t.Fatalf("cgroupAncestorPaths(root) = %v, want just the file", got) + } + // A path outside the hierarchy still reads itself: a test harness pointing + // these at a temp directory has no ancestors, and must not lose its file. + if got := cgroupAncestorPaths("/tmp/fake/cpu.max"); !slices.Equal(got, []string{"/tmp/fake/cpu.max"}) { + t.Fatalf("cgroupAncestorPaths(outside) = %v, want just the file", got) + } + if got := cgroupAncestorPaths(""); got != nil { + t.Fatalf("cgroupAncestorPaths(\"\") = %v, want none", got) + } +} + +// The quota a process is throttled against is the tightest anywhere above it, +// and it has to be paired with the period from the same cgroup — a quota from +// one level over a period from another describes no real budget. +func TestEffectiveCgroupCPUQuotaTakesTheTightestAncestor(t *testing.T) { + root := t.TempDir() + leaf := filepath.Join(root, "system.slice", "silo.service") + slice := filepath.Join(root, "system.slice") + for _, dir := range []string{leaf, slice} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + + // cgroupAncestorPaths only climbs inside the real cgroup mount, so the walk + // is exercised here by handing effectiveCgroupCPUQuota each level directly. + quotaAt := func(dir string) float64 { + _, cores := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(dir, "cpu.max")}) + return cores + } + + // The service says "max" while its slice allows two cores. + write(leaf, "cpu.max", "max 100000\n") + write(slice, "cpu.max", "200000 100000\n") + if got := quotaAt(leaf); got != 0 { + t.Fatalf("leaf alone = %v cores, want 0 — it imposes none", got) + } + if got := quotaAt(slice); got != 2 { + t.Fatalf("slice = %v cores, want 2", got) + } + + // A leaf tighter than its slice wins, and a looser one loses. + write(leaf, "cpu.max", "100000 100000\n") + if _, got := effectiveCgroupCPUQuota(cgroupCPUPath{quota: filepath.Join(leaf, "cpu.max")}); got != 1 { + t.Fatalf("tighter leaf = %v cores, want 1", got) + } +} + +// v1 keeps quota and period in separate files, so both have to move together as +// the walk climbs. +func TestEffectiveCgroupCPUQuotaPairsQuotaWithItsOwnPeriod(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "cpu.cfs_quota_us"), []byte("400000\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cpu.cfs_period_us"), []byte("100000\n"), 0o644); err != nil { + t.Fatal(err) + } + _, got := effectiveCgroupCPUQuota(cgroupCPUPath{ + quota: filepath.Join(dir, "cpu.cfs_quota_us"), + period: filepath.Join(dir, "cpu.cfs_period_us"), + }) + if got != 4 { + t.Fatalf("v1 quota = %v cores, want 4", got) + } +} + +// The quota and the usage measured against it have to come from the same +// cgroup. A quota on a shared ancestor is spent by every service under it, so +// this process's own CPU time over that quota describes nothing: Silo at ten +// percent beside a sibling at ninety would report ten, while the group is +// saturated and being throttled. +func TestEffectiveCgroupCPUQuotaReturnsTheLevelThatBinds(t *testing.T) { + root := t.TempDir() + // The walk only climbs inside the cgroup mount, so the fixture becomes one. + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + // The service imposes nothing; the slice above it allows two cores and + // carries the usage of everything under it. + write(leaf, "cpu.max", "max 100000\n") + write(leaf, "cpu.stat", "usage_usec 1000000\n") + write(root, "cpu.max", "200000 100000\n") + write(root, "cpu.stat", "usage_usec 9000000\n") + + binding, cores := effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 2 { + t.Fatalf("cores = %v, want the slice's 2", cores) + } + if want := filepath.Join(root, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the slice's own %q", binding.usage, want) + } + + // Nothing above it binds: the level stays the leaf's, so an unconstrained + // or leaf-limited process still measures itself. + write(root, "cpu.max", "max 100000\n") + write(leaf, "cpu.max", "100000 100000\n") + binding, cores = effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 1 { + t.Fatalf("cores = %v, want the leaf's 1", cores) + } + if want := filepath.Join(leaf, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the leaf's own %q", binding.usage, want) + } +} + +// A quota on a shared ancestor and a tighter cpuset on the leaf are caps on +// different populations. Taking the ancestor's usage — which counts every +// sibling under that quota — and dividing it by this process's smaller private +// cpuset pins the node at a hundred percent while Silo is idle. +func TestCgroupCPUMovesUsageDownWhenTheCpusetBinds(t *testing.T) { + root := t.TempDir() + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + // The slice allows eight cores and has burned a lot of CPU across every + // service under it; this one has burned very little and may use two CPUs. + write(root, "cpu.max", "800000 100000\n") + write(root, "cpu.stat", "usage_usec 9000000\n") + write(leaf, "cpu.max", "max 100000\n") + write(leaf, "cpu.stat", "usage_usec 1000000\n") + + s := &Sampler{cgroupCPUPaths: []cgroupCPUPath{{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: filepath.Join(leaf, "cpu.max"), + }}} + + // No cpuset: the slice's quota binds, so its usage is the right numerator. + sample, quota := s.cgroupCPU(time.Now(), 0) + if quota != 8 { + t.Fatalf("quota = %v, want the slice's 8 cores", quota) + } + if want := int64(9_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want the slice's %d", sample.usageNS, want) + } + + // A two-CPU cpuset is tighter, and it applies to this cgroup — so the + // measurement moves back down with it. + sample, quota = s.cgroupCPU(time.Now(), 2) + if quota != 2 { + t.Fatalf("quota = %v, want the cpuset's 2", quota) + } + if want := int64(1_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want this cgroup's own %d, not the slice's", sample.usageNS, want) + } + + // A cpuset looser than the quota changes nothing. + sample, quota = s.cgroupCPU(time.Now(), 32) + if quota != 8 { + t.Fatalf("quota = %v, want the slice's 8 to still bind", quota) + } + if want := int64(9_000_000) * int64(time.Microsecond); sample.usageNS != want { + t.Fatalf("usage = %d, want the slice's %d", sample.usageNS, want) + } +} + +// Two cgroups publishing the same quota are not equivalent. The ancestor's is +// shared with siblings that can exhaust it, so it is the level whose usage +// describes what is actually being throttled: Silo at 0.2 cores beside a +// sibling at 1.8 under a shared two-core parent reads ten percent from the leaf +// while the parent is saturated. +func TestEffectiveCgroupCPUQuotaPrefersTheOuterLevelOnATie(t *testing.T) { + root := t.TempDir() + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(dir, name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatalf("write %s/%s: %v", dir, name, err) + } + } + write(leaf, "cpu.max", "200000 100000\n") + write(leaf, "cpu.stat", "usage_usec 200000\n") + write(root, "cpu.max", "200000 100000\n") + write(root, "cpu.stat", "usage_usec 2000000\n") + + binding, cores := effectiveCgroupCPUQuota(cgroupCPUPath{ + usage: filepath.Join(leaf, "cpu.stat"), + usageKey: cgroupCPUUsageKey, + usageUnit: 1, + quota: filepath.Join(leaf, "cpu.max"), + }) + if cores != 2 { + t.Fatalf("cores = %v, want 2 from either level", cores) + } + if want := filepath.Join(root, "cpu.stat"); binding.usage != want { + t.Fatalf("binding usage = %q, want the shared parent's %q", binding.usage, want) + } +} diff --git a/internal/nodemetrics/collector.go b/internal/nodemetrics/collector.go new file mode 100644 index 000000000..3a3a5f682 --- /dev/null +++ b/internal/nodemetrics/collector.go @@ -0,0 +1,207 @@ +package nodemetrics + +import ( + "log/slog" + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +// Prometheus exposure. +// +// The collector is built over the published snapshot rather than over +// promauto gauges written during sampling. A scrape therefore reports exactly +// the numbers the health response and the admin API report for the same +// instant, and — more importantly — a scrape can never wait on sampling, so a +// wedged mount or a hung nvidia-smi cannot turn into a Prometheus scrape +// timeout that looks like the node being down. +// +// Names carry the streamapp_ prefix the rest of the server already uses, and +// the node_ infix so a scrape of an integrated deployment separates host +// resources from request and domain metrics. +var ( + descCPUPercent = prometheus.NewDesc( + "streamapp_node_cpu_percent", + "Aggregate CPU busy percentage across all cores over the last sampling interval.", + nil, nil) + descLoad1 = prometheus.NewDesc( + "streamapp_node_load1", + "1-minute load average.", + nil, nil) + descMemoryUsed = prometheus.NewDesc( + "streamapp_node_memory_used_bytes", + "Memory in use, corrected by the cgroup limit and usage when running under one.", + nil, nil) + descMemoryTotal = prometheus.NewDesc( + "streamapp_node_memory_total_bytes", + "Memory available to this process's cgroup, or the host's when unconstrained.", + nil, nil) + descDiskUsed = prometheus.NewDesc( + "streamapp_node_disk_used_bytes", + "Used bytes on a sampled mount, labeled by role rather than by path.", + []string{diskMountLabel}, nil) + descDiskTotal = prometheus.NewDesc( + "streamapp_node_disk_total_bytes", + "Total bytes on a sampled mount, labeled by role rather than by path.", + []string{diskMountLabel}, nil) + descDiskStale = prometheus.NewDesc( + "streamapp_node_disk_stale", + "1 when a mount's used/total bytes are carried over from an earlier pass because its probe has not returned, 0 when they are current.", + []string{diskMountLabel}, nil) + descNetworkRx = prometheus.NewDesc( + "streamapp_node_network_rx_bps", + "Aggregate received bits per second, loopback excluded.", + nil, nil) + descNetworkTx = prometheus.NewDesc( + "streamapp_node_network_tx_bps", + "Aggregate transmitted bits per second, loopback excluded.", + nil, nil) + descGPUVideoBusy = prometheus.NewDesc( + "streamapp_node_gpu_video_busy_percent", + "GPU video engine busy percentage. From DRM fdinfo this covers only this node's own transcodes.", + []string{gpuDeviceLabel}, nil) + descGPURenderBusy = prometheus.NewDesc( + "streamapp_node_gpu_render_busy_percent", + "GPU render engine busy percentage. From DRM fdinfo this covers only this node's own transcodes.", + []string{gpuDeviceLabel}, nil) + descGPUTotalBusy = prometheus.NewDesc( + "streamapp_node_gpu_busy_percent", + "Whole-GPU utilization including workloads from other tenants, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) + descGPUSessions = prometheus.NewDesc( + "streamapp_node_gpu_sessions", + "Active GPU workloads this node has pinned to a device.", + []string{gpuDeviceLabel}, nil) + descGPUVRAMUsed = prometheus.NewDesc( + "streamapp_node_gpu_vram_used_bytes", + "GPU memory in use, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) + descGPUVRAMTotal = prometheus.NewDesc( + "streamapp_node_gpu_vram_total_bytes", + "Total GPU memory, where an enrichment source reports it.", + []string{gpuDeviceLabel}, nil) +) + +// gpuDeviceLabel is the Prometheus label name every per-GPU series is keyed by, +// and diskMountLabel the one every per-mount series is keyed by. Its value is +// DiskStats.Role, not a path; see the comment on the disk descriptors. +const ( + gpuDeviceLabel = "device" + diskMountLabel = "mount" +) + +// collector adapts a Sampler to prometheus.Collector. +type collector struct{ sampler *Sampler } + +// Describe is deliberately unimplemented (an unchecked collector): the disk and +// GPU label sets are discovered at sample time and legitimately change when a +// mount or a card appears, which a checked collector would reject. +func (collector) Describe(chan<- *prometheus.Desc) {} + +// Collect reads the latest snapshot. It performs no I/O and takes no lock the +// sampler holds while doing I/O, so it cannot block a scrape. +func (c collector) Collect(ch chan<- prometheus.Metric) { + snapshot := c.sampler.Snapshot() + if !snapshot.Available { + return + } + gauge := func(desc *prometheus.Desc, value float64, labels ...string) { + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labels...) + } + if system := snapshot.System; system != nil { + gauge(descCPUPercent, float64(system.CPUPct)) + gauge(descLoad1, system.Load1) + gauge(descMemoryUsed, float64(system.MemUsedMB)*float64(bytesPerMB)) + gauge(descMemoryTotal, float64(system.MemTotalMB)*float64(bytesPerMB)) + gauge(descNetworkRx, float64(system.NetRxBps)) + gauge(descNetworkTx, float64(system.NetTxBps)) + const bytesPerGB = float64(1024 * 1024 * 1024) + for _, disk := range system.Disks { + if disk.Unavailable || disk.Role == "" { + // A path this node cannot measure has no value to report, and a + // zero would read as an empty disk in every alert rule. + continue + } + // A stale mount keeps exporting, with a series that says so. + // + // Unlike an unavailable one it has a real measurement, only an old + // one — and `stale` is set as soon as a probe outlives its five + // second budget, which a network mount does routinely without + // anything being wrong. Dropping the series there would blank a + // dashboard for a disk that is fine and merely slow to answer. + // Re-exporting old numbers under a fresh scrape timestamp with + // nothing to qualify them is the other half of the problem, though: + // a fill alert would sit green forever on a mount that stopped + // answering at 40% and has been filling since. So the values ship + // with their staleness beside them, the same pairing the JSON + // surfaces carry, and an alert can read both. + stale := 0.0 + if disk.Stale { + stale = 1 + } + gauge(descDiskUsed, disk.UsedGB*bytesPerGB, disk.Role) + gauge(descDiskTotal, disk.TotalGB*bytesPerGB, disk.Role) + gauge(descDiskStale, stale, disk.Role) + } + } + const bytesPerMBFloat = float64(1024 * 1024) + for _, gpu := range snapshot.GPU { + // Every measurement is exported only when the snapshot actually holds + // one. The JSON surfaces carry `source` alongside and can render the + // difference between unmeasured and idle; a Prometheus sample cannot, so + // an exported 0 would read as an idle GPU on every dashboard and alert — + // including for a card that is busy and merely unobservable. An absent + // series is the honest shape for a number that was not taken. + if gpu.VideoBusyPct != nil { + gauge(descGPUVideoBusy, float64(*gpu.VideoBusyPct), gpu.Device) + } + if gpu.RenderBusyPct != nil { + gauge(descGPURenderBusy, float64(*gpu.RenderBusyPct), gpu.Device) + } + // The engine readings above describe this node's own work when they come + // from fdinfo; this one describes the card. A shared GPU saturated by + // another tenant is the case where they disagree, and it is the one an + // operator most needs to alert on — a transcode that will not get the + // silicon it was planned onto. + if gpu.TotalBusyPct != nil { + gauge(descGPUTotalBusy, float64(*gpu.TotalBusyPct), gpu.Device) + } + // Sessions always ships: it comes from this process's own workload + // accounting, not from a driver, so it is exact whatever the driver can + // or cannot tell us — and a busy GPU with no engine reading is precisely + // when an operator needs it. + gauge(descGPUSessions, float64(gpu.Sessions), gpu.Device) + if gpu.VRAMUsedMB != nil { + gauge(descGPUVRAMUsed, float64(*gpu.VRAMUsedMB)*bytesPerMBFloat, gpu.Device) + } + if gpu.VRAMTotalMB != nil { + gauge(descGPUVRAMTotal, float64(*gpu.VRAMTotalMB)*bytesPerMBFloat, gpu.Device) + } + } +} + +// Disk series are labeled by DiskStats.Role rather than by path. +// +// /metrics is deliberately unauthenticated on the same listener that serves the +// API and the SPA, so anything labeled here is public. A library root's path is +// deployment layout, not a host resource counter: publishing it would let any +// anonymous client enumerate a deployment's media mounts, which is precisely +// what the admin-authenticated /admin/system/resources exists to gate. Roles +// keep the series useful — scratch is the volume that kills transcodes when it +// fills, and library ordering is stable for a given configuration — while the +// paths themselves stay behind auth. The role is assigned once when the sample +// is built, so this scrape and the node's /health name a mount identically. + +// collectorRegistration keeps the default registry to one node collector. +// Integrated mode constructs one sampler, but a test — or a future deployment +// that ran two — must not panic the process on a duplicate registration. +var collectorRegistration sync.Once + +// registerCollector publishes a sampler's readings on the default registry. +func registerCollector(sampler *Sampler) { + collectorRegistration.Do(func() { + if err := prometheus.Register(collector{sampler: sampler}); err != nil { + slog.Warn("node metrics collector not registered", "component", "nodemetrics", "error", err) + } + }) +} diff --git a/internal/nodemetrics/collector_test.go b/internal/nodemetrics/collector_test.go new file mode 100644 index 000000000..869285915 --- /dev/null +++ b/internal/nodemetrics/collector_test.go @@ -0,0 +1,380 @@ +package nodemetrics + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +// gatherNames collects one scrape's metric names and the first sample value per +// name, which is enough to assert the exposition without asserting on the +// registry's formatting. +func gatherNames(t *testing.T, sampler *Sampler) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + values := make(map[string]float64, len(families)) + for _, family := range families { + for _, metric := range family.GetMetric() { + if _, seen := values[family.GetName()]; seen { + continue + } + values[family.GetName()] = metric.GetGauge().GetValue() + } + } + return values +} + +func TestCollectorExposesSnapshot(t *testing.T) { + f := newDiskFixture(t, "/transcode") + f.answer("/transcode", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + s := f.sampler + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + + values := gatherNames(t, s) + for _, name := range []string{ + "streamapp_node_cpu_percent", + "streamapp_node_load1", + "streamapp_node_memory_used_bytes", + "streamapp_node_memory_total_bytes", + "streamapp_node_disk_used_bytes", + "streamapp_node_disk_total_bytes", + "streamapp_node_network_rx_bps", + "streamapp_node_network_tx_bps", + "streamapp_node_gpu_video_busy_percent", + "streamapp_node_gpu_busy_percent", + "streamapp_node_gpu_sessions", + "streamapp_node_gpu_vram_used_bytes", + "streamapp_node_gpu_vram_total_bytes", + } { + if _, ok := values[name]; !ok { + t.Fatalf("%s missing from the scrape (got %v)", name, values) + } + } + // nvidia-smi reports encoder and decoder but nothing for the render engine, + // and this card has no DRM counters to supply one. Exporting the missing + // column as 0 would draw an idle 3D engine for a GPU nobody measured. + if _, ok := values["streamapp_node_gpu_render_busy_percent"]; ok { + t.Fatalf("render busy exported for an nvidia-smi-only card: %v", values) + } + + // Bytes on the wire, gibibytes in the JSON: the conversion has to survive. + if got := values["streamapp_node_disk_used_bytes"]; got != float64(100<<30) { + t.Fatalf("disk used = %v, want %v bytes", got, float64(100<<30)) + } + if got := values["streamapp_node_gpu_sessions"]; got != 2 { + t.Fatalf("gpu sessions = %v, want 2", got) + } +} + +// /metrics is unauthenticated on the same listener that serves the API and the +// SPA. Labeling disk series by path would let anyone who can reach the server +// enumerate its media mounts — the layout the admin-authenticated resources +// endpoint exists to gate. +func TestCollectorDoesNotLabelDisksByPath(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/nas/movies", "/srv/private/kids-shows") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.answer("/mnt/nas/movies", fsStats{UsedBytes: 2 << 30, TotalBytes: 20 << 30, FSID: "b:1"}) + f.answer("/srv/private/kids-shows", fsStats{UsedBytes: 3 << 30, TotalBytes: 30 << 30, FSID: "c:1"}) + + f.sampleAndSettle(t, 3) + f.sampleAndSettle(t, 3) + + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: f.sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + + labels := map[string]bool{} + for _, family := range families { + if family.GetName() != "streamapp_node_disk_used_bytes" { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if strings.Contains(label.GetValue(), "/") { + t.Fatalf("%s carries a filesystem path in label %s=%q", + family.GetName(), label.GetName(), label.GetValue()) + } + labels[label.GetValue()] = true + } + } + } + for _, want := range []string{"scratch", "library-1", "library-2"} { + if !labels[want] { + t.Fatalf("mount label %q missing from %v", want, labels) + } + } +} + +// A host that cannot be sampled must publish nothing rather than a wall of +// zeros that alert rules would read as a healthy idle machine. +func TestCollectorExposesNothingWhenUnavailable(t *testing.T) { + tree := newProcTree(t) + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.goos = "darwin" + s.sample(context.Background()) + + if values := gatherNames(t, s); len(values) != 0 { + t.Fatalf("scrape returned %v on an unsampled host, want nothing", values) + } +} + +// A scrape must never wait on sampling. If Collect took a lock the disk prober +// holds, a wedged mount would turn into a scrape timeout that looks like the +// node being down — which is the exact failure this package exists to avoid. +func TestCollectorDoesNotBlockOnAWedgedMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/nfs") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.wedge(t, "/nfs") + + f.sampler.sample(context.Background()) + f.awaitProbes(t, 1) // only /transcode can finish; /nfs is parked + + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: f.sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + type result struct { + families []*dto.MetricFamily + err error + } + scraped := make(chan result, 1) + go func() { + families, err := registry.Gather() + scraped <- result{families: families, err: err} + }() + select { + case got := <-scraped: + if got.err != nil { + t.Fatalf("gather: %v", got.err) + } + if len(got.families) == 0 { + t.Fatal("scrape returned no metrics while a mount was wedged") + } + case <-time.After(5 * time.Second): + t.Fatal("scrape blocked behind a wedged mount") + } +} + +// Registration is guarded so a second sampler cannot panic the process on a +// duplicate collector. +func TestRegisterCollectorIsIdempotent(t *testing.T) { + tree := newProcTree(t) + first := newTestSampler(t, tree, newFakeClock(), Options{}) + second := newTestSampler(t, tree, newFakeClock(), Options{}) + registerCollector(first) + registerCollector(second) +} + +// The positional library label is a promise about which volume a series +// describes, and Prometheus has no way to signal that the promise moved: a +// mount going unavailable used to renumber every library after it, so an alert +// rule keyed on mount="library-1" silently followed a different disk with no +// gap in the series to show it happened. +func TestCollectorKeepsLibraryLabelsPositional(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/nas/movies", "/mnt/nas/shows") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + // /mnt/nas/movies is never answered, so it stays unavailable — the media + // root that lives on another node, or the export whose server went away. + f.answer("/mnt/nas/shows", fsStats{UsedBytes: 3 << 30, TotalBytes: 30 << 30, FSID: "c:1"}) + + f.sampleAndSettle(t, 3) + f.sampleAndSettle(t, 3) + + used := diskSeriesByLabel(t, f.sampler) + if _, reported := used["library-1"]; reported { + t.Fatalf("unavailable mount emitted a series: %v", used) + } + // The measurable root keeps the index its position earns, rather than + // sliding into the missing one's label. + if got, ok := used["library-2"]; !ok || got != float64(3<<30) { + t.Fatalf("library-2 = %v (present=%t), want the second library root's %v bytes", + got, ok, float64(3<<30)) + } +} + +// diskSeriesByLabel collects streamapp_node_disk_used_bytes keyed by its mount +// label. +func diskSeriesByLabel(t *testing.T, sampler *Sampler) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + byLabel := map[string]float64{} + for _, family := range families { + if family.GetName() != "streamapp_node_disk_used_bytes" { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == gpuDeviceLabel { + continue + } + byLabel[label.GetValue()] = metric.GetGauge().GetValue() + } + } + } + return byLabel +} + +// A GPU nothing could measure has zeros that mean "not taken", not "idle". The +// JSON surfaces carry `source` alongside and render the difference; a Prometheus +// sample cannot, so an exported 0 would show a busy-but-unobservable card as +// idle on every dashboard. The session count still ships: it comes from this +// process's own accounting rather than a driver. +func TestCollectorOmitsUnmeasuredGPUEngineGauges(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "cuda:0", Vendor: vendorNVIDIA, Sessions: 2, Source: SourceUnavailable, + }}, + }) + + values := gatherNames(t, sampler) + if _, present := values["streamapp_node_gpu_video_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a video busy percentage") + } + if _, present := values["streamapp_node_gpu_render_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a render busy percentage") + } + if _, present := values["streamapp_node_gpu_busy_percent"]; present { + t.Fatal("an unmeasured GPU exported a whole-GPU utilization") + } + if got := values["streamapp_node_gpu_sessions"]; got != 2 { + t.Fatalf("gpu sessions = %v, want the workload count exported regardless", got) + } +} + +// Whole-GPU utilization is what nvidia-smi can see and fdinfo cannot: the card's +// own busyness, other tenants included. Without it an operator watching only the +// engine gauges sees this node's idle transcoder and no sign that the card it is +// planned onto is saturated by someone else. +func TestCollectorExportsWholeGPUUtilization(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "cuda:0", Vendor: vendorNVIDIA, Sessions: 0, + VideoBusyPct: ptr(3), TotalBusyPct: ptr(94), Source: SourceNVIDIASMI, + }}, + }) + + values := gatherNames(t, sampler) + if got := values["streamapp_node_gpu_busy_percent"]; got != 94 { + t.Fatalf("whole-GPU busy = %v, want the card's 94 rather than this node's 3", got) + } +} + +// A measured source exports both engines, including a genuine zero — which does +// mean idle. +func TestCollectorExportsMeasuredGPUEngineGauges(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{}, + GPU: []GPUStats{{ + Device: "/dev/dri/renderD128", Sessions: 0, + VideoBusyPct: ptr(0), RenderBusyPct: ptr(0), Source: SourceFdinfo, + }}, + }) + + values := gatherNames(t, sampler) + if _, present := values["streamapp_node_gpu_video_busy_percent"]; !present { + t.Fatal("a measured idle GPU omitted its video busy percentage") + } + if _, present := values["streamapp_node_gpu_render_busy_percent"]; !present { + t.Fatal("a measured idle GPU omitted its render busy percentage") + } +} + +// gatherLabeled reads one metric family keyed by its single label value, so a +// test can assert per-mount or per-device rather than on whichever series the +// registry happened to order first. +func gatherLabeled(t *testing.T, sampler *Sampler, name string) map[string]float64 { + t.Helper() + registry := prometheus.NewRegistry() + if err := registry.Register(collector{sampler: sampler}); err != nil { + t.Fatalf("register collector: %v", err) + } + families, err := registry.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + values := map[string]float64{} + for _, family := range families { + if family.GetName() != name { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + values[label.GetValue()] = metric.GetGauge().GetValue() + } + } + } + return values +} + +// A mount whose probe has not come back keeps exporting its last real numbers — +// dropping them would blank a dashboard for a network mount that is merely slow, +// which sets Stale routinely — but a scrape carries no `stale` field the way the +// JSON surfaces do. Without a series saying so, a fill alert sits green forever +// on a volume that stopped answering at 40% and has been filling since. +func TestCollectorMarksStaleDiskReadings(t *testing.T) { + sampler := NewFixedSamplerForTest(Snapshot{ + Available: true, + System: &SystemStats{Disks: []DiskStats{ + {Role: ScratchDiskRole, UsedGB: 100, TotalGB: 500}, + {Role: "library-1", UsedGB: 400, TotalGB: 500, Stale: true}, + {Role: "library-2", Unavailable: true}, + }}, + }) + + stale := gatherLabeled(t, sampler, "streamapp_node_disk_stale") + if got, ok := stale[ScratchDiskRole]; !ok || got != 0 { + t.Fatalf("scratch stale = %v (present=%v), want a measured 0", got, ok) + } + if got, ok := stale["library-1"]; !ok || got != 1 { + t.Fatalf("library-1 stale = %v (present=%v), want 1", got, ok) + } + // An unavailable mount has no measurement at all, so it exports nothing — + // including no staleness, which would imply there were numbers to qualify. + if _, ok := stale["library-2"]; ok { + t.Fatalf("unavailable mount exported a staleness series: %v", stale) + } + + // The values themselves still ship for the stale mount: they are real, only + // old, and an operator needs to see a volume that was nearly full. + used := gatherLabeled(t, sampler, "streamapp_node_disk_used_bytes") + if got, ok := used["library-1"]; !ok || got != 400*float64(1024*1024*1024) { + t.Fatalf("library-1 used = %v (present=%v), want the carried-over reading kept", got, ok) + } + if _, ok := used["library-2"]; ok { + t.Fatalf("unavailable mount exported a used-bytes series: %v", used) + } +} diff --git a/internal/nodemetrics/disk.go b/internal/nodemetrics/disk.go new file mode 100644 index 000000000..de52427bf --- /dev/null +++ b/internal/nodemetrics/disk.go @@ -0,0 +1,331 @@ +package nodemetrics + +import ( + "log/slog" + "strconv" + "time" +) + +// maxSampledDisks caps how many mounts a snapshot reports. A deployment can +// have dozens of library roots, and a health response that grows with the +// library count would eventually be the reason health requests are slow. +const maxSampledDisks = 8 + +// diskProbeTimeout is how long a probe may be outstanding before the entry it +// belongs to is reported stale. It is not a cancellation: statfs(2) on a dead +// NFS server is uninterruptible, so the goroutine stays parked until the mount +// recovers or the process exits. What the timeout bounds is how long a reader +// is told numbers are current. +const diskProbeTimeout = 5 * time.Second + +// fsStats is one filesystem's capacity, in the portable shape this package +// needs from statfs(2). +type fsStats struct { + UsedBytes uint64 + TotalBytes uint64 + // FSID identifies the filesystem itself, so two paths on one volume — the + // common case where scratch and media live on the same disk — are reported + // once instead of twice with identical numbers. It is empty when the + // filesystem publishes no usable id (FUSE mounts do not), in which case each + // path is reported separately rather than collapsed onto a shared non-id. + FSID string +} + +// fsCapacity converts raw statfs(2) block counts into the used/total shape this +// package reports. It is separate from the syscall so the arithmetic — which is +// where the reserved-block subtlety lives — can be exercised directly. +// +// A filesystem can only report more free blocks than it holds if the numbers +// are nonsense, so the subtraction is guarded rather than allowed to wrap an +// unsigned counter into a petabyte. +func fsCapacity(blocks, free, available, blockSize uint64) fsStats { + if free > blocks { + free = blocks + } + used := (blocks - free) * blockSize + return fsStats{UsedBytes: used, TotalBytes: used + available*blockSize} +} + +// diskEntry is one path's probe state. Probes run detached from the sample +// loop, so this holds the last good answer for readers to fall back to. +type diskEntry struct { + path string + // inFlight is what keeps a permanently stuck mount from accumulating one + // parked goroutine per sample. A path already being probed is skipped + // entirely; there is never more than one goroutine per path. + inFlight bool + startedAt time.Time + haveGood bool + good fsStats + goodAt time.Time + lastErr bool + unreachable bool +} + +// stale reports whether this entry's last good numbers should be flagged as +// carried over. Either the current probe has outlived its budget — the wedged +// network mount case — or no probe has landed for longer than one full sampling +// cycle plus that budget. +func (e *diskEntry) stale(now time.Time, interval time.Duration) bool { + if e.lastErr { + return true + } + if e.inFlight && now.Sub(e.startedAt) > diskProbeTimeout { + return true + } + return now.Sub(e.goodAt) > interval+diskProbeTimeout +} + +// maxOutstandingDiskProbes is the ceiling on statfs goroutines this sampler may +// have parked at once, across every path it has ever been asked about. +// +// Bounding the paths offered per sample is not enough on its own. A probe stuck +// on a dead mount is kept — dropping its entry would only let the next sample +// start a second goroutine against the same mount — so a deployment whose +// library roots churn while mounts are wedged would retire one set of parked +// goroutines' paths and immediately be free to park a fresh set for the +// replacements. Repeat that and the count grows without limit. This ceiling is +// what makes it a fixed cost instead: once it is reached no new probe starts, +// which is the correct backpressure, since a sampler with this many mounts +// wedged has nothing useful left to measure anyway. +const maxOutstandingDiskProbes = maxSampledDisks + +// refreshDisks starts a probe for every path that is not already being probed +// and returns immediately. It never waits for a result: the caller is the +// sample loop, and the whole point of this package is that one bad mount cannot +// delay a node's health answer. +func (s *Sampler) refreshDisks(paths []string, now time.Time) { + s.diskMu.Lock() + defer s.diskMu.Unlock() + + wanted := make(map[string]bool, len(paths)) + for _, path := range paths { + if path != "" { + wanted[path] = true + } + } + s.pruneDisksLocked(wanted) + + seen := make(map[string]bool, len(paths)) + candidates := make([]*diskEntry, 0, len(paths)) + for _, path := range paths { + if path == "" || seen[path] { + continue + } + seen[path] = true + entry := s.disks[path] + if entry == nil { + entry = &diskEntry{path: path} + s.disks[path] = entry + // Reporting order is the order the caller offered, whatever order + // the probes below are started in. + s.diskOrder = append(s.diskOrder, path) + } + candidates = append(candidates, entry) + } + + for _, entry := range s.probeOrderLocked(candidates) { + if entry.inFlight { + continue + } + if s.probesInFlight >= maxOutstandingDiskProbes { + s.noteProbeBudgetExhaustedLocked() + break + } + entry.inFlight = true + entry.startedAt = now + s.probesInFlight++ + go s.probeDisk(entry) + } +} + +// probeOrderLocked is the order this sample offers entries to the probe budget. +// +// Scratch stays first: it is the mount transcode admission reads, so it is the +// one that must get a freed slot. The rest rotate, and that is not fairness for +// its own sake. The budget is a global ceiling, and a probe parked on a wedged +// mount holds its slot until the mount recovers or the process exits — so a +// deployment with one dead mount permanently runs one slot short. Offering the +// same list in the same order every sample would then spend the whole remaining +// budget on the same prefix and never reach the last path at all: it would be +// reported unavailable indefinitely, which is a lie about a disk that is fine +// and would have answered instantly. Advancing the start by one each sample +// costs a path its refresh roughly once per cycle and guarantees every mount is +// measured. +// +// Callers must hold diskMu. +func (s *Sampler) probeOrderLocked(candidates []*diskEntry) []*diskEntry { + rotatable := candidates + ordered := make([]*diskEntry, 0, len(candidates)) + if s.scratchDir != "" && len(candidates) > 0 && candidates[0].path == s.scratchDir { + ordered = append(ordered, candidates[0]) + rotatable = candidates[1:] + } + if len(rotatable) == 0 { + return ordered + } + start := s.diskProbeCursor % len(rotatable) + s.diskProbeCursor = (start + 1) % len(rotatable) + for i := range rotatable { + ordered = append(ordered, rotatable[(start+i)%len(rotatable)]) + } + return ordered +} + +// noteProbeBudgetExhaustedLocked logs the first sample in which the probe +// ceiling stopped new work, and nothing further until it clears. Every entry it +// skips keeps reporting its last good numbers marked stale, so without a line +// here the state reads as mounts that merely went quiet. +// Callers must hold diskMu. +func (s *Sampler) noteProbeBudgetExhaustedLocked() { + if s.probeBudgetExhausted { + return + } + s.probeBudgetExhausted = true + slog.Warn("node metrics disk probes are at their ceiling; mounts are not being re-measured", + "component", "nodemetrics", "outstanding", s.probesInFlight, "limit", maxOutstandingDiskProbes) +} + +// pruneDisksLocked forgets paths no longer offered, so a server whose libraries +// churn over months does not accumulate an entry per path ever configured. +// An entry with a probe still parked is kept: dropping it would let the next +// sample start a second goroutine against the same wedged mount, which is +// exactly what the in-flight guard exists to prevent. +// Callers must hold diskMu. +func (s *Sampler) pruneDisksLocked(wanted map[string]bool) { + kept := s.diskOrder[:0] + for _, path := range s.diskOrder { + entry := s.disks[path] + if wanted[path] || (entry != nil && entry.inFlight) { + kept = append(kept, path) + continue + } + delete(s.disks, path) + } + s.diskOrder = kept +} + +// probeDisk runs one statfs and records the outcome. It runs on its own +// goroutine and may never return; that is expected and is why nothing waits on +// it. +func (s *Sampler) probeDisk(entry *diskEntry) { + stats, err := s.statfs(entry.path) + + s.diskMu.Lock() + entry.inFlight = false + if s.probesInFlight > 0 { + s.probesInFlight-- + } + if s.probeBudgetExhausted && s.probesInFlight < maxOutstandingDiskProbes { + s.probeBudgetExhausted = false + slog.Info("node metrics disk probes are below their ceiling again", "component", "nodemetrics") + } + if err != nil { + entry.lastErr = true + // A path that has never been measured and just failed is not a mount + // this node can see at all — a media root that exists on another node, + // or a scratch dir that has not been created yet. + entry.unreachable = !entry.haveGood + } else { + entry.lastErr = false + entry.unreachable = false + entry.haveGood = true + entry.good = stats + entry.goodAt = s.now() + } + s.diskMu.Unlock() + + if s.diskProbeDone != nil { + s.diskProbeDone <- entry.path + } +} + +// diskStats reports the latest known state of every tracked path, in the order +// the paths were first offered — scratch dir first, since that is the volume a +// full disk breaks first — deduplicated by filesystem and capped. +func (s *Sampler) diskStats(paths []string, now time.Time) []DiskStats { + s.diskMu.Lock() + defer s.diskMu.Unlock() + + wanted := make(map[string]bool, len(paths)) + for _, path := range paths { + wanted[path] = true + } + + out := make([]DiskStats, 0, min(len(s.diskOrder), maxSampledDisks)) + seenFS := make(map[string]bool, len(s.diskOrder)) + libraries := 0 + for _, path := range s.diskOrder { + if !wanted[path] { + continue + } + entry := s.disks[path] + if entry == nil { + continue + } + scratch := s.scratchDir != "" && path == s.scratchDir + // The role is assigned before the measurability check, so the index + // belongs to the mount rather than to its luck this pass. Numbering + // only the measurable ones would slide every library root up a place + // the moment one went unavailable, and a Prometheus alert keyed on + // library-1 would silently follow a different volume. + role := ScratchDiskRole + if !scratch { + libraries++ + role = "library-" + strconv.Itoa(libraries) + } + if !entry.haveGood { + // Report it rather than hiding it: a media root this node cannot + // see is a deployment fact an operator needs, and silently dropping + // it looks identical to the path not being configured. + out = append(out, DiskStats{Path: path, Role: role, Unavailable: true, Scratch: scratch}) + } else { + if entry.good.FSID != "" { + if seenFS[entry.good.FSID] { + continue + } + seenFS[entry.good.FSID] = true + } + out = append(out, DiskStats{ + Path: path, + Role: role, + UsedGB: bytesToGB(entry.good.UsedBytes), + TotalGB: bytesToGB(entry.good.TotalBytes), + Stale: entry.stale(now, s.interval), + Scratch: scratch, + }) + } + // The cap applies to every entry, measured or not. A host whose library + // roots all live on other nodes produces nothing but unavailable + // entries, and those grow with the library count just as measured ones + // would. + if len(out) >= maxSampledDisks { + break + } + } + return out +} + +// formatFSID renders a statfs f_fsid, or "" when the filesystem published none. +// +// A zero f_fsid is not an identity: the FUSE protocol has no fsid field at all, +// so every rclone, mergerfs and s3fs mount reports zero — and those are exactly +// the mounts a media server uses as library roots. Formatting that as "0:0" +// would make two unrelated mounts look like one filesystem, and the second one +// would be silently dropped from the disk panel, from Prometheus, and from the +// fullest-mount warning. A mount at 98% nobody can see is worse than a +// duplicated row. +func formatFSID(a, b int64) string { + if a == 0 && b == 0 { + return "" + } + return strconv.FormatInt(a, 16) + ":" + strconv.FormatInt(b, 16) +} + +// bytesToGB converts to gibibytes with two decimals kept, which is the +// precision a capacity readout is read at. +func bytesToGB(value uint64) float64 { + const bytesPerGB = float64(1024 * 1024 * 1024) + gb := float64(value) / bytesPerGB + return float64(int64(gb*100+0.5)) / 100 +} diff --git a/internal/nodemetrics/disk_test.go b/internal/nodemetrics/disk_test.go new file mode 100644 index 000000000..e15a192f4 --- /dev/null +++ b/internal/nodemetrics/disk_test.go @@ -0,0 +1,660 @@ +package nodemetrics + +import ( + "context" + "errors" + "os" + "strconv" + "sync" + "testing" + "time" +) + +// diskFixture wires a sampler whose statfs is fully controlled by the test and +// whose probe completions are observable, so nothing here waits on a sleep. +type diskFixture struct { + sampler *Sampler + clock *fakeClock + // mu guards answers and block, which probe goroutines read. + mu sync.Mutex + // answers maps a path to the result its probe returns. + answers map[string]fsStats + // block, when set for a path, parks that probe until it is closed — + // standing in for statfs(2) on a dead NFS server. + block map[string]chan struct{} + done chan string +} + +// wedge makes every subsequent probe of path park forever, as a dead NFS server +// does. Call it only when no probe of that path is in flight, or the test is +// asserting on which sample started the parked probe. +func (f *diskFixture) wedge(t *testing.T, path string) { + t.Helper() + gate := make(chan struct{}) + f.mu.Lock() + f.block[path] = gate + f.mu.Unlock() + t.Cleanup(func() { close(gate) }) +} + +func (f *diskFixture) answer(path string, stats fsStats) { + f.mu.Lock() + defer f.mu.Unlock() + f.answers[path] = stats +} + +func newDiskFixture(t *testing.T, paths ...string) *diskFixture { + t.Helper() + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + clock := newFakeClock() + + f := &diskFixture{ + clock: clock, + answers: map[string]fsStats{}, + block: map[string]chan struct{}{}, + done: make(chan string, 64), + } + roots := paths + scratch := "" + if len(roots) > 0 { + scratch, roots = roots[0], roots[1:] + } + f.sampler = newTestSampler(t, tree, clock, Options{ + ScratchDir: staticScratch(scratch), + MediaRoots: func(context.Context) []string { return roots }, + }) + f.sampler.diskProbeDone = f.done + f.sampler.statfs = func(path string) (fsStats, error) { + f.mu.Lock() + gate, blocked := f.block[path] + stats, known := f.answers[path] + f.mu.Unlock() + if blocked { + <-gate + } + if !known { + return fsStats{}, os.ErrNotExist + } + return stats, nil + } + return f +} + +// sampleAndSettle runs one sampling pass and waits for the probes it started, +// so the next pass begins with nothing in flight. Without this, whether a probe +// launched by the previous pass is still running is a race, and every assertion +// about in-flight state becomes timing-dependent. +func (f *diskFixture) sampleAndSettle(t *testing.T, probes int) []DiskStats { + t.Helper() + disks := f.disks(t) + f.awaitProbes(t, probes) + return disks +} + +// awaitProbes blocks until n probe completions have been reported. Waiting on +// the sampler's own signal keeps the test deterministic without a sleep. +func (f *diskFixture) awaitProbes(t *testing.T, n int) { + t.Helper() + for range n { + select { + case <-f.done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a disk probe to complete") + } + } +} + +func (f *diskFixture) disks(t *testing.T) []DiskStats { + t.Helper() + f.sampler.sample(context.Background()) + system := f.sampler.Snapshot().System + if system == nil { + t.Fatal("no system stats in snapshot") + } + return system.Disks +} + +// Scratch and a media root frequently live on one volume. Reporting both would +// double a dashboard's disk row and make one filling disk look like two. +func TestDiskStatsDeduplicatesByFilesystem(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media", "/other") + f.answer("/transcode", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/media", fsStats{UsedBytes: 100 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/other", fsStats{UsedBytes: 7 << 30, TotalBytes: 8 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 3) // the first pass only launches the probes + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 2 { + t.Fatalf("disks = %+v, want the two distinct filesystems", disks) + } + if disks[0].Path != "/transcode" { + t.Fatalf("disks[0].Path = %q, want the scratch dir first", disks[0].Path) + } + if disks[0].UsedGB != 100 || disks[0].TotalGB != 500 { + t.Fatalf("disks[0] = %+v, want 100/500 GB", disks[0]) + } + if disks[1].Path != "/other" { + t.Fatalf("disks[1].Path = %q, want the second filesystem", disks[1].Path) + } + for _, disk := range disks { + if disk.Stale || disk.Unavailable { + t.Fatalf("disk %+v flagged stale/unavailable on a fresh probe", disk) + } + } +} + +// The API stores a node's sample opaquely and does not know its transcode +// directory, so the scratch entry has to identify itself: the admission guard +// and the Prometheus label both find it by this flag, not by matching a path. +func TestDiskStatsFlagsTheScratchMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media", "/missing") + f.answer("/transcode", fsStats{UsedBytes: 96 << 30, TotalBytes: 100 << 30, FSID: "a:1"}) + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 3) + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 3 { + t.Fatalf("disks = %+v, want three entries", disks) + } + if !disks[0].Scratch || disks[0].Path != "/transcode" { + t.Fatalf("disks[0] = %+v, want the scratch dir flagged and first", disks[0]) + } + for _, disk := range disks[1:] { + if disk.Scratch { + t.Fatalf("non-scratch mount flagged as scratch: %+v", disk) + } + } +} + +// A scratch dir that does not exist yet is still the scratch entry: an +// unavailable reading has to be distinguishable from a media root's, and the +// admission guard depends on telling "cannot measure" from "not the scratch". +func TestDiskStatsFlagsAnUnavailableScratchMount(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media") + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 2) + disks := f.sampleAndSettle(t, 2) + + if len(disks) == 0 || disks[0].Path != "/transcode" { + t.Fatalf("disks = %+v, want the scratch dir first", disks) + } + if !disks[0].Unavailable || !disks[0].Scratch { + t.Fatalf("disks[0] = %+v, want an unavailable scratch entry", disks[0]) + } +} + +// A sampler with no scratch dir — a proxy node — flags nothing, so a reader +// never mistakes a media root for the transcode volume. +func TestDiskStatsFlagsNoScratchWithoutAScratchDir(t *testing.T) { + f := newDiskFixture(t, "", "/media") + f.answer("/media", fsStats{UsedBytes: 10 << 30, TotalBytes: 100 << 30, FSID: "b:2"}) + + f.sampleAndSettle(t, 1) + disks := f.sampleAndSettle(t, 1) + + for _, disk := range disks { + if disk.Scratch { + t.Fatalf("mount flagged as scratch with no scratch dir configured: %+v", disk) + } + } +} + +// The contract that matters most: a mount whose server died reports its last +// good numbers marked stale, and never delays a sample. +func TestDiskStatsReportsHungMountAsStaleWithoutBlocking(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/nfs") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + f.answer("/nfs", fsStats{UsedBytes: 900 << 30, TotalBytes: 1000 << 30, FSID: "n:1"}) + + f.sampleAndSettle(t, 2) + if disks := f.sampleAndSettle(t, 2); disks[1].Stale { + t.Fatalf("disks[1] = %+v, want fresh before the mount wedges", disks[1]) + } + + // The NFS server goes away: every subsequent probe parks forever. Nothing is + // in flight here, so the next sample is unambiguously the one that parks. + f.wedge(t, "/nfs") + + // Well past the probe budget and one sampling interval. + f.clock.advance(time.Minute) + wedgedAt := f.clock.at + disks := f.disks(t) + + f.awaitProbes(t, 1) // the scratch probe still completes normally + if len(disks) != 2 { + t.Fatalf("disks = %+v, want both mounts reported", disks) + } + if disks[1].Path != "/nfs" || !disks[1].Stale { + t.Fatalf("disks[1] = %+v, want /nfs marked stale", disks[1]) + } + if disks[1].UsedGB != 900 || disks[1].TotalGB != 1000 { + t.Fatalf("disks[1] = %+v, want the last good numbers preserved", disks[1]) + } + + // A permanently stuck mount must not accumulate a goroutine per sample. + for range 5 { + f.clock.advance(time.Minute) + f.sampleAndSettle(t, 1) // only the scratch probe can complete + } + f.sampler.diskMu.Lock() + entry := f.sampler.disks["/nfs"] + inFlight := entry.inFlight + startedAt := entry.startedAt + f.sampler.diskMu.Unlock() + if !inFlight { + t.Fatal("stuck mount is not marked in flight") + } + // startedAt still points at the sample that launched the parked probe, which + // is only true if no later sample launched another one. + if !startedAt.Equal(wedgedAt) { + t.Fatalf("stuck mount was re-probed: startedAt = %v, want %v", startedAt, wedgedAt) + } +} + +// A media root that exists on other nodes but not this one is reported rather +// than hidden: an operator needs to see the gap, and zeros would read as an +// empty disk. +func TestDiskStatsReportsUnseenPathAsUnavailable(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media-on-another-node") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + + f.sampleAndSettle(t, 2) + disks := f.sampleAndSettle(t, 2) + + if len(disks) != 2 { + t.Fatalf("disks = %+v, want both paths reported", disks) + } + if !disks[1].Unavailable { + t.Fatalf("disks[1] = %+v, want Unavailable", disks[1]) + } + if disks[1].UsedGB != 0 || disks[1].TotalGB != 0 { + t.Fatalf("disks[1] = %+v, want no capacity numbers alongside Unavailable", disks[1]) + } +} + +// A deployment with dozens of library roots must not grow the health response +// without bound. Only the capped set is probed, so that is also how many probe +// completions this can wait for. +func TestDiskStatsCapsMountCount(t *testing.T) { + paths := make([]string, 0, 12) + for i := range 12 { + paths = append(paths, "/mount"+itoa(i)) + } + f := newDiskFixture(t, paths...) + for i, path := range paths { + f.answer(path, fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "fs:" + itoa(i)}) + } + + f.sampleAndSettle(t, maxSampledDisks) + disks := f.sampleAndSettle(t, maxSampledDisks) + + if len(disks) != maxSampledDisks { + t.Fatalf("len(disks) = %d, want the cap of %d", len(disks), maxSampledDisks) + } +} + +// The cap exists so a health response does not grow with the library count, and +// a path this host cannot measure costs an entry just like a measured one. An +// API host whose library roots live on the nodes reports nothing but +// unavailable entries. +func TestDiskStatsCapsUnavailableMountsToo(t *testing.T) { + paths := make([]string, 0, 20) + for i := range 20 { + paths = append(paths, "/mount"+itoa(i)) + } + f := newDiskFixture(t, paths...) + // Only the scratch dir can be measured; every library root fails statfs. + f.answer(paths[0], fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + + f.sampleAndSettle(t, maxSampledDisks) + disks := f.sampleAndSettle(t, maxSampledDisks) + + if len(disks) != maxSampledDisks { + t.Fatalf("len(disks) = %d, want the cap of %d even when the mounts are unavailable", len(disks), maxSampledDisks) + } +} + +// Several filesystems leave statfs's f_fsid zero — FUSE has no fsid in its +// protocol at all, which covers rclone, mergerfs and s3fs. Treating that as an +// identity collapses unrelated media roots onto one entry, and the mount that +// disappears is a real volume with real capacity nobody is watching any more. +func TestDiskStatsDoesNotDeduplicateMountsWithoutAnFSID(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/mnt/rclone-movies", "/mnt/rclone-tv") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + // What osStatfs reports for a FUSE mount: no usable filesystem id. + f.answer("/mnt/rclone-movies", fsStats{UsedBytes: 1 << 30, TotalBytes: 100 << 30}) + f.answer("/mnt/rclone-tv", fsStats{UsedBytes: 98 << 30, TotalBytes: 100 << 30}) + + f.sampleAndSettle(t, 3) + disks := f.sampleAndSettle(t, 3) + + if len(disks) != 3 { + t.Fatalf("disks = %+v, want all three mounts reported", disks) + } + if disks[2].Path != "/mnt/rclone-tv" || disks[2].UsedGB != 98 { + t.Fatalf("disks[2] = %+v, want the nearly full second FUSE mount kept", disks[2]) + } +} + +// The dedup itself must still work for filesystems that do publish an id. +func TestFormatFSIDDropsAZeroIdentity(t *testing.T) { + if got := formatFSID(0, 0); got != "" { + t.Fatalf("formatFSID(0, 0) = %q, want no identity", got) + } + if got := formatFSID(0x1a, 0x2b); got != "1a:2b" { + t.Fatalf("formatFSID(0x1a, 0x2b) = %q, want 1a:2b", got) + } +} + +// Library roots change over a server's life. Entries for paths no longer +// configured must be forgotten rather than accumulate for months. +func TestDiskStatsForgetsPathsNoLongerConfigured(t *testing.T) { + roots := []string{"/media-a"} + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + clock := newFakeClock() + done := make(chan string, 16) + + s := newTestSampler(t, tree, clock, Options{ + ScratchDir: staticScratch("/transcode"), + MediaRoots: func(context.Context) []string { return roots }, + }) + s.diskProbeDone = done + s.statfs = func(path string) (fsStats, error) { + return fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: path}, nil + } + + s.sample(context.Background()) + for range 2 { + <-done + } + if disks := s.Snapshot().System.Disks; len(disks) != 2 { + t.Fatalf("disks = %+v, want both paths tracked", disks) + } + + // The library moves to a different root. + roots = []string{"/media-b"} + s.sample(context.Background()) + for range 2 { + <-done + } + s.sample(context.Background()) + + s.diskMu.Lock() + _, stillTracked := s.disks["/media-a"] + order := append([]string(nil), s.diskOrder...) + s.diskMu.Unlock() + if stillTracked { + t.Fatalf("the removed root is still tracked: %v", order) + } + if len(order) != 2 { + t.Fatalf("diskOrder = %v, want only the two current paths", order) + } +} + +func TestOSStatfsOnUnsupportedPlatformIsNotFatal(t *testing.T) { + // Guards the non-Linux build path: the fallback must return an error rather + // than panic, since NewSampler installs it unconditionally. + if _, err := osStatfs("/definitely/not/a/path"); err == nil { + t.Fatal("statfs on a missing path returned no error") + } else if errors.Is(err, errors.ErrUnsupported) { + t.Log("platform has no statfs; sampler correctly reports paths unavailable") + } +} + +// A mount's role is what the unauthenticated surfaces name it by, so it has to +// be assigned to the mount rather than to its luck this pass: numbering only the +// measurable entries would slide every library root up a place the moment one +// went unavailable. +func TestDiskStatsAssignsPositionalRoles(t *testing.T) { + f := newDiskFixture(t, "/transcode", "/media/movies", "/media/shows") + f.answer("/transcode", fsStats{UsedBytes: 1 << 30, TotalBytes: 10 << 30, FSID: "a:1"}) + // /media/movies is never answered, so it stays unavailable. + f.answer("/media/shows", fsStats{UsedBytes: 3 << 30, TotalBytes: 30 << 30, FSID: "c:1"}) + + f.sampleAndSettle(t, 3) + disks := f.sampleAndSettle(t, 3) + + got := map[string]string{} + for _, disk := range disks { + got[disk.Path] = disk.Role + } + want := map[string]string{ + "/transcode": ScratchDiskRole, + "/media/movies": "library-1", + "/media/shows": "library-2", + } + for path, role := range want { + if got[path] != role { + t.Fatalf("role for %s = %q, want %q (all: %v)", path, got[path], role, got) + } + } +} + +// statfs on a dead network mount is uninterruptible, so every probe started is a +// goroutine that may never return. Bounding only the published output would let +// a deployment with forty library roots start forty probes every interval to +// fill eight slots. +func TestDiskPathsAreBoundedByTheSampleCap(t *testing.T) { + roots := make([]string, 0, maxSampledDisks+4) + for i := range maxSampledDisks + 4 { + roots = append(roots, "/media/root-"+strconv.Itoa(i)) + } + f := newDiskFixture(t, append([]string{"/transcode"}, roots...)...) + + paths := f.sampler.diskPaths(context.Background()) + if len(paths) != maxSampledDisks { + t.Fatalf("probed paths = %d, want the cap of %d", len(paths), maxSampledDisks) + } + // The scratch dir is what admission control reads, so it is never the entry + // the cap drops. + if paths[0] != "/transcode" { + t.Fatalf("first probed path = %q, want the scratch dir", paths[0]) + } +} + +// Bounding the paths offered per sample is not enough on its own. A probe stuck +// on a dead mount is deliberately kept, so a deployment whose library roots +// churn while mounts are wedged would retire one set of parked goroutines' +// paths and immediately be free to park a fresh set for the replacements. +// Without a ceiling the parked count grows every time that repeats. +func TestDiskProbesAreBoundedAcrossReconfiguration(t *testing.T) { + f := newDiskFixture(t, "/transcode") + // Every mount wedges: the probe goroutine parks and never returns, which is + // what statfs on a dead network mount actually does. + roots := make([]string, 0, maxOutstandingDiskProbes*3) + for i := range cap(roots) { + path := "/media/wedged-" + strconv.Itoa(i) + roots = append(roots, path) + f.wedge(t, path) + } + f.wedge(t, "/transcode") + + // Offer a fresh set of roots each pass, as a library reconfiguration would. + for pass := range 3 { + window := roots[pass*maxOutstandingDiskProbes : (pass+1)*maxOutstandingDiskProbes] + f.sampler.refreshDisks(append([]string{"/transcode"}, window...), f.clock.now()) + } + + f.sampler.diskMu.Lock() + outstanding := f.sampler.probesInFlight + f.sampler.diskMu.Unlock() + if outstanding > maxOutstandingDiskProbes { + t.Fatalf("outstanding probes = %d, want at most %d", outstanding, maxOutstandingDiskProbes) + } +} + +// A probe parked on a mount that has since been retired holds one global probe +// slot until the process exits. Offering the same paths in the same order every +// sample then spends the remaining budget on the same prefix, and the path at +// the end of the list is never measured even once — reported unavailable for as +// long as the dead mount stays dead, which is a lie about a healthy disk. +func TestRefreshDisksRotatesProbesPastAWedgedRetiredMount(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{ScratchDir: staticScratch("/transcode")}) + done := make(chan string, 64) + s.diskProbeDone = done + + gate := make(chan struct{}) + t.Cleanup(func() { close(gate) }) + var mu sync.Mutex + probed := map[string]bool{} + s.statfs = func(path string) (fsStats, error) { + if path == "/dead" { + <-gate + return fsStats{}, os.ErrNotExist + } + mu.Lock() + probed[path] = true + mu.Unlock() + return fsStats{UsedBytes: 1 << 30, TotalBytes: 2 << 30, FSID: path}, nil + } + + now := clock.now() + // One sample parks a probe on a mount that the next configuration drops. + s.refreshDisks([]string{"/dead"}, now) + + live := []string{"/transcode"} + for i := range maxOutstandingDiskProbes - 1 { + live = append(live, "/media/"+strconv.Itoa(i)) + } + // live fills the ceiling exactly and one slot is gone for good, so each pass + // can start one fewer probe than there are paths. + perPass := len(live) - 1 + for range len(live) { + s.refreshDisks(live, now) + for range perPass { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a disk probe to complete") + } + } + } + + mu.Lock() + defer mu.Unlock() + for _, path := range live { + if !probed[path] { + t.Fatalf("%s was never probed across %d passes; the retired mount starved it", path, len(live)) + } + } +} + +// Scratch keeps its priority through the rotation: it is the mount transcode +// admission reads, so it must be the one that gets a freed slot. +func TestRefreshDisksAlwaysOffersScratchFirst(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{ScratchDir: staticScratch("/transcode")}) + // Which mount is the scratch one is resolved once per pass, so the ordering + // below is asking a question a pass has to have asked first. + s.diskPaths(context.Background()) + + candidates := []*diskEntry{{path: "/transcode"}, {path: "/a"}, {path: "/b"}, {path: "/c"}} + for pass := range 6 { + ordered := s.probeOrderLocked(candidates) + if ordered[0].path != "/transcode" { + t.Fatalf("pass %d offered %q first, want the scratch dir", pass, ordered[0].path) + } + if len(ordered) != len(candidates) { + t.Fatalf("pass %d offered %d entries, want all %d", pass, len(ordered), len(candidates)) + } + } +} + +// Capacity is measured against what this process can write, not against the +// device: a filesystem that reserves blocks for root hands them to nobody else, +// and counting them as headroom is what makes a volume with nothing left read +// as 95% full — exactly where the scratch admission guard sits. +func TestFSCapacityExcludesBlocksReservedFromThisProcess(t *testing.T) { + const block = 4096 + // 100 GiB with a 5% root reserve, filled to the point an unprivileged + // process can write nothing more. + total := uint64(100<<30) / block + reserved := total / 20 + stats := fsCapacity(total, reserved, 0, block) + + if stats.UsedBytes != (total-reserved)*block { + t.Fatalf("UsedBytes = %d, want the reserve counted as used, as df does", stats.UsedBytes) + } + if stats.TotalBytes != stats.UsedBytes { + t.Fatalf("TotalBytes = %d, want %d: no writable bytes remain", stats.TotalBytes, stats.UsedBytes) + } + + // An empty volume reports the whole writable capacity, which is the device + // size less the reserve rather than the device size. + empty := fsCapacity(total, total, total-reserved, block) + if empty.UsedBytes != 0 { + t.Fatalf("UsedBytes = %d on an empty volume, want 0", empty.UsedBytes) + } + if empty.TotalBytes != (total-reserved)*block { + t.Fatalf("TotalBytes = %d, want the reserve excluded from capacity", empty.TotalBytes) + } + + // Nonsense counters must not wrap an unsigned subtraction into a petabyte. + if got := fsCapacity(10, 20, 5, block); got.UsedBytes != 0 { + t.Fatalf("UsedBytes = %d for free > total, want 0", got.UsedBytes) + } +} + +// staticScratch is the provider for a host whose scratch dir never moves, which +// is every test but the one covering a hot-reloaded one. +func staticScratch(path string) func() string { + return func() string { return path } +} + +// playback.transcode_dir is hot-reloadable, and in integrated mode this host is +// the one transcoding. A sampler that captured the directory at startup would +// keep reporting headroom on a volume nothing writes to while the new one fills +// unwatched — and the scratch role is what transcode admission reads. +func TestScratchDirFollowsAHotReloadedTranscodeDir(t *testing.T) { + f := newDiskFixture(t, "/transcode-old") + f.answer("/transcode-old", fsStats{UsedBytes: 10 << 30, TotalBytes: 500 << 30, FSID: "a:1"}) + f.answer("/transcode-new", fsStats{UsedBytes: 490 << 30, TotalBytes: 500 << 30, FSID: "b:2"}) + current := "/transcode-old" + f.sampler.scratchDirFn = func() string { return current } + + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + scratch := scratchDisk(t, f.sampler) + if scratch.Path != "/transcode-old" { + t.Fatalf("scratch = %q, want the configured /transcode-old", scratch.Path) + } + + // An operator repoints the setting at a nearly full volume. + current = "/transcode-new" + f.sampleAndSettle(t, 1) + f.sampleAndSettle(t, 1) + scratch = scratchDisk(t, f.sampler) + if scratch.Path != "/transcode-new" { + t.Fatalf("scratch = %q, want the reloaded /transcode-new", scratch.Path) + } + if scratch.UsedGB < 400 { + t.Fatalf("scratch used = %v GiB, want the new volume's near-full reading", scratch.UsedGB) + } +} + +// scratchDisk returns the disk carrying the scratch role, which is the one +// transcode admission reads. +func scratchDisk(t *testing.T, s *Sampler) DiskStats { + t.Helper() + for _, disk := range s.Snapshot().System.Disks { + if disk.Role == ScratchDiskRole { + return disk + } + } + t.Fatalf("no disk carried the scratch role: %+v", s.Snapshot().System.Disks) + return DiskStats{} +} diff --git a/internal/nodemetrics/fdinfo.go b/internal/nodemetrics/fdinfo.go new file mode 100644 index 000000000..a49e13a66 --- /dev/null +++ b/internal/nodemetrics/fdinfo.go @@ -0,0 +1,290 @@ +package nodemetrics + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// DRM fdinfo baseline. +// +// Every process holding a DRM device publishes per-engine nanosecond counters +// under /proc//fdinfo/. This needs no capabilities, no extra binaries +// and no driver-specific library, and it measures exactly the thing worth +// measuring on a transcode node: how much GPU engine time *our* ffmpeg children +// are consuming, per device. Its one blind spot is other tenants — a GPU shared +// with something outside this process looks idle here — which is what the +// nvidia-smi enrichment exists to cover where it can. +// +// Field names differ by driver (i915, xe, amdgpu), so engines are classified by +// name shape rather than matched against a fixed per-driver table: a driver we +// have never seen still reports usefully as long as it follows the DRM fdinfo +// convention. + +// engineClass distinguishes the two engine groups worth separating for a media +// workload: fixed-function video (encode/decode) and everything general-purpose. +type engineClass int + +const ( + engineOther engineClass = iota + engineVideo + engineRender +) + +const ( + fdinfoEnginePrefix = "drm-engine-" + fdinfoPdevKey = "drm-pdev" + fdinfoClientIDKey = "drm-client-id" +) + +// classifyEngine maps a drm-engine-* field name to its class. +// +// - i915 reports render / video / video-enhance / copy +// - xe reports rcs / vcs / vecs / bcs / ccs +// - amdgpu reports gfx / compute / enc / dec / jpeg / vcn +func classifyEngine(field string) engineClass { + name, ok := strings.CutPrefix(field, fdinfoEnginePrefix) + if !ok { + return engineOther + } + name = strings.ToLower(name) + switch { + case strings.HasPrefix(name, "video"), strings.HasPrefix(name, "enc"), + strings.HasPrefix(name, "dec"), strings.HasPrefix(name, "vcn"), + strings.HasPrefix(name, "jpeg"), strings.HasPrefix(name, "vcs"), + strings.HasPrefix(name, "vecs"): + return engineVideo + case strings.HasPrefix(name, "render"), strings.HasPrefix(name, "gfx"), + strings.HasPrefix(name, "compute"), strings.HasPrefix(name, "rcs"), + strings.HasPrefix(name, "ccs"): + return engineRender + default: + // Copy/blitter engines and anything unrecognized: real work, but not + // work a video pipeline's headroom is judged by. + return engineOther + } +} + +// engineCounters is cumulative engine time for one device. +type engineCounters struct { + videoNS uint64 + renderNS uint64 +} + +// fdinfoClient is one DRM client (a drm-client-id on a pdev). A process holds +// the same client on several fds — ffmpeg dups its device fd across filter and +// encoder contexts — and each fd reports the client's full counters, so +// counting per fd would multiply one GPU's busyness by its fd count. +type fdinfoClient struct { + pdev string + clientID string +} + +// readFdinfoCounters reads cumulative engine time per DRM client across the +// given processes, deduplicating the fds one client is held on. +// +// Counters stay per client rather than being summed per device here, because +// only a client's own counter is monotone: a device total falls whenever one of +// several clients exits, and a caller diffing that total would read the drop as +// negative work for every transcode still running on the card. Summing is the +// caller's job, after it has taken per-client deltas — see deviceEngineDeltas. +func readFdinfoCounters(procDir string, pids []int) map[fdinfoClient]engineCounters { + clients := make(map[fdinfoClient]engineCounters) + for _, pid := range pids { + dir := filepath.Join(procDir, strconv.Itoa(pid), "fdinfo") + entries, err := os.ReadDir(dir) + if err != nil { + // A transcode that exited between listing and reading is the normal + // case, not an error worth reporting. + continue + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + continue + } + client, counters, ok := parseFdinfo(pid, string(raw)) + if !ok { + continue + } + // Highest wins: two fds on one client report the same counters, and + // a read that raced an update reports slightly less. + existing := clients[client] + if counters.videoNS > existing.videoNS { + existing.videoNS = counters.videoNS + } + if counters.renderNS > existing.renderNS { + existing.renderNS = counters.renderNS + } + clients[client] = existing + } + } + return clients +} + +// deviceEngineDeltas converts two per-client readings into the engine time each +// device accumulated between them. +// +// A client absent from the previous reading contributes its whole counter: the +// only clients read here are ffmpeg children this process spawned, so a client +// that was not there last pass started during this interval and every +// nanosecond on its counter was earned inside it. +// +// A client that vanished contributes nothing, and — the point of doing this per +// client — costs the device nothing either. The device sum drops when a +// transcode exits, so diffing sums would report zero busy for the whole card +// while its surviving transcodes ran flat out. +func deviceEngineDeltas(previous, current map[fdinfoClient]engineCounters) map[string]engineCounters { + deltas := make(map[string]engineCounters, len(current)) + for client, counters := range current { + before := previous[client] + delta := deltas[client.pdev] + // Counters are monotone per client, so a fall is a driver reset or a + // reused client id, not negative work. + if counters.videoNS > before.videoNS { + delta.videoNS += counters.videoNS - before.videoNS + } + if counters.renderNS > before.renderNS { + delta.renderNS += counters.renderNS - before.renderNS + } + deltas[client.pdev] = delta + } + return deltas +} + +// parseFdinfo reads one /proc//fdinfo/ file. Only DRM fds carry +// drm-pdev; everything else (sockets, files, the transcode output itself) is +// rejected by the missing key. +func parseFdinfo(pid int, content string) (fdinfoClient, engineCounters, bool) { + var client fdinfoClient + var counters engineCounters + for line := range strings.Lines(content) { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case fdinfoPdevKey: + client.pdev = NormalizePCIAddress(value) + case fdinfoClientIDKey: + client.clientID = value + default: + class := classifyEngine(key) + if class == engineOther { + continue + } + // "12345678 ns" — the unit column is part of the convention. + fields := strings.Fields(value) + if len(fields) == 0 { + continue + } + ns, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + if class == engineVideo { + counters.videoNS += ns + } else { + counters.renderNS += ns + } + } + } + if client.pdev == "" { + return fdinfoClient{}, engineCounters{}, false + } + if client.clientID == "" { + // A driver that publishes no drm-client-id leaves one bucket per process + // per device, merged by highest counter. Keying by fd instead would + // multiply a GPU's busyness by the fd count (ffmpeg dups its device fd), + // and keying by the counter values would mint a new identity every time + // the client did work — which is precisely the identity the interval + // deltas have to follow. Two genuinely distinct anonymous clients in one + // process collapse to the busier of the two; that undercounts, which is + // the safe direction and only affects drivers old enough to omit the id. + client.clientID = "anon:pid:" + strconv.Itoa(pid) + } + return client, counters, true +} + +// engineBusyPercent converts an engine-time delta into a busy percentage over +// the elapsed wall time. +// +// The delta is already per-device work done in the interval (deviceEngineDeltas +// takes it per client, so client churn never produces a negative one). It can +// exceed the interval on a device with several engines of one class running +// concurrently, which clamps to 100 rather than reporting an impossible figure. +func engineBusyPercent(deltaNS uint64, elapsedNS int64) int { + if elapsedNS <= 0 || deltaNS == 0 { + return 0 + } + return clampPercent(int(deltaNS * 100 / uint64(elapsedNS))) +} + +// defaultFFmpegChildren returns the pids of this process's direct children that +// are ffmpeg. +// +// Direct children only, deliberately: those are the processes this node +// spawned and is accountable for. Walking the whole process table would pick up +// every other tenant's encoder on a shared host and report their GPU time as +// ours. +func defaultFFmpegChildren(procDir string, pid int) []int { + taskDir := filepath.Join(procDir, strconv.Itoa(pid), "task") + tasks, err := os.ReadDir(taskDir) + if err != nil { + return nil + } + seen := make(map[int]bool) + var pids []int + for _, task := range tasks { + raw, err := os.ReadFile(filepath.Join(taskDir, task.Name(), "children")) + if err != nil { + continue + } + for _, field := range strings.Fields(string(raw)) { + child, err := strconv.Atoi(field) + if err != nil || seen[child] { + continue + } + seen[child] = true + if !processIsFFmpeg(procDir, child) { + continue + } + pids = append(pids, child) + } + } + return pids +} + +// processIsFFmpeg reports whether a pid's comm names an ffmpeg binary. comm is +// truncated to 15 bytes by the kernel, so this is a substring test rather than +// an equality one. +func processIsFFmpeg(procDir string, pid int) bool { + raw, err := os.ReadFile(filepath.Join(procDir, strconv.Itoa(pid), "comm")) + if err != nil { + return false + } + return strings.Contains(strings.ToLower(strings.TrimSpace(string(raw))), "ffmpeg") +} + +// NormalizePCIAddress makes PCI addresses from different sources comparable. +// DRM fdinfo prints a 16-bit domain (0000:03:00.0), nvidia-smi a 32-bit one +// (00000000:03:00.0), and neither guarantees a case. +func NormalizePCIAddress(address string) string { + address = strings.ToLower(strings.TrimSpace(address)) + domain, rest, ok := strings.Cut(address, ":") + if !ok { + return address + } + value, err := strconv.ParseUint(domain, 16, 64) + if err != nil { + return address + } + return fmt.Sprintf("%04x:%s", value, rest) +} diff --git a/internal/nodemetrics/fdinfo_test.go b/internal/nodemetrics/fdinfo_test.go new file mode 100644 index 000000000..739d6191e --- /dev/null +++ b/internal/nodemetrics/fdinfo_test.go @@ -0,0 +1,349 @@ +package nodemetrics + +import ( + "context" + "testing" + "time" +) + +// i915 names its engines render/video/video-enhance and reports one fd per +// context; ffmpeg dups the device fd, so the same drm-client-id shows up more +// than once with identical counters. +const i915Fdinfo = `pos: 0 +flags: 02100002 +mnt_id: 26 +drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-render: 1000000000 ns +drm-engine-copy: 500000000 ns +drm-engine-video: 2000000000 ns +drm-engine-video-enhance: 0 ns +` + +// amdgpu names them gfx/enc/dec and prints a 32-bit-looking domain. +const amdgpuFdinfo = `pos: 0 +drm-driver: amdgpu +drm-pdev: 0000:03:00.0 +drm-client-id: 7 +drm-engine-gfx: 400000000 ns +drm-engine-enc0: 800000000 ns +drm-engine-dec0: 100000000 ns +` + +// A non-DRM fd (a segment being written) must be ignored entirely. +const plainFdinfo = `pos: 4096 +flags: 02100002 +mnt_id: 26 +` + +func TestReadFdinfoCountersDeduplicatesClientsAndClassifiesEngines(t *testing.T) { + tree := newProcTree(t) + // Two fds on one i915 client, plus an unrelated file fd. + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4242/fdinfo/7", i915Fdinfo) + tree.write("4242/fdinfo/9", plainFdinfo) + // A second process on a different card. + tree.write("4243/fdinfo/3", amdgpuFdinfo) + + clients := readFdinfoCounters(tree.root, []int{4242, 4243, 9999}) + if len(clients) != 2 { + t.Fatalf("clients = %v, want one client per card", clients) + } + + intel := clients[fdinfoClient{pdev: "0000:00:02.0", clientID: "42"}] + // Counted once despite two fds; the copy engine is excluded. + if intel.videoNS != 2_000_000_000 || intel.renderNS != 1_000_000_000 { + t.Fatalf("i915 counters = %+v, want video 2e9 render 1e9 counted once", intel) + } + + amd := clients[fdinfoClient{pdev: "0000:03:00.0", clientID: "7"}] + if amd.renderNS != 400_000_000 { + t.Fatalf("amdgpu render = %d, want gfx classified as render", amd.renderNS) + } + if amd.videoNS != 900_000_000 { + t.Fatalf("amdgpu video = %d, want enc0+dec0 classified as video", amd.videoNS) + } + + // Summing is the caller's job, and only after per-client deltas: these + // counters are cumulative per client, so a device total is not diffable. + deltas := deviceEngineDeltas(nil, clients) + if got := deltas["0000:00:02.0"].videoNS; got != 2_000_000_000 { + t.Fatalf("first-reading delta = %d, want the client's whole counter", got) + } +} + +// A driver that publishes no drm-client-id must still produce one stable +// identity per process and device, or the fd dups would multiply its busyness +// and the identity would change every time the client did work. +func TestReadFdinfoCountersKeysAnonymousClientsByProcess(t *testing.T) { + anon := `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-engine-video: 2000000000 ns +` + tree := newProcTree(t) + tree.write("4242/fdinfo/3", anon) + tree.write("4242/fdinfo/7", anon) + + clients := readFdinfoCounters(tree.root, []int{4242}) + if len(clients) != 1 { + t.Fatalf("clients = %v, want the two fds collapsed onto one client", clients) + } + for client, counters := range clients { + if client.clientID != "anon:pid:4242" { + t.Fatalf("clientID = %q, want an identity that survives the client doing work", client.clientID) + } + if counters.videoNS != 2_000_000_000 { + t.Fatalf("videoNS = %d, want the dup counted once", counters.videoNS) + } + } +} + +// PCI addresses arrive with different domain widths and cases depending on the +// source; they have to collapse to one key or a device is counted twice. +func TestNormalizePCIAddress(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"0000:00:02.0", "0000:00:02.0"}, + {"00000000:03:00.0", "0000:03:00.0"}, + {"0000:03:00.0", "0000:03:00.0"}, + {" 00000000:0A:00.0 ", "0000:0a:00.0"}, + {"not-an-address", "not-an-address"}, + } { + if got := NormalizePCIAddress(tc.in); got != tc.want { + t.Fatalf("NormalizePCIAddress(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestSampleGPUMapsPdevToDevicePathAndComputesBusy(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", i915Fdinfo) + + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return []int{4242} }, + DeviceSessions: func() map[string]int { return map[string]int{"/dev/dri/renderD128": 2} }, + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{ + // nvidia-smi-style wide domain, to prove normalization is what + // joins the two views rather than string equality. + Path: "/dev/dri/renderD128", + PCIAddress: "00000000:00:02.0", + Vendor: "intel", + }} + }, + }) + + s.sample(context.Background()) + first := s.Snapshot().GPU + if len(first) != 1 { + t.Fatalf("GPU = %+v, want one device", first) + } + if first[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path, not the PCI address", first[0].Device) + } + if first[0].Vendor != "intel" || first[0].Sessions != 2 { + t.Fatalf("GPU[0] = %+v, want vendor intel and 2 sessions", first[0]) + } + if first[0].Source != SourceFdinfo { + t.Fatalf("Source = %q, want %q", first[0].Source, SourceFdinfo) + } + if first[0].VideoBusyPct != nil { + t.Fatalf("VideoBusyPct = %d on the first sample, want it unset (nothing to diff against)", *first[0].VideoBusyPct) + } + + // Over 4s of wall time: +2s of video engine, +1s of render engine. + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-render: 2000000000 ns +drm-engine-video: 4000000000 ns +`) + clock.advance(4 * time.Second) + s.sample(context.Background()) + + second := s.Snapshot().GPU[0] + if got := enginePct(t, second.VideoBusyPct); got != 50 { + t.Fatalf("VideoBusyPct = %d, want 50", got) + } + if got := enginePct(t, second.RenderBusyPct); got != 25 { + t.Fatalf("RenderBusyPct = %d, want 25", got) + } + if second.TotalBusyPct != nil { + t.Fatal("TotalBusyPct set without an enrichment source") + } +} + +// One transcode exiting must not erase the work the others did in that +// interval. Their engine time is what the whole GPU panel is read from, and a +// node with normal session churn loses a client inside most intervals. +func TestSampleGPUKeepsSurvivingTranscodeBusyWhenAPeerExits(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // Two transcodes on one card: the survivor at 2s of video engine time, the + // one about to exit at 5s. + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4243/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 43 +drm-engine-video: 5000000000 ns +`) + + pids := []int{4242, 4243} + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return pids }, + }) + s.sample(context.Background()) + + // The peer exits while the survivor saturates the video engine: +5s of + // engine time over a 5s interval. The device total is 7s before and 7s + // after, so a per-device baseline sees no gain and reports an idle GPU. + pids = []int{4242} + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-video: 7000000000 ns +`) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + if got := enginePct(t, s.Snapshot().GPU[0].VideoBusyPct); got != 100 { + t.Fatalf("VideoBusyPct = %d while the surviving transcode ran the engine flat out, want 100", got) + } +} + +// A transcode exiting takes its accumulated engine time out of the device +// total, so the per-device sum falls. That is bookkeeping, not negative work. +func TestSampleGPUClampsNegativeDeltaWhenATranscodeExits(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", i915Fdinfo) + tree.write("4243/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 43 +drm-engine-video: 5000000000 ns +`) + + pids := []int{4242, 4243} + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return pids }, + }) + s.sample(context.Background()) + + // The second transcode exits; only the first process remains. + pids = []int{4242} + clock.advance(5 * time.Second) + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if video, render := enginePct(t, gpu[0].VideoBusyPct), enginePct(t, gpu[0].RenderBusyPct); video != 0 || render != 0 { + t.Fatalf("busy = %d/%d after a transcode exited, want a measured 0/0", video, render) + } + + // The reduced total is the new baseline: the surviving transcode's next + // interval is measured against it, not against the pre-exit sum. + tree.write("4242/fdinfo/3", `drm-driver: i915 +drm-pdev: 0000:00:02.0 +drm-client-id: 42 +drm-engine-video: 3000000000 ns +`) + clock.advance(10 * time.Second) + s.sample(context.Background()) + if got := enginePct(t, s.Snapshot().GPU[0].VideoBusyPct); got != 10 { + t.Fatalf("VideoBusyPct after re-baselining = %d, want 10", got) + } +} + +// A device with no DRM counters and no enrichment still has to appear, so an +// operator sees the GPU exists — but it must say so rather than report zeros as +// a measurement. +func TestSampleGPUReportsKnownDeviceAsUnavailable(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:00:02.0", Vendor: "intel"}} + }, + }) + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 || gpu[0].Source != SourceUnavailable { + t.Fatalf("GPU = %+v, want one device sourced %q", gpu, SourceUnavailable) + } +} + +func TestDefaultFFmpegChildrenMatchesOnlyFFmpegChildren(t *testing.T) { + tree := newProcTree(t) + tree.write("100/task/100/children", "200 300\n") + tree.write("100/task/105/children", "400\n") + tree.write("200/comm", "ffmpeg\n") + // The kernel truncates comm to 15 bytes, which is why this is a substring + // test and not an equality one. + tree.write("300/comm", "ffmpeg-static-b\n") + tree.write("400/comm", "postgres\n") + + pids := defaultFFmpegChildren(tree.root, 100) + if len(pids) != 2 { + t.Fatalf("pids = %v, want the two ffmpeg children", pids) + } + for _, pid := range pids { + if pid == 400 { + t.Fatalf("pids = %v, want the non-ffmpeg child excluded", pids) + } + } +} + +func TestEngineBusyPercentClamps(t *testing.T) { + elapsed := (5 * time.Second).Nanoseconds() + if got := engineBusyPercent(10*uint64(elapsed), elapsed); got != 100 { + t.Fatalf("busy over 100%% = %d, want clamped to 100", got) + } + if got := engineBusyPercent(0, elapsed); got != 0 { + t.Fatalf("busy with no work = %d, want 0", got) + } + if got := engineBusyPercent(100, 0); got != 0 { + t.Fatalf("busy with no elapsed time = %d, want 0", got) + } +} + +// Counters are only monotone per client. A client whose counter fell (a driver +// reset, or a reused client id) must contribute nothing rather than wrap. +func TestDeviceEngineDeltasIgnoresCounterRegressions(t *testing.T) { + client := fdinfoClient{pdev: "0000:00:02.0", clientID: "42"} + previous := map[fdinfoClient]engineCounters{client: {videoNS: 5_000_000_000}} + current := map[fdinfoClient]engineCounters{client: {videoNS: 1_000_000_000}} + if got := deviceEngineDeltas(previous, current)["0000:00:02.0"].videoNS; got != 0 { + t.Fatalf("delta on a counter regression = %d, want 0", got) + } +} + +// enginePct reads an engine percentage a test requires to be present, so a +// missing measurement fails as itself rather than as a nil dereference. +func enginePct(t *testing.T, got *int) int { + t.Helper() + if got == nil { + t.Fatal("engine percentage is unset, want a measurement") + } + return *got +} diff --git a/internal/nodemetrics/meminfo.go b/internal/nodemetrics/meminfo.go new file mode 100644 index 000000000..24b3c9037 --- /dev/null +++ b/internal/nodemetrics/meminfo.go @@ -0,0 +1,226 @@ +package nodemetrics + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +const bytesPerKB = int64(1024) + +// cgroupInactiveFileKeyV2 is the cgroup v2 memory.stat key for reclaimable +// page cache. cgroup v1's memory.stat spells the same figure +// "total_inactive_file". +const ( + cgroupInactiveFileKeyV2 = "inactive_file" + // cgroupInactiveFileKeyV1 is the same quantity under cgroup v1, which + // prefixes its memory.stat totals. + cgroupInactiveFileKeyV1 = "total_inactive_file" +) + +// CgroupMemoryLimitPaths returns the memory-limit files to consult, cgroup v2 +// first. A container that has a limit publishes it in exactly one of these; a +// host has neither. +// +// It returns a fresh slice per call so a caller iterating it cannot reorder the +// preference for everyone else. +func CgroupMemoryLimitPaths() []string { + paths := make([]string, 0, len(cgroupMemoryUsagePaths)) + for _, level := range cgroupMemoryUsagePaths { + paths = append(paths, level.limit) + } + return paths +} + +// cgroupUsagePath pairs one cgroup level's limit with the current-usage file, +// the stat file, and the key that names its page cache. +// +// All four move together because a limit and the usage measured against it have +// to describe the same cgroup. When the binding limit comes from an ancestor — +// a pod cgroup shared with sidecars, a systemd slice shared with other services +// — the memory that fills it is everything charged to that ancestor, not just +// this process's leaf. Pairing the parent's capacity with the leaf's working +// set shows headroom that does not exist, right up until the parent OOMs. +type cgroupUsagePath struct { + limit string + usage string + stat string + inactiveFile string +} + +// cgroupMemoryUsagePaths lists where to read current memory charge, v2 first. +// +// Usage counts reclaimable page cache, so reporting it raw would show a node +// that merely read a large file as nearly out of memory; subtracting inactive +// file pages yields the working set, which is the number that actually predicts +// an OOM kill (and is what `docker stats` reports). +var cgroupMemoryUsagePaths = []cgroupUsagePath{ + { + limit: "/sys/fs/cgroup/memory.max", + usage: "/sys/fs/cgroup/memory.current", + stat: "/sys/fs/cgroup/memory.stat", + inactiveFile: cgroupInactiveFileKeyV2, + }, + { + limit: "/sys/fs/cgroup/memory/memory.limit_in_bytes", + usage: "/sys/fs/cgroup/memory/memory.usage_in_bytes", + stat: "/sys/fs/cgroup/memory/memory.stat", + inactiveFile: cgroupInactiveFileKeyV1, + }, +} + +// ReadMeminfoTotalBytes returns MemTotal from a /proc/meminfo-formatted file. +// +// This lives here rather than beside its first caller because two unrelated +// subsystems need the same answer — Postgres auto-tuning sizes shared_buffers +// from it, and node metrics report it — and a host's memory is one fact, not +// two implementations of one fact. +func ReadMeminfoTotalBytes(path string) (int64, error) { + fields, err := ReadMeminfoBytes(path) + if err != nil { + return 0, err + } + total, ok := fields["MemTotal"] + if !ok { + return 0, errors.New("MemTotal not found") + } + return total, nil +} + +// ReadMeminfoBytes parses a /proc/meminfo-formatted file into bytes per key. +// Keys are returned without the trailing colon. Lines that do not parse are +// skipped rather than failing the read: meminfo grows new keys across kernel +// versions, and one unfamiliar line must not cost the caller every familiar +// one. +func ReadMeminfoBytes(path string) (map[string]int64, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + values := make(map[string]int64) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + key, value, ok := parseMeminfoLine(scanner.Text()) + if !ok { + continue + } + values[key] = value + } + if err := scanner.Err(); err != nil { + return nil, err + } + return values, nil +} + +// parseMeminfoLine reads one "Key: kB" line, converting to bytes. The +// unit column is optional (a handful of meminfo entries are bare counts). +func parseMeminfoLine(line string) (string, int64, bool) { + key, rest, ok := strings.Cut(line, ":") + if !ok { + return "", 0, false + } + fields := strings.Fields(rest) + if len(fields) == 0 { + return "", 0, false + } + value, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return "", 0, false + } + if len(fields) > 1 && strings.EqualFold(fields[1], "kB") { + value *= bytesPerKB + } + return strings.TrimSpace(key), value, true +} + +// EffectiveMemoryLimitBytes returns the tightest cgroup memory limit in force +// on this process, or 0 when none is set. +// +// The root-level limit files alone are only right inside a cgroup-namespaced +// container. A systemd unit with MemoryMax=, or a leaf inheriting a tighter +// limit from a slice or pod ancestor, publishes its limit at this process's +// own cgroup or somewhere above it while the root file reads "max" — so this +// walks the process's own cgroup, every ancestor, and the mount root, and +// takes the tightest concrete limit found, mirroring what the sampler reports +// (the kernel OOM-kills against the tightest level). +func EffectiveMemoryLimitBytes() int64 { + return effectiveMemoryLimitBytes("/proc", cgroupMemoryUsagePaths) +} + +func effectiveMemoryLimitBytes(procDir string, layouts []cgroupUsagePath) int64 { + relative := cgroupRelativePaths(procDir) + tightest := int64(0) + for _, level := range withCgroupSelfUsagePaths(relative, layouts) { + limit, err := ReadCgroupMemoryLimit(level.limit) + if err != nil || limit <= 0 { + continue + } + if tightest == 0 || limit < tightest { + tightest = limit + } + } + return tightest +} + +// ReadCgroupMemoryLimit reads a cgroup memory-limit file and returns the limit +// in bytes, or an error when the cgroup imposes none. +// +// "No limit" is spelled three different ways depending on kernel and runtime — +// the literal "max", an empty file, or a saturated sentinel close to 2^63 — and +// all three must read as "ask the host instead", never as an enormous budget. +func ReadCgroupMemoryLimit(path string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value := strings.TrimSpace(string(raw)) + if value == "" || value == "max" { + return 0, fmt.Errorf("no cgroup memory limit") + } + mem, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, err + } + // Docker may expose a huge sentinel when no concrete memory limit is set. + if mem <= 0 || mem > 1<<60 { + return 0, fmt.Errorf("no concrete cgroup memory limit") + } + return mem, nil +} + +// readCgroupSingleValue reads a file holding one integer. +func readCgroupSingleValue(path string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value, err := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64) + if err != nil { + return 0, err + } + if value < 0 { + return 0, fmt.Errorf("negative cgroup value") + } + return value, nil +} + +// readCgroupStatKey reads one key out of a "key value" cgroup stat file. +func readCgroupStatKey(path, key string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + for line := range strings.Lines(string(raw)) { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != key { + continue + } + return strconv.ParseInt(fields[1], 10, 64) + } + return 0, fmt.Errorf("%s not found in %s", key, path) +} diff --git a/internal/nodemetrics/meminfo_test.go b/internal/nodemetrics/meminfo_test.go new file mode 100644 index 000000000..23b9c3513 --- /dev/null +++ b/internal/nodemetrics/meminfo_test.go @@ -0,0 +1,61 @@ +package nodemetrics + +import ( + "os" + "path/filepath" + "testing" +) + +// A systemd unit with MemoryMax= publishes its limit on the slice above the +// leaf while both the leaf and the mount root read "max". The effective limit +// must come from whichever level actually binds, not from the root files +// alone. +func TestEffectiveMemoryLimitBytesFindsTheBindingAncestor(t *testing.T) { + root := t.TempDir() + previousRoot := cgroupMountRoot + cgroupMountRoot = root + t.Cleanup(func() { cgroupMountRoot = previousRoot }) + + leaf := filepath.Join(root, "system.slice", "silo.service") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatalf("create %s: %v", leaf, err) + } + write := func(path, body string) { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + procDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(procDir, "self"), 0o755); err != nil { + t.Fatalf("create proc self dir: %v", err) + } + write(filepath.Join(procDir, "self", "cgroup"), "0::/system.slice/silo.service\n") + + layouts := []cgroupUsagePath{{ + limit: filepath.Join(root, "memory.max"), + usage: filepath.Join(root, "memory.current"), + stat: filepath.Join(root, "memory.stat"), + inactiveFile: cgroupInactiveFileKeyV2, + }} + const gib = int64(1) << 30 + + write(filepath.Join(root, "memory.max"), "max\n") + write(filepath.Join(root, "system.slice", "memory.max"), "2147483648\n") + write(filepath.Join(leaf, "memory.max"), "max\n") + if got := effectiveMemoryLimitBytes(procDir, layouts); got != 2*gib { + t.Fatalf("effective limit = %d, want the slice's 2GiB", got) + } + + // A leaf tighter than its ancestor binds instead. + write(filepath.Join(leaf, "memory.max"), "1073741824\n") + if got := effectiveMemoryLimitBytes(procDir, layouts); got != gib { + t.Fatalf("effective limit = %d, want the leaf's 1GiB", got) + } + + // No concrete limit anywhere reads as no limit, never as a sentinel. + write(filepath.Join(leaf, "memory.max"), "max\n") + write(filepath.Join(root, "system.slice", "memory.max"), "max\n") + if got := effectiveMemoryLimitBytes(procDir, layouts); got != 0 { + t.Fatalf("effective limit = %d, want 0 for no limit", got) + } +} diff --git a/internal/nodemetrics/nvidia.go b/internal/nodemetrics/nvidia.go new file mode 100644 index 000000000..457ffe61c --- /dev/null +++ b/internal/nodemetrics/nvidia.go @@ -0,0 +1,262 @@ +package nodemetrics + +import ( + "context" + "errors" + "log/slog" + "os/exec" + "strconv" + "strings" + "sync" + "time" +) + +// nvidiaSMITimeout bounds one query. A wedged driver makes nvidia-smi hang +// indefinitely, and a metrics sample must never inherit that. +const nvidiaSMITimeout = 3 * time.Second + +// sourceFailureLimit is how many consecutive failures retire an enrichment +// source for the life of the process. +// +// A source that has failed five times running is not having a bad moment: the +// binary is missing its driver, the container lacks the device, or the query +// syntax is not supported by the installed version. Retrying it forever would +// spawn a doomed subprocess every 5 seconds on every node, which costs more +// than the signal is worth. Recovery is a node restart, which is also what +// installing or fixing the toolkit requires. +const sourceFailureLimit = 5 + +// nvidiaSMIFields is the query column order. utilization.gpu is whole-GPU +// busyness including other tenants; the encoder/decoder columns are the +// fixed-function video engines, which is what a transcode node is actually +// competing for. +const nvidiaSMIFields = "index,uuid,pci.bus_id,utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total" + +// runNVIDIASMI is the execution seam. Tests replace it instead of installing a +// fake binary on PATH. +var runNVIDIASMI = func(ctx context.Context) ([]byte, error) { + path, err := exec.LookPath("nvidia-smi") + if err != nil { + return nil, err + } + return exec.CommandContext(ctx, path, + "--query-gpu="+nvidiaSMIFields, + "--format=csv,noheader,nounits").Output() +} + +// nvidiaGPU is one parsed nvidia-smi row. +// +// The measurement columns are optional because a successful row can still be +// only partly measurable: a driver reports "[N/A]" or "[Not Supported]" per +// column for engines or memory it cannot see, and coercing those to zero would +// publish an unobservable video engine as idle and unsupported VRAM as 0 bytes +// under an "nvidia-smi" source that claims they were measured. +type nvidiaGPU struct { + Index int + UUID string + PCIAddress string + GPUUtil *int + EncoderUtil *int + DecoderUtil *int + MemUsedMB *int64 + MemTotalMB *int64 +} + +// videoUtil is the higher of the two fixed-function video engines, or nil when +// the driver reported neither. +func (g nvidiaGPU) videoUtil() *int { + switch { + case g.EncoderUtil != nil && g.DecoderUtil != nil: + return ptr(max(*g.EncoderUtil, *g.DecoderUtil)) + case g.EncoderUtil != nil: + return g.EncoderUtil + default: + return g.DecoderUtil + } +} + +func ptr[T any](value T) *T { return &value } + +// sourceRetryInterval is how long a retired source waits before one +// probationary query. +// +// The breaker exists so a host without the NVIDIA toolkit stops spawning a +// doomed subprocess every five seconds, and at this interval it still does: +// one exec per ten minutes instead of a hundred and twenty. What it must not do +// is confuse "this host has no toolkit" with "this driver is resetting", which +// look identical for the handful of samples the limit counts. Retiring the +// source until the process restarts turns a recoverable outage into a node that +// reports no GPU utilization or VRAM for as long as it stays up. +const sourceRetryInterval = 10 * time.Minute + +// sourceBreaker retires an enrichment source after repeated failure, and lets +// it back in on probation. +type sourceBreaker struct { + // mu guards the fields below. The sampling goroutine is their only regular + // writer, but reset comes from whatever goroutine serves a re-probe. + mu sync.Mutex + name string + failures int + tripped bool + // retryAt is when a tripped source may next be tried. It advances on every + // attempt, so one probationary query runs per interval whether or not it + // succeeds. + retryAt time.Time + logOnce sync.Once +} + +// allow reports whether the source may be queried, admitting one probationary +// query per sourceRetryInterval once the breaker has tripped. +func (b *sourceBreaker) allow(now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + if !b.tripped { + return true + } + if now.Before(b.retryAt) { + return false + } + b.retryAt = now.Add(sourceRetryInterval) + return true +} + +// succeeded clears the failure count, and closes the breaker when the query +// that succeeded was a probationary one. +func (b *sourceBreaker) succeeded() { + b.mu.Lock() + defer b.mu.Unlock() + b.failures = 0 + if b.tripped { + b.tripped = false + slog.Info("node metrics source answered again", "component", "nodemetrics", "source", b.name) + } +} + +// failed records one failure and trips the breaker at the limit, logging the +// retirement exactly once so a node without the toolkit does not narrate it +// every interval. A probationary query that fails costs nothing further: allow +// has already pushed the next attempt out by a full interval. +func (b *sourceBreaker) failed(now time.Time, err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.tripped { + return + } + b.failures++ + if b.failures < sourceFailureLimit { + return + } + b.tripped = true + b.retryAt = now.Add(sourceRetryInterval) + b.logOnce.Do(func() { + slog.Info("node metrics source unavailable; retrying occasionally", + "component", "nodemetrics", "source", b.name, "failures", b.failures, + "retry_interval", sourceRetryInterval, "error", err) + }) +} + +// reset returns the source to service immediately. +// +// It is what an operator's hardware re-probe calls: a re-probe is the explicit +// statement that something changed underneath this node, which is exactly the +// event the retry interval is otherwise waiting to discover on its own. +func (b *sourceBreaker) reset() { + b.mu.Lock() + defer b.mu.Unlock() + b.failures = 0 + b.tripped = false + b.retryAt = time.Time{} +} + +// queryNVIDIA runs one bounded nvidia-smi query, honoring the breaker. +func (s *Sampler) queryNVIDIA(ctx context.Context) []nvidiaGPU { + now := s.now() + if !s.nvidiaBreaker.allow(now) { + return nil + } + queryCtx, cancel := context.WithTimeout(ctx, nvidiaSMITimeout) + defer cancel() + output, err := s.runNVIDIASMI(queryCtx) + if err != nil { + s.nvidiaBreaker.failed(now, err) + return nil + } + gpus := parseNVIDIASMI(output) + if len(gpus) == 0 { + // A successful command that says nothing is as useless as a failure and + // is how a stale query syntax presents. + s.nvidiaBreaker.failed(now, errNoNVIDIARows) + return nil + } + s.nvidiaBreaker.succeeded() + return gpus +} + +// RetrySources returns every retired enrichment source to service. +// +// A node's hardware re-probe calls it: the operator is saying that something +// changed underneath this process, which is the same event the breaker's retry +// interval would otherwise take up to sourceRetryInterval to notice on its own. +// A driver reinstalled or a toolkit added should show in the next sample, not in +// ten minutes. +func (s *Sampler) RetrySources() { + if s == nil { + return + } + s.nvidiaBreaker.reset() +} + +// errNoNVIDIARows marks a query that succeeded but produced nothing parseable. +var errNoNVIDIARows = errors.New("nvidia-smi returned no parseable rows") + +// parseNVIDIASMI reads "csv,noheader,nounits" rows. Malformed rows are skipped +// individually; a driver that reports "[N/A]" for one column on one GPU must +// not cost the reading for the others. +func parseNVIDIASMI(output []byte) []nvidiaGPU { + var gpus []nvidiaGPU + for line := range strings.Lines(string(output)) { + fields := strings.Split(line, ",") + if len(fields) < 8 { + continue + } + index, err := strconv.Atoi(strings.TrimSpace(fields[0])) + if err != nil { + continue + } + address := NormalizePCIAddress(fields[2]) + uuid := strings.TrimSpace(fields[1]) + if address == "" && uuid == "" { + continue + } + gpus = append(gpus, nvidiaGPU{ + Index: index, + UUID: uuid, + PCIAddress: address, + GPUUtil: parseNVIDIAInt(fields[3]), + EncoderUtil: parseNVIDIAInt(fields[4]), + DecoderUtil: parseNVIDIAInt(fields[5]), + MemUsedMB: parseNVIDIAInt64(fields[6]), + MemTotalMB: parseNVIDIAInt64(fields[7]), + }) + } + return gpus +} + +// parseNVIDIAInt reads one numeric column, or nil for the driver's "[N/A]" and +// "[Not Supported]" placeholders — which say the value was not measured, not +// that it is zero. +func parseNVIDIAInt(field string) *int { + value, err := strconv.Atoi(strings.TrimSpace(field)) + if err != nil || value < 0 { + return nil + } + return &value +} + +func parseNVIDIAInt64(field string) *int64 { + value := parseNVIDIAInt(field) + if value == nil { + return nil + } + return ptr(int64(*value)) +} diff --git a/internal/nodemetrics/nvidia_test.go b/internal/nodemetrics/nvidia_test.go new file mode 100644 index 000000000..49df6d4a3 --- /dev/null +++ b/internal/nodemetrics/nvidia_test.go @@ -0,0 +1,414 @@ +package nodemetrics + +import ( + "context" + "errors" + "testing" + "time" +) + +const nvidiaSMIOutput = "0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n" + + "1, GPU-5678efgh, 00000000:04:00.0, 5, 0, 0, 100, 8192\n" + +func TestParseNVIDIASMI(t *testing.T) { + gpus := parseNVIDIASMI([]byte(nvidiaSMIOutput)) + if len(gpus) != 2 { + t.Fatalf("gpus = %+v, want 2", gpus) + } + first := gpus[0] + if first.Index != 0 || first.UUID != "GPU-1234abcd" { + t.Fatalf("gpus[0] identity = %+v", first) + } + // The wide domain nvidia-smi prints has to normalize to the sysfs form or it + // will never join with a DRM device. + if first.PCIAddress != "0000:03:00.0" { + t.Fatalf("PCIAddress = %q, want the normalized sysfs form", first.PCIAddress) + } + if *first.GPUUtil != 71 || *first.EncoderUtil != 63 || *first.DecoderUtil != 12 { + t.Fatalf("gpus[0] utilization = %d/%d/%d", *first.GPUUtil, *first.EncoderUtil, *first.DecoderUtil) + } + if *first.MemUsedMB != 812 || *first.MemTotalMB != 8192 { + t.Fatalf("gpus[0] memory = %d/%d", *first.MemUsedMB, *first.MemTotalMB) + } +} + +// Drivers print "[N/A]" for a column a card does not support. One unsupported +// column must not discard the whole row — nor be read as a measured zero, which +// would show an engine nobody can see as idle. +func TestParseNVIDIASMIToleratesPlaceholders(t *testing.T) { + gpus := parseNVIDIASMI([]byte("0, GPU-x, 00000000:03:00.0, [N/A], [Not Supported], 4, 100, [N/A]\n")) + if len(gpus) != 1 { + t.Fatalf("gpus = %+v, want the row kept", gpus) + } + got := gpus[0] + if got.GPUUtil != nil || got.EncoderUtil != nil || got.MemTotalMB != nil { + t.Fatalf("gpus[0] = %+v, want the placeholder columns unset", got) + } + if got.DecoderUtil == nil || *got.DecoderUtil != 4 { + t.Fatalf("DecoderUtil = %v, want the reported 4 preserved", got.DecoderUtil) + } + if got.MemUsedMB == nil || *got.MemUsedMB != 100 { + t.Fatalf("MemUsedMB = %v, want the reported 100 preserved", got.MemUsedMB) + } + // One reported engine still gives a video reading; both missing gives none. + if video := got.videoUtil(); video == nil || *video != 4 { + t.Fatalf("videoUtil() = %v, want the one engine the driver did report", video) + } + if video := (nvidiaGPU{}).videoUtil(); video != nil { + t.Fatalf("videoUtil() = %d with neither engine reported, want none", *video) + } +} + +// A card that reports memory but no video engines keeps the nvidia-smi source +// for the columns it did answer, and simply carries no engine reading. +func TestSampleGPUKeepsPartialNVIDIAReadings(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, [N/A], [N/A], 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if gpu[0].Source != SourceNVIDIASMI { + t.Fatalf("Source = %q, want %q for the columns it did measure", gpu[0].Source, SourceNVIDIASMI) + } + if gpu[0].VideoBusyPct != nil { + t.Fatalf("VideoBusyPct = %d, want no reading for engines the driver cannot see", *gpu[0].VideoBusyPct) + } + if gpu[0].TotalBusyPct == nil || *gpu[0].TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want the 71 nvidia-smi did report", gpu[0].TotalBusyPct) + } + if gpu[0].VRAMTotalMB == nil || *gpu[0].VRAMTotalMB != 8192 { + t.Fatalf("VRAMTotalMB = %v, want the 8192 nvidia-smi did report", gpu[0].VRAMTotalMB) + } +} + +func TestSampleGPUEnrichesWithNVIDIASMI(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return map[string]int{"cuda:0": 3} }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return []byte(nvidiaSMIOutput), nil } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 2 { + t.Fatalf("GPU = %+v, want both cards", gpu) + } + first := gpu[0] + // The proprietary driver exposes no DRM node this process can read, so the + // device is named the way playback addresses it. + if first.Device != "cuda:0" { + t.Fatalf("Device = %q, want cuda:0", first.Device) + } + if first.Vendor != "nvidia" || first.Source != SourceNVIDIASMI { + t.Fatalf("GPU[0] = %+v, want nvidia via nvidia-smi", first) + } + if first.Sessions != 3 { + t.Fatalf("Sessions = %d, want the balancer's count for cuda:0", first.Sessions) + } + if first.TotalBusyPct == nil || *first.TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want 71", first.TotalBusyPct) + } + if got := enginePct(t, first.VideoBusyPct); got != 63 { + t.Fatalf("VideoBusyPct = %d, want the busier of encoder/decoder", got) + } + if first.VRAMUsedMB == nil || *first.VRAMUsedMB != 812 { + t.Fatalf("VRAMUsedMB = %v, want 812", first.VRAMUsedMB) + } +} + +// A GPU that has both DRM counters and an nvidia-smi row must be one entry +// crediting both sources, not two entries. +func TestSampleGPUMergesFdinfoAndNVIDIASMIOnOneDevice(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + tree.write("4242/fdinfo/3", `drm-driver: nvidia-drm +drm-pdev: 0000:03:00.0 +drm-client-id: 1 +drm-engine-video: 1000000000 ns +`) + + s := newTestSampler(t, tree, clock, Options{ + FFmpegChildren: func() []int { return []int{4242} }, + DeviceIdentities: func() []DeviceIdentity { + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", Vendor: "nvidia"}} + }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want the two views merged onto one device", gpu) + } + if gpu[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path kept", gpu[0].Device) + } + if gpu[0].Source != SourceFdinfoNVIDIASMI { + t.Fatalf("Source = %q, want %q", gpu[0].Source, SourceFdinfoNVIDIASMI) + } + if gpu[0].TotalBusyPct == nil || *gpu[0].TotalBusyPct != 71 { + t.Fatalf("TotalBusyPct = %v, want the whole-GPU reading", gpu[0].TotalBusyPct) + } +} + +// NVENC workloads are counted under a CUDA name or a GPU UUID, but a card whose +// DRM node this process can read is displayed by its render path. Looking +// sessions up by display name alone reports an idle GPU on an NVIDIA node that +// is transcoding. +func TestSampleGPUJoinsNVENCSessionsThroughDeviceAliases(t *testing.T) { + for _, tc := range []struct { + name string + sessions map[string]int + want int + }{ + {name: "counted by cuda index", sessions: map[string]int{"cuda:0": 3}, want: 3}, + {name: "counted by gpu uuid", sessions: map[string]int{"GPU-1234abcd": 2}, want: 2}, + {name: "counted by render path", sessions: map[string]int{"/dev/dri/renderD128": 1}, want: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return tc.sessions }, + DeviceIdentities: func() []DeviceIdentity { + // The proprietary driver with modeset does expose a render + // node, so this is the ordinary bare-metal NVIDIA shape. + return []DeviceIdentity{{Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", Vendor: "nvidia"}} + }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-1234abcd, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.sample(context.Background()) + + gpu := s.Snapshot().GPU + if len(gpu) != 1 { + t.Fatalf("GPU = %+v, want one device", gpu) + } + if gpu[0].Device != "/dev/dri/renderD128" { + t.Fatalf("Device = %q, want the render node path", gpu[0].Device) + } + if gpu[0].Sessions != tc.want { + t.Fatalf("Sessions = %d, want %d for %v", gpu[0].Sessions, tc.want, tc.sessions) + } + }) + } +} + +// Two cards must not both claim a count keyed by a name only one of them +// answers to. +func TestSampleGPUDoesNotDoubleCountSessionsAcrossDevices(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{ + DeviceSessions: func() map[string]int { return map[string]int{"cuda:1": 2} }, + }) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return []byte(nvidiaSMIOutput), nil } + s.sample(context.Background()) + + total := 0 + for _, gpu := range s.Snapshot().GPU { + total += gpu.Sessions + if gpu.Device == "cuda:0" && gpu.Sessions != 0 { + t.Fatalf("cuda:0 claimed %d sessions belonging to cuda:1", gpu.Sessions) + } + } + if total != 2 { + t.Fatalf("sessions across devices = %d, want the 2 counted exactly once", total) + } +} + +// A host without the NVIDIA toolkit fails this query every 5 seconds forever. +// The breaker stops us from spawning a doomed subprocess for the life of the +// process. +func TestNVIDIACircuitBreakerRetiresSourceAfterRepeatedFailure(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + return nil, errors.New("nvidia-smi: command not found") + } + + for range sourceFailureLimit + 5 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d failures", calls, sourceFailureLimit) + } +} + +// A successful command that parses to nothing is as useless as a failure, and +// is how an unsupported query syntax presents. +func TestNVIDIACircuitBreakerCountsEmptyOutputAsFailure(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + return []byte("\n"), nil + } + for range sourceFailureLimit + 3 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d empty answers", calls, sourceFailureLimit) + } +} + +// A transient failure must not retire the source: a driver busy for one sample +// is normal, and losing the only NVIDIA signal over it would be a regression an +// operator cannot recover from without a restart. +func TestNVIDIACircuitBreakerResetsOnSuccess(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + fail := true + calls := 0 + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("busy") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit - 1 { + s.queryNVIDIA(context.Background()) + } + fail = false + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("recovered query returned %d gpus, want 2", len(gpus)) + } + fail = true + for range sourceFailureLimit - 1 { + s.queryNVIDIA(context.Background()) + } + if calls != 2*(sourceFailureLimit-1)+1 { + t.Fatalf("nvidia-smi invoked %d times, want the failure count reset by the success", calls) + } +} + +// Retiring a source until the process restarts confuses "this host has no +// toolkit" with "this driver is resetting" — they look identical for the five +// samples the limit counts. An NVENC node that hit a transient driver outage +// would then report no utilization and no VRAM for as long as it stayed up. +func TestNVIDIACircuitBreakerHalfOpensAfterItsRetryInterval(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + fail := true + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("nvidia-smi: driver/library version mismatch") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit + 5 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired after %d failures", calls, sourceFailureLimit) + } + + // Still retired just short of the interval: the point of the breaker is that + // a toolkit-less host stops paying for a subprocess every few seconds. + clock.advance(sourceRetryInterval - time.Second) + s.queryNVIDIA(context.Background()) + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times before the retry interval elapsed, want %d", calls, sourceFailureLimit) + } + + // One probationary query per interval, and a failure buys another interval + // rather than a burst. + clock.advance(2 * time.Second) + s.queryNVIDIA(context.Background()) + s.queryNVIDIA(context.Background()) + if calls != sourceFailureLimit+1 { + t.Fatalf("nvidia-smi invoked %d times, want exactly one probationary query", calls) + } + + // The driver comes back. The probationary query that succeeds closes the + // breaker, and sampling resumes at its normal rate. + fail = false + clock.advance(sourceRetryInterval) + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("recovered query returned %d gpus, want 2", len(gpus)) + } + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("query after recovery returned %d gpus, want the breaker closed", len(gpus)) + } + if calls != sourceFailureLimit+3 { + t.Fatalf("nvidia-smi invoked %d times, want the breaker to stop gating once it answered", calls) + } +} + +// A re-probe is an operator saying something changed underneath this node, so +// it must not wait out the retry interval to find out. +func TestRetrySourcesReturnsARetiredSourceImmediately(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + calls := 0 + fail := true + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + calls++ + if fail { + return nil, errors.New("nvidia-smi: command not found") + } + return []byte(nvidiaSMIOutput), nil + } + + for range sourceFailureLimit + 2 { + s.queryNVIDIA(context.Background()) + } + if calls != sourceFailureLimit { + t.Fatalf("nvidia-smi invoked %d times, want it retired", calls) + } + + // The toolkit is installed; the operator re-probes rather than waiting. + fail = false + s.RetrySources() + if gpus := s.queryNVIDIA(context.Background()); len(gpus) != 2 { + t.Fatalf("query after RetrySources returned %d gpus, want the source back in service", len(gpus)) + } +} diff --git a/internal/nodemetrics/sampler.go b/internal/nodemetrics/sampler.go new file mode 100644 index 000000000..559cead5f --- /dev/null +++ b/internal/nodemetrics/sampler.go @@ -0,0 +1,581 @@ +package nodemetrics + +import ( + "context" + "log/slog" + "os" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// DefaultInterval is how often the sampler takes a reading. Rate-based fields +// (CPU, network, GPU engine busy) are averages over exactly this window, so it +// doubles as the resolution of every derived percentage. Five seconds is short +// enough that an operator watching a transcode start sees it, and long enough +// that the sampling itself is not the load. +const DefaultInterval = 5 * time.Second + +// Options configures a Sampler. Every provider is optional: a nil provider +// means that dimension simply is not reported, which is the correct behavior +// for a proxy node with no scratch dir or a host with no GPU. +type Options struct { + // ScratchDir returns the transcode working directory. It is sampled first + // because it is the volume whose filling up silently kills transcodes. + // + // A provider rather than a value because playback.transcode_dir is + // hot-reloadable: a host that captured it at startup would keep measuring + // the volume it used to transcode onto, reporting ample headroom on a disk + // nothing writes to while the one filling up goes unwatched. Called on the + // sampling goroutine each pass, so it must be cheap; a nil provider means + // this host has no scratch dir, which is the right answer for a proxy. + ScratchDir func() string + // MediaRoots returns library folder paths to sample alongside the scratch + // dir. It is called on the sampling goroutine each pass, so it must be cheap + // and must respect ctx; a nil provider samples the scratch dir only. + // + // What it returns is taken as the whole truth: paths outside the set are + // pruned, losing their cached readings, and omitted from the sample. A + // provider backed by anything fallible therefore has to answer with its last + // known set rather than nothing, or a momentary failure reads as every + // library mount disappearing. It is the only provider here that crosses a + // database or network boundary — the others read local state whose failure + // is permanent rather than transient — which is why it is the only one that + // needs to. + MediaRoots func(ctx context.Context) []string + // FFmpegChildren returns the pids whose DRM fdinfo counts as this node's GPU + // work. The default is this process's direct ffmpeg children. + FFmpegChildren func() []int + // DeviceSessions returns active GPU workloads per device, keyed the way the + // playback device balancer keys them (a render node path, or "cuda:N"). + DeviceSessions func() map[string]int + // DeviceIdentities returns this host's render devices, used to translate + // the PCI addresses DRM reports into the /dev/dri paths every other surface + // speaks. + DeviceIdentities func() []DeviceIdentity + // Interval overrides DefaultInterval. + Interval time.Duration + // Now overrides the clock. Tests inject one instead of sleeping. + Now func() time.Time +} + +// Sampler periodically reads host and GPU resource usage into a snapshot. +// +// Exactly one goroutine performs sampling, which is why the per-sample delta +// state below needs no locking; everything a caller can reach concurrently is +// either atomic (the published snapshot) or explicitly guarded (the disk +// entries, which detached probe goroutines also write). +type Sampler struct { + interval time.Duration + now func() time.Time + goos string + // scratchDirFn is the live source; scratchDir is what it answered for the + // pass in flight, resolved once in diskPaths so the three readers below + // cannot disagree about which mount is the scratch one mid-pass. Owned by + // the sampling goroutine. + scratchDirFn func() string + scratchDir string + mediaRoots func(ctx context.Context) []string + sessions func() map[string]int + identities func() []DeviceIdentity + ffmpegPIDs func() []int + + // Path seams. Production values point at the real filesystem; tests point + // them at a fake /proc tree. + procDir string + // hostProcDir is where an LXC's lxcfs-virtualized /proc files can be + // bind-mounted when this sampler runs in Docker nested inside an LXC + // container. See procDirFor for why it takes priority when present. + hostProcDir string + cgroupUsagePaths []cgroupUsagePath + // cgroupCPUSetPaths are the cpuset files consulted when no CFS quota binds. + cgroupCPUSetPaths []string + cgroupCPUPaths []cgroupCPUPath + + snapshot atomic.Pointer[Snapshot] + + // Delta state, owned by the sampling goroutine. + prevCPU cpuTimes + prevNet netCounters + // prevGPU is keyed by DRM client, not by device: only a client's own engine + // counter is monotone, so a per-device baseline would read every client exit + // as negative work for the whole card. + prevGPU map[fdinfoClient]engineCounters + prevGPUAt time.Time + prevCgroupCPU cgroupCPUSample + // droppedRoots is how many configured media roots the last pass left + // unsampled because of the maxSampledDisks cap; see noteDroppedRoots. + droppedRoots int + + // Disk probe state, shared with detached probe goroutines. + diskMu sync.Mutex + disks map[string]*diskEntry + diskOrder []string + statfs func(string) (fsStats, error) + // probesInFlight is how many statfs goroutines are outstanding right now, + // bounded by maxOutstandingDiskProbes. Guarded by diskMu. + probesInFlight int + // diskProbeCursor rotates which non-scratch mount is offered a probe slot + // first, so a slot parked forever on a wedged mount cannot starve the path + // at the end of the list. See probeOrderLocked. + diskProbeCursor int + // probeBudgetExhausted latches the ceiling warning to one line per episode. + probeBudgetExhausted bool + // diskProbeDone, when non-nil, receives each completed probe's path. Tests + // wait on it instead of sleeping; production leaves it nil. + diskProbeDone chan string + + runNVIDIASMI func(ctx context.Context) ([]byte, error) + nvidiaBreaker *sourceBreaker +} + +// NewSampler creates a sampler. It performs no I/O; nothing is read until +// Start. +func NewSampler(opts Options) *Sampler { + interval := opts.Interval + if interval <= 0 { + interval = DefaultInterval + } + now := opts.Now + if now == nil { + now = time.Now + } + procDir := "/proc" + hostProcDir := "/host/proc" + // Read from this process's own /proc, never the lxcfs-mounted host view: + // the question is which cgroup *this* process belongs to. + cgroupSelf := cgroupRelativePaths(procDir) + ffmpegPIDs := opts.FFmpegChildren + if ffmpegPIDs == nil { + pid := os.Getpid() + ffmpegPIDs = func() []int { return defaultFFmpegChildren(procDir, pid) } + } + s := &Sampler{ + interval: interval, + now: now, + goos: runtime.GOOS, + scratchDirFn: opts.ScratchDir, + mediaRoots: opts.MediaRoots, + sessions: opts.DeviceSessions, + identities: opts.DeviceIdentities, + ffmpegPIDs: ffmpegPIDs, + procDir: procDir, + hostProcDir: hostProcDir, + // Each cgroup read is tried at this process's own cgroup before the + // mount root, so a systemd unit with CPUQuota= or MemoryMax= is + // measured against its limit rather than against the whole machine. + // See cgrouppath.go; a container is unaffected either way. + cgroupUsagePaths: withCgroupSelfUsagePaths(cgroupSelf, cgroupMemoryUsagePaths), + cgroupCPUPaths: withCgroupSelfCPUPaths(cgroupSelf, cgroupCPUPaths), + cgroupCPUSetPaths: withCgroupSelfPaths(cgroupSelf, cgroupCPUSetPaths), + prevGPU: map[fdinfoClient]engineCounters{}, + disks: map[string]*diskEntry{}, + statfs: osStatfs, + runNVIDIASMI: runNVIDIASMI, + nvidiaBreaker: &sourceBreaker{name: "nvidia-smi"}, + } + s.snapshot.Store(&Snapshot{}) + return s +} + +// NewFixedSamplerForTest returns a sampler that always answers with the given +// snapshot and never samples anything. +// +// It exists for tests in other packages — the node health and status handlers, +// the admin resources endpoint — that need a known reading without a Linux host +// under them. Nothing in production calls it. +func NewFixedSamplerForTest(snapshot Snapshot) *Sampler { + s := NewSampler(Options{}) + s.snapshot.Store(&snapshot) + return s +} + +// Snapshot returns the most recent sample. It never blocks, never fails, and +// never does I/O — it is safe to call from an HTTP handler on the request path, +// which is the whole reason the sampling loop exists. +func (s *Sampler) Snapshot() Snapshot { + if s == nil { + return Snapshot{} + } + if snapshot := s.snapshot.Load(); snapshot != nil { + return *snapshot + } + return Snapshot{} +} + +// Start samples in the background until ctx is canceled. It returns +// immediately. +// +// On a non-Linux host the loop still runs but does nothing, so callers do not +// need their own platform checks and the published snapshot stays a truthful +// Available=false rather than an absent one. +func (s *Sampler) Start(ctx context.Context) { + if s == nil || ctx == nil { + return + } + registerCollector(s) + go func() { + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + s.sample(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.sample(ctx) + } + } + }() +} + +// sample takes one reading and publishes it. +func (s *Sampler) sample(ctx context.Context) { + if s.goos != "linux" { + s.snapshot.Store(&Snapshot{}) + return + } + now := s.now() + snapshot := Snapshot{ + Available: true, + SampledAt: now, + System: s.sampleSystem(ctx, now), + GPU: s.sampleGPU(ctx, now), + } + s.snapshot.Store(&snapshot) +} + +func (s *Sampler) sampleSystem(ctx context.Context, now time.Time) *SystemStats { + cpuPct, cores := s.cpuStats(now) + + net := readNetCounters(s.procDir, now) + rxBps, txBps, _ := netThroughputBps(s.prevNet, net) + if net.valid { + s.prevNet = net + } + + usedBytes, totalBytes := s.memoryStats() + + paths := s.diskPaths(ctx) + s.refreshDisks(paths, now) + + return &SystemStats{ + CPUPct: cpuPct, + Load1: readLoad1(s.procDirFor("loadavg")), + Cores: cores, + MemUsedMB: bytesToMB(usedBytes), + MemTotalMB: bytesToMB(totalBytes), + Disks: s.diskStats(paths, now), + NetRxBps: rxBps, + NetTxBps: txBps, + } +} + +// cpuStats reports busy percentage and the core count that percentage is +// normalized against. +// +// Under a cgroup that caps CPU both come from the cgroup, not the host: +// /proc/stat is host-wide even in a container, so a node capped at two cores of +// a busy 64-core machine would otherwise report the machine's load instead of +// its own — and a node pinned at its quota, which is the state worth alerting +// on, would look nearly idle. Where no cap is in force the host figure stands, +// because then the machine's load is the load this node competes with. +func (s *Sampler) cpuStats(now time.Time) (busyPct, cores int) { + host, hostCores := readCPUTimes(s.procDirFor("stat")) + busyPct, _ = cpuBusyPercent(s.prevCPU, host) + if host.valid { + s.prevCPU = host + } + cores = hostCores + if cores == 0 { + cores = runtime.NumCPU() + } + + // A cpuset caps CPU without setting a quota, so a process pinned to two CPUs + // reads cpu.max = "max" and would otherwise be measured against the whole + // host. Whichever cap is tighter is the one it actually runs under, and + // cgroupCPU decides that, because the answer also settles which cgroup's + // usage the reading comes from. + sample, quota := s.cgroupCPU(now, cgroupCPUSetCores(s.cgroupCPUSetPaths, hostCores)) + if !cgroupCapBinds(quota, hostCores) { + // Nothing caps this process, so its own cgroup is the wrong domain to + // measure: the leaf accounts for Silo alone, and Silo idling beside a + // saturated neighbor is not an idle machine. Uncapped, the CPU a + // neighbor burns is CPU this node cannot have, which is exactly what + // /proc/stat reports — and under lxcfs that file is already narrowed to + // the container, so this stays right there too. The next pass to find a + // cap starts a fresh pair rather than differencing across the switch. + // + // A quota the size of the machine counts as nothing capping it, for the + // same reason a cpuset spanning the machine does: it cannot restrict this + // process beyond what the machine already does, and treating it as a cap + // moves the measurement to a domain that answers a different question. + s.prevCgroupCPU = cgroupCPUSample{} + return busyPct, cores + } + cores = cgroupQuotaCores(quota, hostCores) + if !sample.valid { + return busyPct, cores + } + // No clamp against the host's capacity here: a quota that reached this far + // is one cgroupCapBinds accepted, which means it is smaller than the machine. + // A quota the machine cannot supply took the branch above and is reported + // from /proc/stat instead, which is the same answer without dividing one + // population's CPU time by another's capacity. + // + // Once the cgroup can be read it is the only honest source, so its answer + // stands even when this pass cannot derive one (the first sample, or a + // counter reset). Falling back to the host figure would silently mix two + // different machines' busyness across intervals. + cgroupPct, _ := cgroupCPUPercent(s.prevCgroupCPU, sample, quota) + s.prevCgroupCPU = sample + return cgroupPct, cores +} + +// diskPaths lists the mounts to sample, scratch dir first, bounded at +// maxSampledDisks. +// +// The bound is on what is *probed*, not only on what is published, because a +// probe is not free and cannot be taken back. Each path gets its own statfs +// goroutine every interval, and statfs on a dead network mount is +// uninterruptible — the goroutine parks until the mount recovers or the process +// exits. A deployment with forty library roots would otherwise start forty +// probes every five seconds to fill eight slots, and every unreachable root +// would leave a goroutine parked forever. Capping the input makes the worst +// case a fixed eight parked goroutines regardless of library count. +// +// The scratch dir is always first and so is never the entry dropped: it is the +// one mount admission control reads. +func (s *Sampler) diskPaths(ctx context.Context) []string { + s.scratchDir = "" + if s.scratchDirFn != nil { + s.scratchDir = strings.TrimSpace(s.scratchDirFn()) + } + var paths []string + if s.scratchDir != "" { + paths = append(paths, s.scratchDir) + } + if s.mediaRoots != nil { + for _, root := range s.mediaRoots(ctx) { + if root != "" && !slices.Contains(paths, root) { + paths = append(paths, root) + } + } + } + if len(paths) <= maxSampledDisks { + s.noteDroppedRoots(0) + return paths + } + s.noteDroppedRoots(len(paths) - maxSampledDisks) + return paths[:maxSampledDisks] +} + +// noteDroppedRoots reports that the disk sample does not cover every configured +// media root, so the omission is a log line rather than a silent truncation an +// operator would read as "every mount is fine". +// +// Logged on transitions only: the sampling loop runs every few seconds, and a +// library count is a standing property, not an event. Called from the sampling +// goroutine, which owns this field. +func (s *Sampler) noteDroppedRoots(dropped int) { + if dropped == s.droppedRoots { + return + } + s.droppedRoots = dropped + if dropped == 0 { + slog.Info("node metrics disk sampling now covers every configured root", "component", "nodemetrics") + return + } + slog.Info("node metrics disk sampling is capped; the roots past the cap are not reported", + "component", "nodemetrics", "sampled", maxSampledDisks, "not_sampled", dropped) +} + +// sampleGPU merges three independent views of the host's GPUs: the hardware +// inventory (which devices exist), DRM fdinfo (what our own transcodes are +// doing on them), and nvidia-smi (what everyone is doing on an NVIDIA GPU). +// +// Devices are keyed by normalized PCI address wherever one is known, because +// that is the only identifier all three views share — /dev/dri paths are an +// enumeration order and CUDA indices are an nvidia-smi ordering. +func (s *Sampler) sampleGPU(ctx context.Context, now time.Time) []GPUStats { + sessions := map[string]int{} + if s.sessions != nil { + sessions = s.sessions() + } + + byKey := map[string]*GPUStats{} + order := []string{} + // aliases holds every name a device answers to, because the workload counter + // is keyed by whatever playback was configured with (a render path, a CUDA + // index, a GPU UUID) while entries here are keyed by PCI address. + aliases := map[string][]string{} + upsert := func(key string) *GPUStats { + if existing, ok := byKey[key]; ok { + return existing + } + entry := &GPUStats{Device: key, Source: SourceUnavailable} + byKey[key] = entry + order = append(order, key) + aliases[key] = append(aliases[key], key) + return entry + } + alias := func(key string, names ...string) { + for _, name := range names { + if name != "" && !slices.Contains(aliases[key], name) { + aliases[key] = append(aliases[key], name) + } + } + } + + // 1. Known hardware, so a device with no activity still appears. + pathByPCI := map[string]string{} + for _, identity := range s.identityList() { + key := NormalizePCIAddress(identity.PCIAddress) + if key == "" { + key = identity.Path + } + if key == "" { + continue + } + if identity.PCIAddress != "" && identity.Path != "" { + pathByPCI[NormalizePCIAddress(identity.PCIAddress)] = identity.Path + } + entry := upsert(key) + if identity.Path != "" { + entry.Device = identity.Path + } + alias(key, identity.Path) + entry.Vendor = identity.Vendor + } + + // 2. DRM fdinfo deltas for our own ffmpeg children. + clients := readFdinfoCounters(s.procDir, s.ffmpegPIDs()) + elapsedNS := int64(0) + if !s.prevGPUAt.IsZero() { + elapsedNS = now.Sub(s.prevGPUAt).Nanoseconds() + } + for pdev, delta := range deviceEngineDeltas(s.prevGPU, clients) { + entry := upsert(pdev) + if path, ok := pathByPCI[pdev]; ok { + entry.Device = path + alias(pdev, path) + } + if elapsedNS > 0 { + // Before the second sample there is no interval to divide by, so the + // engines stay unset rather than reporting a freshly started node's + // unknown load as idle. + entry.VideoBusyPct = ptr(engineBusyPercent(delta.videoNS, elapsedNS)) + entry.RenderBusyPct = ptr(engineBusyPercent(delta.renderNS, elapsedNS)) + } + entry.Source = SourceFdinfo + } + // Clients that vanished (their transcode exited) drop out of the baseline + // with their counters, so the next client on that device is not measured + // against a stale origin. + s.prevGPU = clients + s.prevGPUAt = now + + // 3. nvidia-smi enrichment: the only signal for NVIDIA, and whole-GPU + // (other tenants included) where it applies. + for _, gpu := range s.queryNVIDIA(ctx) { + cudaName := "cuda:" + strconv.Itoa(gpu.Index) + key := gpu.PCIAddress + if key == "" { + key = cudaName + } + entry := upsert(key) + if entry.Device == key && entry.Vendor == "" { + // No DRM node for it (the proprietary driver exposes none this + // process can read), so name it the way playback addresses it. + entry.Device = cudaName + } + // NVENC workloads are counted under a CUDA name or a GPU UUID even when + // the device is displayed by its render path, so both have to resolve to + // this entry. + alias(key, cudaName, gpu.UUID) + entry.Vendor = vendorNVIDIA + // Each column is carried only when the driver actually reported it: a + // successful row can still answer "[N/A]" for engines or memory it + // cannot see, and publishing those as zero would show an unobservable + // video engine as idle and unsupported VRAM as 0 bytes. + entry.TotalBusyPct = gpu.GPUUtil + entry.VRAMUsedMB = gpu.MemUsedMB + entry.VRAMTotalMB = gpu.MemTotalMB + if entry.Source == SourceFdinfo { + entry.Source = SourceFdinfoNVIDIASMI + } else { + entry.Source = SourceNVIDIASMI + // fdinfo is unimplemented by the proprietary driver, so the video + // engines only have an nvidia-smi reading — which stays unset when + // the driver answered "[N/A]" for both of them. + entry.VideoBusyPct = gpu.videoUtil() + } + } + + // 4. Workloads this process knows it started, on devices nothing above + // named. Session accounting comes from the playback allocator rather than + // from a driver, so it is true whether or not any measurement source + // answered — and NVENC workloads are keyed by CUDA index or GPU uuid, which + // only the nvidia-smi step supplies. With that step failed or its breaker + // tripped, an NVENC node would otherwise report zero sessions while it + // transcodes, and one exposing no readable render node would vanish from the + // sample entirely. Nothing here can say *which* card a bare "cuda:0" is + // without nvidia-smi, so it stands as its own unmeasured entry: an honest + // "a workload is running on this device and nobody could measure it" beats a + // silent zero. + for name, count := range sessions { + if name == "" || count <= 0 { + continue + } + if slices.ContainsFunc(order, func(key string) bool { return slices.Contains(aliases[key], name) }) { + continue + } + upsert(name) + } + + if len(order) == 0 { + return nil + } + claimed := map[string]bool{} + out := make([]GPUStats, 0, len(order)) + for _, key := range order { + entry := byKey[key] + entry.Sessions = deviceSessions(sessions, aliases[key], claimed) + out = append(out, *entry) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Device < out[j].Device }) + return out +} + +// deviceSessions totals the balancer's workload counts for one device. +// +// The balancer counts a workload under whatever playback was configured with, +// which is not always the name this package displays: an NVENC job on a card +// that also has a readable render node is counted as "cuda:0" while the entry +// is shown as /dev/dri/renderD128. Looking the count up by display name alone +// reports 0 sessions on a node that is transcoding. Every alias is summed once +// — claimed keeps a name shared by two entries from being counted twice. +// The display name is always among the aliases: it is only ever set to one. +func deviceSessions(sessions map[string]int, aliases []string, claimed map[string]bool) int { + total := 0 + for _, name := range aliases { + if name == "" || claimed[name] { + continue + } + claimed[name] = true + total += sessions[name] + } + return total +} + +func (s *Sampler) identityList() []DeviceIdentity { + if s.identities == nil { + return nil + } + return s.identities() +} diff --git a/internal/nodemetrics/sampler_test.go b/internal/nodemetrics/sampler_test.go new file mode 100644 index 000000000..b49d16f1a --- /dev/null +++ b/internal/nodemetrics/sampler_test.go @@ -0,0 +1,1320 @@ +package nodemetrics + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// fakeClock advances only when a test says so, so every rate in this package is +// computed against an interval the test chose rather than against however long +// the test machine happened to take. +type fakeClock struct{ at time.Time } + +func newFakeClock() *fakeClock { + return &fakeClock{at: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} +} + +func (c *fakeClock) now() time.Time { return c.at } + +func (c *fakeClock) advance(d time.Duration) { c.at = c.at.Add(d) } + +// procTree builds a fake /proc under t.TempDir and returns its path. +type procTree struct { + t *testing.T + root string +} + +func newProcTree(t *testing.T) *procTree { + t.Helper() + return &procTree{t: t, root: t.TempDir()} +} + +// write creates a file relative to the tree root, making parents as needed. +func (p *procTree) write(relPath, content string) { + p.t.Helper() + full := filepath.Join(p.root, relPath) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + p.t.Fatalf("mkdir %s: %v", full, err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + p.t.Fatalf("write %s: %v", full, err) + } +} + +// newTestSampler wires a sampler onto a fake /proc with a fake clock and no +// real subprocess, statfs or hardware access. +func newTestSampler(t *testing.T, tree *procTree, clock *fakeClock, opts Options) *Sampler { + t.Helper() + if opts.Now == nil { + opts.Now = clock.now + } + if opts.FFmpegChildren == nil { + opts.FFmpegChildren = func() []int { return nil } + } + s := NewSampler(opts) + s.goos = "linux" + s.procDir = tree.root + // Point hostProcDir at a path that does not exist by default so tests are + // deterministic regardless of whether the machine running them happens to + // have a real /host/proc; tests that exercise the lxcfs override set this + // explicitly to a tree that does. + s.hostProcDir = filepath.Join(tree.root, "no-such-host-proc") + // Every cgroup source is cleared, not just the ones a given test sets: these + // default to real host paths, and on Linux a test that forgot one reads the + // machine it is running on instead of its fixture. + s.cgroupUsagePaths = nil + s.cgroupCPUPaths = nil + s.cgroupCPUSetPaths = nil + s.statfs = func(string) (fsStats, error) { return fsStats{}, os.ErrNotExist } + s.runNVIDIASMI = func(context.Context) ([]byte, error) { return nil, os.ErrNotExist } + return s +} + +func TestCPUBusyPercentFromProcStatDeltas(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + // user nice system idle iowait irq softirq steal + tree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + tree.write("loadavg", "3.20 1.10 0.90 2/512 9\n") + tree.write("meminfo", "MemTotal: 1024 kB\nMemAvailable: 512 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("first sample CPUPct = %d, want 0 (no previous reading to diff against)", got) + } + + // 400 jiffies of work out of 1000 elapsed. + tree.write("stat", "cpu 300 0 300 1400 0 0 0 0\ncpu0 150 0 150 700 0 0 0 0\ncpu1 150 0 150 700 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 40 { + t.Fatalf("CPUPct = %d, want 40", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2", system.Cores) + } + if system.Load1 != 3.20 { + t.Fatalf("Load1 = %v, want 3.2", system.Load1) + } +} + +// A counter that went backwards means the two readings do not describe one +// continuous run. Reporting the difference would produce a nonsense percentage. +func TestCPUBusyPercentClampsCounterReset(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 1000 0 1000 8000 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + tree.write("stat", "cpu 10 0 10 80 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("CPUPct after counter reset = %d, want 0", got) + } + + // The reset reading becomes the new baseline, so the next interval is + // measured normally rather than against the pre-reset counter. + tree.write("stat", "cpu 60 0 60 180 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 50 { + t.Fatalf("CPUPct after re-baselining = %d, want 50", got) + } +} + +func TestNetworkThroughputExcludesLoopbackAndClampsResets(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + netDev := func(loRx, eth0Rx, eth0Tx int) string { + return "Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + + " lo: " + itoa(loRx) + " 1 0 0 0 0 0 0 " + itoa(loRx) + " 1 0 0 0 0 0 0\n" + + " eth0: " + itoa(eth0Rx) + " 1 0 0 0 0 0 0 " + itoa(eth0Tx) + " 1 0 0 0 0 0 0\n" + } + tree.write("net/dev", netDev(1_000_000, 1000, 2000)) + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + // eth0 gains 5000 rx and 10000 tx bytes over 5s; loopback gains a lot and + // must not appear. + tree.write("net/dev", netDev(99_000_000, 6000, 12000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.NetRxBps != 8000 { // 5000 bytes / 5s * 8 + t.Fatalf("NetRxBps = %d, want 8000", system.NetRxBps) + } + if system.NetTxBps != 16000 { // 10000 bytes / 5s * 8 + t.Fatalf("NetTxBps = %d, want 16000", system.NetTxBps) + } + + // An interface restarting resets the aggregate; that must read as zero, not + // as a negative or a wrapped spike. + tree.write("net/dev", netDev(99_000_000, 10, 20)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + system = s.Snapshot().System + if system.NetRxBps != 0 || system.NetTxBps != 0 { + t.Fatalf("throughput after counter reset = %d/%d, want 0/0", system.NetRxBps, system.NetTxBps) + } +} + +// An interface entering the namespace mid-run — a veth or tunnel moved into a +// running container — arrives carrying its lifetime totals. With an aggregate +// baseline those totals would read as one interval's traffic; per-interface +// pairing gives the newcomer this reading as its baseline and counts it from +// the next interval, while an interface leaving takes only its own baseline +// with it instead of masking the survivors behind a shrunken aggregate. +func TestNetworkThroughputSurvivesInterfaceChurn(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + header := "Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + iface := func(name string, rx, tx int) string { + return " " + name + ": " + itoa(rx) + " 1 0 0 0 0 0 0 " + itoa(tx) + " 1 0 0 0 0 0 0\n" + } + + tree.write("net/dev", header+iface("eth0", 1000, 2000)) + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + // tailscale0 appears between readings carrying 40 GB of lifetime history; + // eth0 moves by its usual 5000/10000. Only eth0's delta may appear. + tree.write("net/dev", header+iface("eth0", 6000, 12000)+iface("tailscale0", 40_000_000_000, 40_000_000_000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + system := s.Snapshot().System + if system.NetRxBps != 8000 || system.NetTxBps != 16000 { + t.Fatalf("throughput with a newborn interface = %d/%d, want 8000/16000", system.NetRxBps, system.NetTxBps) + } + + // From its second reading the newcomer has a baseline and counts. + tree.write("net/dev", header+iface("eth0", 11000, 22000)+iface("tailscale0", 40_000_005_000, 40_000_010_000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + system = s.Snapshot().System + if system.NetRxBps != 16000 || system.NetTxBps != 32000 { + t.Fatalf("throughput once the newcomer has a baseline = %d/%d, want 16000/32000", system.NetRxBps, system.NetTxBps) + } + + // The newcomer vanishes again. Its departure must not hide eth0's real + // traffic, even though the aggregate across interfaces plummeted. + tree.write("net/dev", header+iface("eth0", 16000, 32000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + system = s.Snapshot().System + if system.NetRxBps != 8000 || system.NetTxBps != 16000 { + t.Fatalf("throughput after an interface left = %d/%d, want 8000/16000", system.NetRxBps, system.NetTxBps) + } +} + +func TestMemoryFromMeminfo(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("net/dev", "") + // 8 GiB total, 6 GiB available. + tree.write("meminfo", "MemTotal: 8388608 kB\nMemFree: 512000 kB\nMemAvailable: 6291456 kB\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != 8192 { + t.Fatalf("MemTotalMB = %d, want 8192", system.MemTotalMB) + } + if system.MemUsedMB != 2048 { + t.Fatalf("MemUsedMB = %d, want 2048", system.MemUsedMB) + } +} + +// In a container /proc/meminfo describes the host. The cgroup limit is what the +// kernel will actually OOM-kill against, so it has to win. +func TestMemoryCorrectedByCgroupLimitAndUsage(t *testing.T) { + for _, tc := range []struct { + name string + limitFile string + limitBody string + usageFile string + statFile string + statBody string + inactiveKey string + wantTotalMB int64 + wantUsedMB int64 + usageBody string + }{ + { + name: "cgroup v2", + limitFile: "memory.max", + limitBody: "2147483648\n", // 2 GiB + usageFile: "memory.current", + usageBody: "1073741824\n", // 1 GiB charged + statFile: "memory.stat", + statBody: "anon 536870912\ninactive_file 536870912\n", + inactiveKey: "inactive_file", + wantTotalMB: 2048, + wantUsedMB: 512, // page cache does not count toward the working set + }, + { + name: "cgroup v1", + limitFile: "memory.limit_in_bytes", + limitBody: "4294967296\n", // 4 GiB + usageFile: "memory.usage_in_bytes", + usageBody: "2147483648\n", + statFile: "memory.stat", + statBody: "total_inactive_file 1073741824\n", + inactiveKey: cgroupInactiveFileKeyV1, + wantTotalMB: 4096, + wantUsedMB: 1024, + }, + { + name: "no concrete limit falls back to the host", + limitFile: "memory.max", + limitBody: "max\n", + usageFile: "memory.current", + usageBody: "", + statFile: "memory.stat", + statBody: "", + inactiveKey: "inactive_file", + wantTotalMB: 8192, + wantUsedMB: 2048, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("net/dev", "") + tree.write("meminfo", "MemTotal: 8388608 kB\nMemAvailable: 6291456 kB\n") + + cgroupDir := t.TempDir() + if err := os.WriteFile(filepath.Join(cgroupDir, tc.limitFile), []byte(tc.limitBody), 0o644); err != nil { + t.Fatal(err) + } + if tc.usageBody != "" { + if err := os.WriteFile(filepath.Join(cgroupDir, tc.usageFile), []byte(tc.usageBody), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cgroupDir, tc.statFile), []byte(tc.statBody), 0o644); err != nil { + t.Fatal(err) + } + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupUsagePaths = []cgroupUsagePath{{ + limit: filepath.Join(cgroupDir, tc.limitFile), + usage: filepath.Join(cgroupDir, tc.usageFile), + stat: filepath.Join(cgroupDir, tc.statFile), + inactiveFile: tc.inactiveKey, + }} + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != tc.wantTotalMB { + t.Fatalf("MemTotalMB = %d, want %d", system.MemTotalMB, tc.wantTotalMB) + } + if system.MemUsedMB != tc.wantUsedMB { + t.Fatalf("MemUsedMB = %d, want %d", system.MemUsedMB, tc.wantUsedMB) + } + }) + } +} + +// In a container /proc/stat describes the host, exactly as /proc/meminfo does. +// A node capped at two cores of a mostly idle 8-core host and pinned at its +// quota is the state worth alerting on, and reading the host would report it as +// nearly idle. +func TestCPUCorrectedByCgroupQuotaAndUsage(t *testing.T) { + for _, tc := range []struct { + name string + paths cgroupCPUPath + quotaFile string + quotaBody string + usageFile string + usageFirst string + usageLater string + periodFile string + periodBody string + }{ + { + name: "cgroup v2", + paths: cgroupCPUPath{usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond}, + usageFile: "cpu.stat", + usageFirst: "usage_usec 1000000\nuser_usec 500000\n", + usageLater: "usage_usec 11000000\nuser_usec 5000000\n", // +10s of CPU + quotaFile: "cpu.max", + quotaBody: "200000 100000\n", // 2 cores + }, + { + name: "cgroup v1", + paths: cgroupCPUPath{usageUnit: time.Nanosecond}, + usageFile: "cpuacct.usage", + usageFirst: "1000000000\n", + usageLater: "11000000000\n", // +10s of CPU + quotaFile: "cpu.cfs_quota_us", + quotaBody: "200000\n", + periodFile: "cpu.cfs_period_us", + periodBody: "100000\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + // The host has 8 cores and is barely busy; the container is not. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 9900)) + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + cgroupDir := t.TempDir() + writeCgroup := func(name, body string) { + if err := os.WriteFile(filepath.Join(cgroupDir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + writeCgroup(tc.usageFile, tc.usageFirst) + writeCgroup(tc.quotaFile, tc.quotaBody) + paths := tc.paths + paths.usage = filepath.Join(cgroupDir, tc.usageFile) + paths.quota = filepath.Join(cgroupDir, tc.quotaFile) + if tc.periodFile != "" { + writeCgroup(tc.periodFile, tc.periodBody) + paths.period = filepath.Join(cgroupDir, tc.periodFile) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{paths} + s.sample(context.Background()) + if got := s.Snapshot().System.CPUPct; got != 0 { + t.Fatalf("first sample CPUPct = %d, want 0 (nothing to diff against)", got) + } + + // 10 seconds of CPU over 5 seconds of wall time on a 2-core quota: + // the container is pegged, while the host is 1% busy. + writeCgroup(tc.usageFile, tc.usageLater) + tree.write("stat", hostStat(200, 19800)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 100 { + t.Fatalf("CPUPct = %d, want 100 (the container's own usage against its own quota)", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the 2 cores the cgroup allows, not the host's 8", system.Cores) + } + }) + } +} + +// An unconstrained cgroup measures this process alone, which is not what a node +// competes for: uncapped, the CPU a neighbor burns is CPU this node cannot have. +// So with no quota the host's own busyness stands, even though the cgroup could +// be read — here Silo uses a quarter of the machine while the machine is at +// three quarters. +func TestCPUWithoutCgroupQuotaReportsTheHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 100 0 0 900 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + + cgroupDir := t.TempDir() + usage := filepath.Join(cgroupDir, "cpu.stat") + if err := os.WriteFile(usage, []byte("usage_usec 0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cgroupDir, "cpu.max"), []byte("max 100000\n"), 0o644); err != nil { + t.Fatal(err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, + usageKey: cgroupCPUUsageKey, + usageUnit: time.Microsecond, + quota: filepath.Join(cgroupDir, "cpu.max"), + }} + s.sample(context.Background()) + + // 5 seconds of CPU over 5 seconds of wall time across 2 cores: half of what + // this process could use, and a quarter of the two-core machine. + if err := os.WriteFile(usage, []byte("usage_usec 5000000\n"), 0o644); err != nil { + t.Fatal(err) + } + // The host meanwhile spent 750 of the interval's 1000 ticks busy. + tree.write("stat", "cpu 850 0 0 1150 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\n") + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 rather than this process's 25", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the host's 2 with no quota set", system.Cores) + } +} + +// A host that cannot be sampled must publish an explicitly unavailable +// snapshot, so a reader can distinguish it from a host that is simply idle. +func TestNonLinuxHostReportsUnavailable(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + s := newTestSampler(t, tree, clock, Options{}) + s.goos = "darwin" + s.sample(context.Background()) + + snapshot := s.Snapshot() + if snapshot.Available { + t.Fatal("Available = true on a non-Linux host") + } + if snapshot.System != nil || snapshot.GPU != nil { + t.Fatalf("snapshot carries data on a non-Linux host: %+v", snapshot) + } +} + +// The exact JSON is a wire contract: node health responses, the admin resources +// endpoint, and the persisted last_stats column all carry this shape. +func TestSnapshotJSONShape(t *testing.T) { + total := 71 + video, render := 63, 12 + vramUsed := int64(812) + vramTotal := int64(8192) + snapshot := Snapshot{ + Available: true, + System: &SystemStats{ + CPUPct: 41, + Load1: 3.2, + Cores: 16, + MemUsedMB: 9011, + MemTotalMB: 32768, + Disks: []DiskStats{ + {Path: "/transcode", UsedGB: 210, TotalGB: 500}, + {Path: "/media", UsedGB: 7100, TotalGB: 8000, Stale: true}, + {Path: "/gone", Unavailable: true}, + }, + NetRxBps: 1200000, + NetTxBps: 98000000, + }, + GPU: []GPUStats{{ + Device: "/dev/dri/renderD128", + Vendor: "intel", + Sessions: 2, + VideoBusyPct: &video, + RenderBusyPct: &render, + TotalBusyPct: &total, + VRAMUsedMB: &vramUsed, + VRAMTotalMB: &vramTotal, + Source: SourceFdinfoNVIDIASMI, + }}, + } + + encoded, err := json.Marshal(snapshot) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatal(err) + } + + system, ok := decoded["system"].(map[string]any) + if !ok { + t.Fatalf("system missing from %s", encoded) + } + for _, key := range []string{"cpu_pct", "load1", "cores", "mem_used_mb", "mem_total_mb", "disks", "net_rx_bps", "net_tx_bps"} { + if _, ok := system[key]; !ok { + t.Fatalf("system.%s missing from %s", key, encoded) + } + } + disks := system["disks"].([]any) + if _, ok := disks[0].(map[string]any)["stale"]; ok { + t.Fatalf("stale emitted for a fresh disk: %s", encoded) + } + if stale, _ := disks[1].(map[string]any)["stale"].(bool); !stale { + t.Fatalf("stale missing for a stale disk: %s", encoded) + } + if unavailable, _ := disks[2].(map[string]any)["unavailable"].(bool); !unavailable { + t.Fatalf("unavailable missing: %s", encoded) + } + + gpu := decoded["gpu"].([]any)[0].(map[string]any) + for _, key := range []string{"device", "vendor", "sessions", "video_busy_pct", "render_busy_pct", "total_busy_pct", "vram_used_mb", "vram_total_mb", "source"} { + if _, ok := gpu[key]; !ok { + t.Fatalf("gpu.%s missing from %s", key, encoded) + } + } + + // Absent enrichment must be absent, not zero: an operator reading + // total_busy_pct: 0 would conclude the GPU is idle when nothing measured it. + bare, err := json.Marshal(Snapshot{Available: true, GPU: []GPUStats{{Device: "/dev/dri/renderD128", Source: SourceFdinfo}}}) + if err != nil { + t.Fatal(err) + } + var bareDecoded map[string]any + if err := json.Unmarshal(bare, &bareDecoded); err != nil { + t.Fatal(err) + } + if _, ok := bareDecoded["system"]; ok { + t.Fatalf("system emitted when absent: %s", bare) + } + bareGPU := bareDecoded["gpu"].([]any)[0].(map[string]any) + for _, key := range []string{"video_busy_pct", "render_busy_pct", "total_busy_pct", "vram_used_mb", "vram_total_mb", "vendor"} { + if _, ok := bareGPU[key]; ok { + t.Fatalf("gpu.%s emitted when absent: %s", key, bare) + } + } +} + +// On an LXC host running Docker nested inside it, this process's own /proc +// sees the raw kernel — the bare-metal core count, memory, and load — while +// its own cgroup is unlimited, because the LXC's cap lives on an ancestor +// cgroup outside its namespace. lxcfs virtualizes /proc/stat, /proc/loadavg, +// and /proc/meminfo to the LXC's own limits, so when those are bind-mounted +// in at hostProcDir, they must win over the container's raw /proc for exactly +// those three files. +func TestHostProcOverridesLxcfsScopedFiles(t *testing.T) { + tree := newProcTree(t) + hostTree := newProcTree(t) + clock := newFakeClock() + + // The nested container's own /proc: a busy 8-core bare-metal host, 64 GiB + // of memory, and a load average that belongs to every other tenant on the + // box too. + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n"+ + "cpu0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0\n"+ + "cpu4 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "40.00 38.00 35.00 12/900 5555\n") + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + // lxcfs's view, bind-mounted at hostProcDir: the LXC is capped at 2 cores + // and 2 GiB, and its own load average. + hostTree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + hostTree.write("loadavg", "1.50 1.20 0.80 1/64 42\n") + hostTree.write("meminfo", "MemTotal: 2097152 kB\nMemAvailable: 1048576 kB\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.hostProcDir = hostTree.root + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the lxcfs-scoped 2, not the host's 8", system.Cores) + } + if system.Load1 != 1.50 { + t.Fatalf("Load1 = %v, want the lxcfs-scoped 1.50, not the raw host's 40.00", system.Load1) + } + if system.MemTotalMB != 2048 { + t.Fatalf("MemTotalMB = %d, want the lxcfs-scoped 2048, not the raw host's 65536", system.MemTotalMB) + } + if system.MemUsedMB != 1024 { + t.Fatalf("MemUsedMB = %d, want 1024 (2048 total - 1024 available)", system.MemUsedMB) + } + + // A second sample drives the CPU busy-percent delta off the lxcfs stat + // file too: 400 more busy jiffies out of 1000 more total. + hostTree.write("stat", "cpu 300 0 300 1400 0 0 0 0\ncpu0 150 0 150 700 0 0 0 0\ncpu1 150 0 150 700 0 0 0 0\n") + tree.write("stat", "cpu 99999 0 0 1 0 0 0 0\n") // the raw host is pegged; must not be read + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system = s.Snapshot().System + if system.CPUPct != 40 { + t.Fatalf("CPUPct = %d, want 40 from the lxcfs stat deltas, not the raw host's near-100", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2 on the second sample too", system.Cores) + } +} + +// Without a bind-mounted lxcfs view — plain Docker, bare metal, or an LXC +// deployment that has not mounted /host/proc — every reading must fall back +// to the container's own /proc exactly as before this feature existed. +func TestHostProcAbsentFallsBackToProcDir(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 100 0 100 800 0 0 0 0\ncpu0 50 0 50 400 0 0 0 0\ncpu1 50 0 50 400 0 0 0 0\n") + tree.write("loadavg", "3.20 1.10 0.90 2/512 9\n") + tree.write("meminfo", "MemTotal: 8388608 kB\nMemAvailable: 6291456 kB\n") + tree.write("net/dev", "Inter-|\n face |\n") + + // newTestSampler already points hostProcDir at a nonexistent path; this + // test just makes that fallback explicit and asserts on it. + s := newTestSampler(t, tree, clock, Options{}) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 2 { + t.Fatalf("Cores = %d, want 2 from procDir", system.Cores) + } + if system.Load1 != 3.20 { + t.Fatalf("Load1 = %v, want 3.2 from procDir", system.Load1) + } + if system.MemTotalMB != 8192 { + t.Fatalf("MemTotalMB = %d, want 8192 from procDir", system.MemTotalMB) + } + if system.MemUsedMB != 2048 { + t.Fatalf("MemUsedMB = %d, want 2048 from procDir", system.MemUsedMB) + } +} + +// Only stat, loadavg, and meminfo are host-proc aware. net/dev is per-netns +// and must stay on the container's own /proc even when hostProcDir has its +// own net/dev sitting right next to the other three files — otherwise a +// nested node would report the LXC host's aggregate network traffic instead +// of its own. +func TestHostProcDoesNotAffectNetDev(t *testing.T) { + tree := newProcTree(t) + hostTree := newProcTree(t) + clock := newFakeClock() + + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + netDev := func(rx, tx int) string { + return "Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + + " eth0: " + itoa(rx) + " 1 0 0 0 0 0 0 " + itoa(tx) + " 1 0 0 0 0 0 0\n" + } + tree.write("net/dev", netDev(1000, 2000)) + + // hostTree has its own net/dev with very different counters. If it were + // ever consulted, the throughput computed below would not match. + hostTree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + hostTree.write("loadavg", "0 0 0 0/0 0\n") + hostTree.write("meminfo", "MemTotal: 1024 kB\n") + hostTree.write("net/dev", netDev(9_000_000, 9_000_000)) + + s := newTestSampler(t, tree, clock, Options{}) + s.hostProcDir = hostTree.root + s.sample(context.Background()) + + tree.write("net/dev", netDev(6000, 12000)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.NetRxBps != 8000 { // 5000 bytes / 5s * 8, from tree's net/dev + t.Fatalf("NetRxBps = %d, want 8000 from procDir's net/dev, not hostProcDir's", system.NetRxBps) + } + if system.NetTxBps != 16000 { // 10000 bytes / 5s * 8 + t.Fatalf("NetTxBps = %d, want 16000 from procDir's net/dev, not hostProcDir's", system.NetTxBps) + } +} + +// itoa avoids importing strconv into every fixture builder above. +func itoa(v int) string { + if v == 0 { + return "0" + } + digits := "" + for v > 0 { + digits = string(rune('0'+v%10)) + digits + v /= 10 + } + return digits +} + +// A container with no memory limit still publishes a readable memory.current. +// Taking it unconditionally paired this process's working set with the host's +// RAM — "1 GiB of 64 GiB" on a machine that is nearly out of memory, because +// the two numbers come from different domains. +func TestMemoryStatsDoesNotMixCgroupUsageWithHostTotal(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 65536 kB\nMemAvailable: 1024 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + // No limit file resolves, which is what an unconstrained container looks + // like, but the usage file is readable. + usage := filepath.Join(t.TempDir(), "memory.current") + if err := os.WriteFile(usage, []byte("1048576\n"), 0o600); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + s.cgroupUsagePaths = []cgroupUsagePath{{limit: filepath.Join(t.TempDir(), "absent"), usage: usage}} + + used, total := s.memoryStats() + if total != 65536*1024 { + t.Fatalf("total = %d, want the host's MemTotal", total) + } + // MemTotal - MemAvailable, from the same file as total. + if want := int64(64512 * 1024); used != want { + t.Fatalf("used = %d, want the host's used figure %d rather than the cgroup working set", used, want) + } +} + +// With a concrete limit both numbers come from the cgroup, which is the whole +// point of the correction. +func TestMemoryStatsUsesCgroupUsageBesideACgroupLimit(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 65536 kB\nMemAvailable: 1024 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + dir := t.TempDir() + limit := filepath.Join(dir, "memory.max") + if err := os.WriteFile(limit, []byte("8388608\n"), 0o600); err != nil { + t.Fatalf("write cgroup limit: %v", err) + } + usage := filepath.Join(dir, "memory.current") + if err := os.WriteFile(usage, []byte("1048576\n"), 0o600); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + s.cgroupUsagePaths = []cgroupUsagePath{{limit: limit, usage: usage}} + + used, total := s.memoryStats() + if total != 8388608 { + t.Fatalf("total = %d, want the cgroup limit", total) + } + if used != 1048576 { + t.Fatalf("used = %d, want the cgroup working set", used) + } +} + +// NVENC workloads are keyed by CUDA index or GPU uuid, and only the nvidia-smi +// step supplies those aliases. With that step failed — a timeout, a missing +// toolkit, a tripped breaker — an NVENC node reported zero sessions while it +// transcoded, and one exposing no readable render node vanished from the sample +// altogether. Session accounting comes from the playback allocator, not a +// driver, so it is true regardless. +func TestSampleGPUReportsSessionsWhenNVIDIAEnrichmentFails(t *testing.T) { + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return nil, errors.New("nvidia-smi not installed") + } + s.identities = func() []DeviceIdentity { return nil } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + gpus := s.sampleGPU(context.Background(), s.now()) + + if len(gpus) != 1 { + t.Fatalf("gpu sample = %+v, want the known workload's device reported", gpus) + } + if gpus[0].Device != "cuda:0" { + t.Fatalf("device = %q, want the name the workload is counted under", gpus[0].Device) + } + if gpus[0].Sessions != 2 { + t.Fatalf("sessions = %d, want the allocator's count", gpus[0].Sessions) + } + if gpus[0].Source != SourceUnavailable { + t.Fatalf("source = %q, want it reported as unmeasured", gpus[0].Source) + } +} + +// A workload whose device an enrichment source did name is not duplicated: the +// alias set already covers it. +func TestSampleGPUDoesNotDuplicateAnAlreadyNamedDevice(t *testing.T) { + tree := newProcTree(t) + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + s.runNVIDIASMI = func(context.Context) ([]byte, error) { + return []byte("0, GPU-x, 00000000:03:00.0, 71, 63, 12, 812, 8192\n"), nil + } + s.identities = func() []DeviceIdentity { return nil } + s.sessions = func() map[string]int { return map[string]int{"cuda:0": 2} } + + gpus := s.sampleGPU(context.Background(), s.now()) + + if len(gpus) != 1 { + t.Fatalf("gpu sample = %+v, want one entry for the enriched device", gpus) + } + if gpus[0].Sessions != 2 { + t.Fatalf("sessions = %d, want the workload joined onto the enriched entry", gpus[0].Sessions) + } +} + +// A leaf cgroup that imposes no memory limit does not mean the process has +// none: a systemd unit inside a slice with MemoryMax=, or a container under a +// limited pod cgroup, is capped by an ancestor while its own memory.max reads +// "max". Taking the first readable limit would report the host's RAM to a +// process the kernel will OOM-kill at 2 GiB. +func TestMemoryLimitTakesTheTightestCgroupInForce(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("stat", "cpu 0 0 0 0 0 0 0 0\n") + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + tree.write("net/dev", "") + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + // Leaf first, as the resolved list is ordered. The leaf allows 8 GiB, but the + // slice containing it allows only 2 — so the first readable limit is not the + // one the kernel kills against, which is what makes this more than an + // ordering detail. + leaf := write("leaf.max", "8589934592\n") + slice := write("slice.max", "2147483648\n") + root := write("root.max", "68719476736\n") + + // The slice's usage counts every service under it, not just this one: that + // is the population its limit actually bounds. + leafUsage := write("leaf.current", "1073741824\n") + sliceUsage := write("slice.current", "1610612736\n") + rootUsage := write("root.current", "3221225472\n") + stat := write("memory.stat", "inactive_file 0\n") + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupUsagePaths = []cgroupUsagePath{ + {limit: leaf, usage: leafUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: slice, usage: sliceUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: root, usage: rootUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + } + s.sample(context.Background()) + + system := s.Snapshot().System + if system.MemTotalMB != 2048 { + t.Fatalf("MemTotalMB = %d, want the 2048 the slice allows, not the leaf's looser 8192", system.MemTotalMB) + } + // Used comes from the *same* cgroup the limit did. Pairing the slice's + // 2 GiB capacity with this service's own 1 GiB would show half the volume + // free while the sibling services under that slice have filled it. + if system.MemUsedMB != 1536 { + t.Fatalf("MemUsedMB = %d, want the 1536 charged to the slice whose limit binds", system.MemUsedMB) + } + + // The other shape of the same bug: a leaf that imposes nothing at all while + // an ancestor does. + unlimited := write("unlimited.max", "max\n") + s = newTestSampler(t, tree, clock, Options{}) + s.cgroupUsagePaths = []cgroupUsagePath{ + {limit: unlimited, usage: leafUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: slice, usage: sliceUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + {limit: root, usage: rootUsage, stat: stat, inactiveFile: cgroupInactiveFileKeyV2}, + } + s.sample(context.Background()) + if got := s.Snapshot().System.MemTotalMB; got != 2048 { + t.Fatalf("MemTotalMB = %d with an unlimited leaf, want the slice's 2048", got) + } +} + +// A cpuset is the other way a deployment caps CPU, and it leaves cpu.max saying +// "max". A process pinned to two CPUs on a sixty-four core host would otherwise +// divide its own busy time by sixty-four and report three percent while it is +// saturated. +func TestCPUCorrectedByCpusetWithoutAQuota(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // An eight-core host, so the two allowed CPUs are visibly a subset. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 9900)) + + dir := t.TempDir() + cpuset := filepath.Join(dir, "cpuset.cpus.effective") + if err := os.WriteFile(cpuset, []byte("2-3\n"), 0o644); err != nil { + t.Fatalf("write cpuset: %v", err) + } + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + if err := os.WriteFile(quota, []byte("max 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.cgroupCPUSetPaths = []string{cpuset} + s.sample(context.Background()) + + // 10 seconds of CPU over 5 seconds of wall time on two allowed CPUs: pegged, + // while the host is 1% busy. + writeUsage(10_000_000) + tree.write("stat", hostStat(200, 19800)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 100 { + t.Fatalf("CPUPct = %d, want 100 — the process is saturating every CPU it may use", system.CPUPct) + } + if system.Cores != 2 { + t.Fatalf("Cores = %d, want the 2 CPUs the cpuset allows, not the host's 8", system.Cores) + } +} + +func TestCountCPUSetEntries(t *testing.T) { + for _, tc := range []struct { + list string + want int + }{ + {"0-3", 4}, + {"0-3,8,12-13", 7}, + {"5", 1}, + {" 0-1 , 4 \n", 3}, + {"", 0}, + {"garbage", 0}, + // A reversed or negative range describes nothing; counting it would + // invent a budget out of a malformed file. + {"5-2", 0}, + {"-1", 0}, + } { + t.Run(tc.list, func(t *testing.T) { + if got := countCPUSetEntries(tc.list); got != tc.want { + t.Fatalf("countCPUSetEntries(%q) = %d, want %d", tc.list, got, tc.want) + } + }) + } +} + +// Every cgroup source defaults to a real host path, so a fixture that forgets +// one silently measures the machine the tests run on. That is invisible on a +// developer's macOS laptop and decides the result on a Linux CI runner — which +// is exactly how TestCPUWithoutCgroupQuotaUsesHostCores started failing only in +// CI. This asserts the constructor leaves nothing pointing at the system. +func TestNewTestSamplerReadsNoRealCgroupPaths(t *testing.T) { + s := newTestSampler(t, newProcTree(t), newFakeClock(), Options{}) + + var configured []string + for _, level := range s.cgroupUsagePaths { + configured = append(configured, level.limit, level.usage, level.stat) + } + for _, level := range s.cgroupCPUPaths { + configured = append(configured, level.usage, level.quota, level.period) + } + configured = append(configured, s.cgroupCPUSetPaths...) + configured = append(configured, s.procDir, s.hostProcDir) + + for _, path := range configured { + if path == "" { + continue + } + if strings.HasPrefix(path, cgroupMountRoot) || path == "/proc" { + t.Fatalf("test sampler reads the real host path %q", path) + } + } +} + +// Two cgroups publishing the same memory limit are not equivalent: the +// ancestor's is shared with siblings that can fill it, so it is the one whose +// usage says how much is left. Reading the leaf instead shows headroom right up +// until the parent OOMs — the mirror of the CPU quota tie-break. +func TestMemoryLimitPrefersTheOuterCgroupOnATie(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 67108864 kB\nMemAvailable: 33554432 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + const twoGiB = "2147483648\n" + stat := write("memory.stat", "inactive_file 0\n") + s.cgroupUsagePaths = []cgroupUsagePath{ + { + limit: write("leaf.max", twoGiB), usage: write("leaf.current", "536870912\n"), + stat: stat, inactiveFile: cgroupInactiveFileKeyV2, + }, + { + limit: write("slice.max", twoGiB), usage: write("slice.current", "1879048192\n"), + stat: stat, inactiveFile: cgroupInactiveFileKeyV2, + }, + } + + used, total := s.memoryStats() + if total != 2147483648 { + t.Fatalf("total = %d, want the 2 GiB both levels publish", total) + } + if want := int64(1879048192); used != want { + t.Fatalf("used = %d, want the %d charged to the shared parent, not this service's own", used, want) + } +} + +// A cgroup limit that merely equals the host's memory is no limit at all, and +// must not start reading cgroup usage just because the tie-break loosened. +func TestMemoryLimitEqualToHostRAMIsNotALimit(t *testing.T) { + tree := newProcTree(t) + tree.write("meminfo", "MemTotal: 65536 kB\nMemAvailable: 1024 kB\n") + s := newTestSampler(t, tree, newFakeClock(), Options{}) + + dir := t.TempDir() + write := func(name, body string) string { + full := filepath.Join(dir, name) + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return full + } + s.cgroupUsagePaths = []cgroupUsagePath{{ + limit: write("memory.max", "67108864\n"), // exactly MemTotal + usage: write("memory.current", "1048576\n"), + stat: write("memory.stat", "inactive_file 0\n"), inactiveFile: cgroupInactiveFileKeyV2, + }} + + used, total := s.memoryStats() + if total != 65536*1024 { + t.Fatalf("total = %d, want the host's MemTotal", total) + } + if want := int64(64512 * 1024); used != want { + t.Fatalf("used = %d, want the host's used figure %d rather than the cgroup working set", used, want) + } +} + +// A quota the machine cannot supply is not a limit — a 128-core quota on a +// 64-core host cannot be spent — so it must not move the measurement into the +// cgroup either. It is the same rule a cpuset spanning the host gets: an idle +// Silo beside a saturated neighbor would otherwise report its own few percent +// as the machine's load, from a quota that restricts it in no way at all. +func TestCPUQuotaAboveHostCapacityReportsTheHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + // A four-core host. + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 4 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 900)) + + dir := t.TempDir() + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + // Eight cores of quota on a four-core host. + if err := os.WriteFile(quota, []byte("800000 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.sample(context.Background()) + + // Silo spends 5 of the interval's 20 core-seconds — a quarter of the host — + // while the host itself is three quarters busy with someone else's work. + writeUsage(5_000_000) + tree.write("stat", hostStat(850, 1150)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.Cores != 4 { + t.Fatalf("Cores = %d, want the host's 4 — an 8-core quota is not a limit here", system.Cores) + } + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 rather than this process's 25", system.CPUPct) + } +} + +// Every unconstrained container and systemd service publishes an effective +// cpuset holding every online CPU, because it inherits the root's. Reading that +// as a two-CPU-style restriction would move the CPU reading to this cgroup's own +// usage on a host where nothing caps it, and Silo idling beside a saturated +// neighbor would report Silo's few percent as the machine's load. +func TestCPUIgnoresACpusetSpanningTheWholeHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 8 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 900)) + + dir := t.TempDir() + // All eight of the host's CPUs — what an unrestricted cgroup reports. + cpuset := filepath.Join(dir, "cpuset.cpus.effective") + if err := os.WriteFile(cpuset, []byte("0-7\n"), 0o644); err != nil { + t.Fatalf("write cpuset: %v", err) + } + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + if err := os.WriteFile(quota, []byte("max 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + if got := cgroupCPUSetCores([]string{cpuset}, 8); got != 0 { + t.Fatalf("cgroupCPUSetCores() = %d for a cpuset holding every host CPU, want 0", got) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.cgroupCPUSetPaths = []string{cpuset} + s.sample(context.Background()) + + // The host spends 750 of the interval's 1000 ticks busy; Silo uses 2 of its + // 40 available core-seconds, which is 5%. + writeUsage(2_000_000) + tree.write("stat", hostStat(850, 1150)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 — nothing caps this process", system.CPUPct) + } + if system.Cores != 8 { + t.Fatalf("Cores = %d, want the host's 8", system.Cores) + } +} + +// A quota sized to the whole box is the ordinary way a deployment says "use this +// machine", and it restricts nothing. The boundary matters on its own: the +// over-capacity case is a misconfiguration, while quota == host cores is +// deliberate and common, and reading it as a cap moves the measurement into the +// cgroup exactly where the machine is shared. +func TestCPUQuotaEqualToTheHostReportsTheHost(t *testing.T) { + tree := newProcTree(t) + clock := newFakeClock() + tree.write("loadavg", "0 0 0 0/0 0\n") + tree.write("meminfo", "MemTotal: 1024 kB\n") + tree.write("net/dev", "") + hostStat := func(busy, idle int) string { + line := "cpu " + itoa(busy) + " 0 0 " + itoa(idle) + " 0 0 0 0\n" + for i := range 4 { + line += "cpu" + itoa(i) + " 0 0 0 0 0 0 0 0\n" + } + return line + } + tree.write("stat", hostStat(100, 900)) + + dir := t.TempDir() + usage := filepath.Join(dir, "cpu.stat") + writeUsage := func(micros int) { + if err := os.WriteFile(usage, []byte("usage_usec "+itoa(micros)+"\n"), 0o644); err != nil { + t.Fatalf("write cgroup usage: %v", err) + } + } + writeUsage(0) + quota := filepath.Join(dir, "cpu.max") + // Exactly the host's four cores. + if err := os.WriteFile(quota, []byte("400000 100000\n"), 0o644); err != nil { + t.Fatalf("write cgroup quota: %v", err) + } + + s := newTestSampler(t, tree, clock, Options{}) + s.cgroupCPUPaths = []cgroupCPUPath{{ + usage: usage, usageKey: cgroupCPUUsageKey, usageUnit: time.Microsecond, quota: quota, + }} + s.sample(context.Background()) + + // Silo is nearly idle; the machine it shares is three quarters busy. + writeUsage(200_000) + tree.write("stat", hostStat(850, 1150)) + clock.advance(5 * time.Second) + s.sample(context.Background()) + + system := s.Snapshot().System + if system.CPUPct != 75 { + t.Fatalf("CPUPct = %d, want the host's 75 — a whole-machine quota caps nothing", system.CPUPct) + } + if system.Cores != 4 { + t.Fatalf("Cores = %d, want the host's 4", system.Cores) + } +} + +// The rule is one predicate, and it is the boundary that carries it: strictly +// smaller than the machine binds, as large as the machine does not, and an +// unknown machine is no reason to discard a cap. +func TestCgroupCapBindsOnlyBelowTheHostSize(t *testing.T) { + for _, test := range []struct { + name string + cores float64 + hostCores int + want bool + }{ + {name: "half the host", cores: 2, hostCores: 4, want: true}, + {name: "just under the host", cores: 3.5, hostCores: 4, want: true}, + {name: "the whole host", cores: 4, hostCores: 4, want: false}, + {name: "more than the host", cores: 8, hostCores: 4, want: false}, + {name: "no cap", cores: 0, hostCores: 4, want: false}, + {name: "unknown host keeps a cap", cores: 2, hostCores: 0, want: true}, + {name: "unknown host still has no cap to keep", cores: 0, hostCores: 0, want: false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := cgroupCapBinds(test.cores, test.hostCores); got != test.want { + t.Fatalf("cgroupCapBinds(%v, %d) = %v, want %v", test.cores, test.hostCores, got, test.want) + } + }) + } +} diff --git a/internal/nodemetrics/snapshot.go b/internal/nodemetrics/snapshot.go new file mode 100644 index 000000000..890007724 --- /dev/null +++ b/internal/nodemetrics/snapshot.go @@ -0,0 +1,202 @@ +// Package nodemetrics samples the host resources a media node actually runs +// out of — CPU, memory, disk, network, and GPU engine busyness — into a small +// in-memory snapshot that health responses, the admin API, and Prometheus all +// read without doing any work of their own. +// +// The contract every reader depends on is that sampling is fully decoupled from +// reading. A sample runs on the sampler's own goroutine on a fixed interval and +// publishes an immutable snapshot with one atomic store; a reader loads the +// latest pointer and never waits, never touches /proc, and never runs a +// subprocess. That is what makes it safe to put these numbers in a node's +// /health response, which is the signal the cluster uses to route streams away +// from a dying node: a wedged NFS mount or a hung nvidia-smi degrades one field +// of one snapshot instead of stalling health for every node in the pool. +// +// All sampling is Linux-only. Everywhere else the snapshot reports +// Available=false and the loop idles, so the package is safe to construct +// unconditionally on any platform. +package nodemetrics + +import "time" + +// Snapshot is one complete sampling pass. Snapshots are immutable once +// published: a sampler builds a new one and swaps it in rather than mutating +// the one readers hold. +type Snapshot struct { + // Available reports whether this host can be sampled at all. It is false on + // non-Linux hosts and before the first sample completes, which is why System + // and GPU are pointers/slices rather than values — "no numbers yet" must be + // distinguishable from "everything is zero". + Available bool `json:"available"` + // SampledAt is when this pass ran. Readers use it to tell a live sample from + // a sampler that stopped ticking. + SampledAt time.Time `json:"sampled_at,omitzero"` + // System is the host's CPU/memory/disk/network sample, omitted when the host + // could not be sampled. + System *SystemStats `json:"system,omitempty"` + // GPU is one entry per GPU this process can say anything about, omitted when + // there are none. + GPU []GPUStats `json:"gpu,omitempty"` +} + +// SystemStats is the host resource sample. +type SystemStats struct { + // CPUPct is aggregate busy percentage across all cores between the previous + // sample and this one, 0-100. Under a cgroup it is this container's own + // usage against its own quota, not the host's. + CPUPct int `json:"cpu_pct"` + // Load1 is the 1-minute load average, which unlike CPUPct also counts + // uninterruptible-sleep tasks — a node blocked on storage looks idle in + // CPUPct and busy here. + Load1 float64 `json:"load1"` + // Cores is how many CPUs this process may run on — the cgroup's quota + // rounded up where one is set, otherwise every CPU the kernel reports. It is + // what CPUPct is already normalized against, and what Load1 must be read + // relative to. + Cores int `json:"cores"` + MemUsedMB int64 `json:"mem_used_mb"` + MemTotalMB int64 `json:"mem_total_mb"` + // Disks holds the sampled mounts: the transcode scratch dir first, then any + // media roots the process was told about. Deduplicated by filesystem, so two + // paths on one volume are reported once. + Disks []DiskStats `json:"disks"` + // NetRxBps and NetTxBps are aggregate throughput in *bits* per second, + // matching the node egress_kbps unit used elsewhere in the cluster, with + // loopback excluded. + NetRxBps int64 `json:"net_rx_bps"` + NetTxBps int64 `json:"net_tx_bps"` +} + +// DiskStats is one sampled mount. +type DiskStats struct { + // Path is where the mount is, and is therefore deployment layout rather + // than a resource counter. It is present only on the surfaces that require + // a credential — a node's bearer-authed /status and the API's + // admin-authenticated /admin/system/resources. The unauthenticated /health + // and /metrics carry Role instead; see RedactPaths. + Path string `json:"path,omitempty"` + // Role names the mount by what it is for rather than where it is: + // "scratch" for the transcode working directory, "library-N" positionally + // for each media root. It is assigned when the sample is built, so every + // surface — health, metrics, the admin API — names a mount identically, and + // the index stays with the mount even when a probe cannot measure it. + Role string `json:"role,omitempty"` + UsedGB float64 `json:"used_gb"` + // TotalGB is the capacity usable by this process: bytes in use plus bytes + // still available to it. Blocks a filesystem reserves for root are in + // neither, so this reads lower than the device's nameplate size on a default + // ext4 volume — and UsedGB/TotalGB is the ratio `df` prints as Use%, which + // is what makes a full volume read as full rather than as 95%. + TotalGB float64 `json:"total_gb"` + // Stale marks numbers carried over from an earlier pass because the current + // probe has not returned — the normal reading for a network mount whose + // server went away. The values are real, just old. + Stale bool `json:"stale,omitempty"` + // Unavailable marks a path that has never been measured successfully (it + // does not exist on this node, or the very first probe is still hanging). + // UsedGB/TotalGB are meaningless when it is set. + Unavailable bool `json:"unavailable,omitempty"` + // Scratch marks the transcode working directory's entry. It is the one mount + // whose filling up breaks transcoding rather than merely browsing, so a + // reader has to be able to find it without knowing the deployment's paths — + // which is exactly the position the API is in, since it stores a node's + // sample opaquely. Set on at most one entry per sample; a media root sharing + // the scratch volume is deduplicated onto the scratch entry, which is + // reported first. + Scratch bool `json:"scratch,omitempty"` +} + +// ScratchDiskRole is the Role of the transcode working directory's entry. +const ScratchDiskRole = "scratch" + +// RedactPaths returns a copy of this snapshot with every filesystem path +// removed, for a surface that answers without a credential. +// +// A node's /health is reachable by anyone who can reach the node, and its disk +// entries would otherwise publish the transcode scratch directory and — on the +// API host — every configured library root. That is deployment layout, not a +// host resource counter: it is exactly what the admin-authenticated +// /admin/system/resources exists to gate, and what /metrics already withholds +// by labeling series with Role. Role survives here, so a reader still learns +// which mount is the scratch volume and how full it is; only where it lives is +// withheld. +// +// GPU device names are deliberately kept. A render node (/dev/dri/renderD128) +// or a CUDA index is a hardware fact present on every Linux host, not a +// property of this deployment, and /metrics already labels its per-GPU series +// with the same value. +func (s Snapshot) RedactPaths() Snapshot { + if s.System == nil || len(s.System.Disks) == 0 { + return s + } + system := *s.System + disks := make([]DiskStats, len(system.Disks)) + for i, disk := range system.Disks { + disk.Path = "" + disks[i] = disk + } + system.Disks = disks + s.System = &system + return s +} + +// vendorNVIDIA is the GPUStats.Vendor value nvidia-smi enrichment reports. +const vendorNVIDIA = "nvidia" + +// GPU measurement sources, in order of how much they can see. +const ( + // SourceUnavailable means nothing could measure this device: no owned + // process held it and no whole-GPU query answered. + SourceUnavailable = "unavailable" + // SourceFdinfo is the unprivileged DRM baseline. It sees only engine time + // spent by this process's own ffmpeg children, never another tenant's. + SourceFdinfo = "fdinfo" + // SourceNVIDIASMI is whole-GPU utilization from nvidia-smi, which is the + // only signal for NVIDIA (the proprietary driver implements no fdinfo). + SourceNVIDIASMI = "nvidia-smi" + // SourceFdinfoNVIDIASMI is both, for a device that has DRM counters and an + // nvidia-smi row. + SourceFdinfoNVIDIASMI = "fdinfo+nvidia-smi" +) + +// GPUStats is one GPU's sample. +type GPUStats struct { + // Device is the render-node path (/dev/dri/renderD128) where one is known, + // otherwise the PCI address or a "cuda:N" index for an NVIDIA GPU with no + // DRM node this process can see. + Device string `json:"device"` + // Vendor is "intel", "nvidia", "amd" or empty when unknown. + Vendor string `json:"vendor,omitempty"` + // Sessions is how many of this process's GPU workloads are currently pinned + // to this device. It comes from the playback device balancer, not from the + // driver, so it is exact for our own work and blind to everyone else's. + Sessions int `json:"sessions"` + // VideoBusyPct and RenderBusyPct are engine busy percentages over the last + // sample interval. From fdinfo they cover only our own processes. + VideoBusyPct *int `json:"video_busy_pct,omitempty"` + RenderBusyPct *int `json:"render_busy_pct,omitempty"` + // TotalBusyPct is whole-GPU utilization including other tenants. + // + // Every measurement above and below is a pointer because availability is per + // field, not per device: a source answers for some columns and not others — + // nvidia-smi prints "[N/A]" for an engine a card cannot report while still + // giving real memory figures, and fdinfo has no reading at all until it has + // two samples to diff. "Nothing measured this" and "measured, and idle" are + // different facts, and an operator must not read the first as the second. + TotalBusyPct *int `json:"total_busy_pct,omitempty"` + VRAMUsedMB *int64 `json:"vram_used_mb,omitempty"` + VRAMTotalMB *int64 `json:"vram_total_mb,omitempty"` + // Source names what produced the numbers above; see the Source* constants. + Source string `json:"source"` +} + +// DeviceIdentity is one render device as the host's hardware detection sees it. +// The sampler needs it only to translate the PCI addresses DRM fdinfo reports +// into the /dev/dri paths every other surface (settings, session accounting, +// the admin UI) speaks, which is why this package takes it as a provider +// instead of doing its own hardware walk. +type DeviceIdentity struct { + Path string + PCIAddress string + Vendor string +} diff --git a/internal/nodemetrics/statfs_other.go b/internal/nodemetrics/statfs_other.go new file mode 100644 index 000000000..3c05a18e7 --- /dev/null +++ b/internal/nodemetrics/statfs_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package nodemetrics + +import "errors" + +// osStatfs has no implementation on platforms without statfs(2). Sampling is +// Linux-only anyway, so this only has to keep the package building. +func osStatfs(string) (fsStats, error) { + return fsStats{}, errors.ErrUnsupported +} diff --git a/internal/nodemetrics/statfs_unix.go b/internal/nodemetrics/statfs_unix.go new file mode 100644 index 000000000..eea12b77e --- /dev/null +++ b/internal/nodemetrics/statfs_unix.go @@ -0,0 +1,35 @@ +//go:build linux || darwin + +package nodemetrics + +import ( + "golang.org/x/sys/unix" +) + +// osStatfs reports one path's filesystem capacity, in the terms that matter to +// a process deciding whether it can keep writing. +// +// Used counts every block the filesystem considers taken (Blocks-Bfree), which +// is what `df` puts in its Used column. Total is that plus the blocks still +// available to *this* process (Bavail) — deliberately not the raw device size. +// A filesystem holds blocks back from unprivileged users (ext4 reserves 5% by +// default), and those are writable by root alone: counting them as capacity +// makes a volume with nothing left for Silo to write read as 95% full, exactly +// where the scratch admission guard is set. A node would then admit sessions +// until the moment it has zero bytes of headroom and start failing transcodes +// mid-stream, which is the failure that guard exists to prevent. +// +// The resulting Used/Total is the same ratio `df` prints as Use%. +// +// This call can block indefinitely on an unresponsive network mount. Callers +// must treat it as such — see probeDisk, which runs it on a goroutine nothing +// waits for. +func osStatfs(path string) (fsStats, error) { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + return fsStats{}, err + } + stats := fsCapacity(st.Blocks, st.Bfree, st.Bavail, uint64(st.Bsize)) + stats.FSID = formatFSID(int64(st.Fsid.Val[0]), int64(st.Fsid.Val[1])) + return stats, nil +} diff --git a/internal/nodemetrics/system.go b/internal/nodemetrics/system.go new file mode 100644 index 000000000..43ff244f5 --- /dev/null +++ b/internal/nodemetrics/system.go @@ -0,0 +1,309 @@ +package nodemetrics + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// cpuTimes is one /proc/stat aggregate reading. Both fields are monotonic jiffy +// counters; only differences between two readings mean anything. +type cpuTimes struct { + busy uint64 + total uint64 + valid bool +} + +// ifaceCounters is one interface's monotonic byte counters. +type ifaceCounters struct { + rx uint64 + tx uint64 +} + +// netCounters is one /proc/net/dev reading, excluding loopback. +// +// Counters are kept per interface rather than pre-summed because deltas are +// only meaningful per interface: an interface that enters the namespace +// mid-run (a veth or tunnel moved into a running container) arrives carrying +// its lifetime totals, and an aggregate baseline would report all of that +// history as one interval's traffic. +type netCounters struct { + interfaces map[string]ifaceCounters + at time.Time + valid bool +} + +// readCPUTimes parses the aggregate "cpu" line of /proc/stat and counts the +// per-core lines beneath it. +// +// Idle and iowait are both subtracted from busy: a core waiting on storage is +// not doing work, and counting it as busy would make every node with a slow +// disk look CPU-bound. +func readCPUTimes(procDir string) (cpuTimes, int) { + raw, err := os.ReadFile(filepath.Join(procDir, "stat")) + if err != nil { + return cpuTimes{}, 0 + } + var times cpuTimes + cores := 0 + for line := range strings.Lines(string(raw)) { + fields := strings.Fields(line) + if len(fields) == 0 || !strings.HasPrefix(fields[0], "cpu") { + continue + } + if fields[0] != "cpu" { + // "cpu0", "cpu1", … — one line per online core. + cores++ + continue + } + if times.valid { + continue + } + var total, idle uint64 + for i, field := range fields[1:] { + value, err := strconv.ParseUint(field, 10, 64) + if err != nil { + continue + } + // Columns are user, nice, system, idle, iowait, irq, softirq, + // steal, guest, guest_nice. guest time is already included in user, + // so summing past steal would double-count it. + if i >= 8 { + break + } + total += value + if i == 3 || i == 4 { + idle += value + } + } + if total == 0 { + continue + } + times = cpuTimes{busy: total - idle, total: total, valid: true} + } + return times, cores +} + +// cpuBusyPercent converts two readings into a busy percentage. +// +// A counter that went backwards means the readings do not describe one +// continuous run (a container was migrated, /proc was remounted, or a test +// rewound the fixture), so the pair is unusable rather than negative. +func cpuBusyPercent(previous, current cpuTimes) (int, bool) { + if !previous.valid || !current.valid { + return 0, false + } + if current.total <= previous.total || current.busy < previous.busy { + return 0, false + } + totalDelta := current.total - previous.total + busyDelta := current.busy - previous.busy + return clampPercent(int((busyDelta*100 + totalDelta/2) / totalDelta)), true +} + +// readLoad1 parses the 1-minute load average from /proc/loadavg. +func readLoad1(procDir string) float64 { + raw, err := os.ReadFile(filepath.Join(procDir, "loadavg")) + if err != nil { + return 0 + } + fields := strings.Fields(string(raw)) + if len(fields) == 0 { + return 0 + } + value, err := strconv.ParseFloat(fields[0], 64) + if err != nil || value < 0 { + return 0 + } + return value +} + +// readNetCounters sums received and transmitted bytes across every interface +// except loopback, which would otherwise double-count traffic a node sends to +// itself (health checks, the local proxy hop). +func readNetCounters(procDir string, at time.Time) netCounters { + raw, err := os.ReadFile(filepath.Join(procDir, "net", "dev")) + if err != nil { + return netCounters{} + } + counters := netCounters{at: at, interfaces: map[string]ifaceCounters{}} + for line := range strings.Lines(string(raw)) { + name, rest, ok := strings.Cut(line, ":") + if !ok { + // The two header lines carry no colon. + continue + } + iface := strings.TrimSpace(name) + if iface == "lo" { + continue + } + fields := strings.Fields(rest) + // Eight receive columns then eight transmit columns; bytes lead each. + if len(fields) < 9 { + continue + } + rx, rxErr := strconv.ParseUint(fields[0], 10, 64) + tx, txErr := strconv.ParseUint(fields[8], 10, 64) + if rxErr != nil || txErr != nil { + continue + } + counters.interfaces[iface] = ifaceCounters{rx: rx, tx: tx} + counters.valid = true + } + return counters +} + +// netThroughputBps converts two readings into bits per second, summing only +// per-interface deltas between interfaces present in both readings. +// +// The pairing is what keeps every kind of interface churn out of an +// operator's bandwidth graph. An interface that appears between readings +// arrives with its lifetime totals — history, not this interval's traffic — +// so it contributes nothing until the next reading gives it a baseline. One +// that disappears takes only its own baseline with it, so the remaining +// interfaces keep measuring instead of being masked by a shrunken aggregate. +// And a single counter that runs backwards (a 32-bit counter on a busy link +// wrapping, or a device slot reused) zeroes only that interface's delta, +// since an invented number is worse than a missing one. +func netThroughputBps(previous, current netCounters) (rxBps, txBps int64, ok bool) { + if !previous.valid || !current.valid { + return 0, 0, false + } + seconds := current.at.Sub(previous.at).Seconds() + if seconds <= 0 { + return 0, 0, false + } + var rxDelta, txDelta uint64 + for name, cur := range current.interfaces { + prev, seen := previous.interfaces[name] + if !seen { + continue + } + if cur.rx >= prev.rx { + rxDelta += cur.rx - prev.rx + } + if cur.tx >= prev.tx { + txDelta += cur.tx - prev.tx + } + } + return int64(float64(rxDelta) * 8 / seconds), int64(float64(txDelta) * 8 / seconds), true +} + +// procDirFor resolves which proc tree to read name ("stat", "loadavg", or +// "meminfo") from — the three files a container's cgroup cannot itself +// correct. +// +// On an LXC host running Docker nested inside it, this process's own /proc is +// the raw kernel view: /proc/stat, /proc/loadavg, and /proc/meminfo describe +// the physical machine rather than the LXC, and this nested container's own +// cgroup shows no limit at all, because the LXC's cap lives on an ancestor +// cgroup outside this container's namespace that it cannot see or read. +// lxcfs, running on the LXC host, virtualizes those same three files to the +// LXC's own limits; when an operator bind-mounts that virtualized view in at +// hostProcDir/, it is the only correct source, so it wins whenever +// present. Otherwise procDir/ — the file this process actually sees — +// is the only option, exactly as on plain Docker or bare metal. +func (s *Sampler) procDirFor(name string) string { + if s.hostProcDir != "" { + if _, err := os.Stat(filepath.Join(s.hostProcDir, name)); err == nil { + return s.hostProcDir + } + } + return s.procDir +} + +// memoryStats reports used and total bytes for this process's memory domain. +// +// /proc/meminfo describes the host even inside a container, so a cgroup limit — +// which is what the kernel will actually OOM-kill against — always wins over +// it, and cgroup usage (page cache excluded) wins over the host's own +// used figure for the same reason. +func (s *Sampler) memoryStats() (usedBytes, totalBytes int64) { + fields, err := ReadMeminfoBytes(filepath.Join(s.procDirFor("meminfo"), "meminfo")) + if err == nil { + totalBytes = fields["MemTotal"] + if available, ok := fields["MemAvailable"]; ok && totalBytes >= available { + usedBytes = totalBytes - available + } + } + + // Every limit in force is read, not just the first: the list runs from this + // process's own cgroup up through its ancestors, and a leaf that says "max" + // can still sit inside a slice or pod cgroup that does not. The kernel + // OOM-kills against the tightest of them, so that is the one to report — and + // the level it came from is kept, because the usage that fills it is + // everything charged to that cgroup, not just this process's own. + var binding *cgroupUsagePath + hostTotal := totalBytes + for i, level := range s.cgroupUsagePaths { + limit, err := ReadCgroupMemoryLimit(level.limit) + if err != nil || limit <= 0 { + continue + } + // A limit at or above the host's memory is not a limit worth reporting; + // it would only make a node look like it has headroom the kernel cannot + // give it. + if hostTotal > 0 && limit >= hostTotal { + continue + } + // Ties go to the outer level, which the list reaches later. Two cgroups + // publishing the same limit are not equivalent: the ancestor's is shared + // with siblings that can fill it, so it is the one whose usage says how + // much of it is left. Reading the leaf instead shows headroom right up + // until the parent OOMs. Compared against the running choice rather than + // against the host figure, so a cgroup limit that merely equals host RAM + // still reads as no limit at all. + if binding == nil || limit <= totalBytes { + totalBytes = limit + binding = &s.cgroupUsagePaths[i] + } + } + + // Only when the total above is the cgroup's. A container with no memory + // limit still publishes a readable memory.current, so taking it + // unconditionally would pair this process's working set with the host's RAM + // — "1 GiB of 64 GiB" on a machine that is nearly out of memory, because the + // two numbers describe different domains. Whichever domain total came from, + // used has to come from the same one. + if binding != nil { + if usage, ok := cgroupMemoryUsage(*binding); ok { + usedBytes = usage + } + } + if totalBytes > 0 && usedBytes > totalBytes { + usedBytes = totalBytes + } + return usedBytes, totalBytes +} + +// cgroupMemoryUsage returns the working set of one memory cgroup: its current +// charge minus reclaimable file pages. +// +// The level is the caller's choice rather than a search, because the only level +// worth measuring is the one whose limit binds — see memoryStats. +func cgroupMemoryUsage(level cgroupUsagePath) (int64, bool) { + usage, err := readCgroupSingleValue(level.usage) + if err != nil { + return 0, false + } + if inactive, err := readCgroupStatKey(level.stat, level.inactiveFile); err == nil && inactive > 0 && inactive <= usage { + usage -= inactive + } + return usage, true +} + +func clampPercent(value int) int { + if value < 0 { + return 0 + } + if value > 100 { + return 100 + } + return value +} + +const bytesPerMB = int64(1024 * 1024) + +func bytesToMB(value int64) int64 { return value / bytesPerMB } diff --git a/internal/nodepool/gpuidentity.go b/internal/nodepool/gpuidentity.go new file mode 100644 index 000000000..d212e1e8f --- /dev/null +++ b/internal/nodepool/gpuidentity.go @@ -0,0 +1,100 @@ +package nodepool + +import ( + "encoding/json" + "slices" +) + +// gpuIdentityView is the minimal projection this package parses out of an +// otherwise opaque capability payload, in the same spirit as +// capabilityDriftView: nodepool must not depend on playback, and identifying a +// GPU only needs the host's boot id and each render device's own identity. +type gpuIdentityView struct { + BootID string `json:"boot_id"` + RenderDeviceDetails []struct { + PCIAddress string `json:"pci_address"` + GPUUUID string `json:"gpu_uuid"` + } `json:"render_device_details"` + // NVIDIAGPUUUIDs covers cards with no readable DRM node, which is the + // ordinary shape of an NVIDIA container: /dev/nvidia* and the toolkit, no + // /dev/dri. NVENC works there and render_device_details is empty, so + // without this the host contributes no identity at all and two containers + // on one card read as two GPUs. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` +} + +// physicalGPUKeys derives one stable key per GPU a node can see, deduplicated +// and sorted so two nodes' key sets can be compared directly. +// +// An NVIDIA uuid is preferred because it follows the card between slots and +// hosts; the PCI address falls back to it, scoped by boot id because a device +// path and a slot only mean the same hardware within one boot of one kernel. A +// device with neither is unidentifiable and contributes no key rather than a +// fake one — a synthetic key would claim two nodes share hardware (or do not) +// on no evidence, and both directions of that claim change how work is placed. +// +// The fallback needs a boot id as much as it needs a slot. Boot id detection is +// best-effort (it reads /proc, which a hardened or sandboxed host may hide even +// while sysfs stays readable), and an empty one scopes the key to nothing: two +// unrelated hosts whose iGPU sits at the near-universal 0000:00:02.0 would both +// derive "|0000:00:02.0" and be routed as one card. So an unscoped slot is +// treated like no identity at all — nodes on such a host are accounted +// independently, which is only what they got before GPU grouping existed. +// +// A payload that cannot be parsed yields no keys: an unreadable report is not +// evidence about hardware. +func physicalGPUKeys(capabilities []byte) []string { + if len(capabilities) == 0 { + return nil + } + var identity gpuIdentityView + if err := json.Unmarshal(capabilities, &identity); err != nil { + return nil + } + total := len(identity.RenderDeviceDetails) + len(identity.NVIDIAGPUUUIDs) + seen := make(map[string]struct{}, total) + keys := make([]string, 0, total) + add := func(key string) { + if key == "" { + return + } + if _, duplicate := seen[key]; duplicate { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } + for _, device := range identity.RenderDeviceDetails { + key := device.GPUUUID + if key == "" { + if device.PCIAddress == "" || identity.BootID == "" { + continue + } + key = identity.BootID + "|" + device.PCIAddress + } + add(key) + } + // A uuid is host-independent, so a card reported only through nvidia-smi + // keys the same way whether or not it also has a render node — which is + // what lets a container with /dev/dri and one without recognize the same + // physical GPU. + for _, uuid := range identity.NVIDIAGPUUUIDs { + add(uuid) + } + if len(keys) == 0 { + return nil + } + slices.Sort(keys) + return keys +} + +// applyPhysicalGPUKeys refreshes a node's derived GPU identities from the +// capability payload it currently carries. Every place a Node is built from +// stored bytes calls it, so the field is never a stale answer to an older +// payload: the row scanner, the pools' load path, and the capability writer. +func applyPhysicalGPUKeys(n *Node) { + if n == nil { + return + } + n.PhysicalGPUKeys = physicalGPUKeys(n.Capabilities) +} diff --git a/internal/nodepool/gpuidentity_test.go b/internal/nodepool/gpuidentity_test.go new file mode 100644 index 000000000..382c29375 --- /dev/null +++ b/internal/nodepool/gpuidentity_test.go @@ -0,0 +1,160 @@ +package nodepool + +import ( + "encoding/json" + "slices" + "testing" + "time" +) + +// An NVIDIA uuid identifies a card wherever it is plugged in; a PCI address +// only identifies a slot, and only within one boot of one kernel. Deriving the +// key this way is what lets the planner — and an admin — see that two nodes are +// sharing one GPU. +func TestPhysicalGPUKeys(t *testing.T) { + tests := []struct { + name string + capabilities string + want []string + }{ + { + name: "prefers gpu uuid over slot identity", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}`, + want: []string{"GPU-aaa"}, + }, + { + name: "falls back to boot-scoped pci address", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}]}`, + want: []string{"boot-1|0000:04:00.0"}, + }, + { + name: "mixed devices are deduped and sorted", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}, + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}, + {"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0","gpu_uuid":"GPU-bbb"}]}`, + want: []string{"GPU-bbb", "boot-1|0000:05:00.0"}, + }, + { + name: "device with no identity contributes no key", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128"},{"path":"/dev/dri/renderD129","gpu_uuid":"GPU-ccc"}]}`, + want: []string{"GPU-ccc"}, + }, + { + // Boot id detection is best-effort. Without one the slot is scoped + // to nothing, and "|0000:00:02.0" — where every Intel iGPU lives — + // would merge unrelated hosts into one GPU group. + name: "a slot with no boot id contributes no key", + capabilities: `{"render_device_details":[{"pci_address":"0000:00:02.0"}]}`, + want: nil, + }, + { + // A uuid is host-independent by construction, so it survives the + // missing boot id that disqualifies its slot. + name: "a uuid still identifies a device on a host with no boot id", + capabilities: `{"render_device_details":[{"pci_address":"0000:03:00.0","gpu_uuid":"GPU-ddd"}]}`, + want: []string{"GPU-ddd"}, + }, + { + // The ordinary NVIDIA container: /dev/nvidia* and the toolkit, no + // /dev/dri at all. NVENC works, render_device_details is empty, and + // without the uuid list the whole host contributes no identity — so + // two containers on one card read as two independent GPUs and the + // shared-GPU tie-break keeps piling work onto the same hardware. + name: "cuda-only host keys by uuid with no render device", + capabilities: `{"boot_id":"boot-1","render_device_details":[],"nvidia_gpu_uuids":["GPU-eee","GPU-fff"]}`, + want: []string{"GPU-eee", "GPU-fff"}, + }, + { + // A uuid is host-independent, so the card a container reaches only + // through nvidia-smi keys identically to the one its neighbor also + // sees through a render node. That identity is the whole point. + name: "a card reported both ways contributes one key", + capabilities: `{"boot_id":"boot-1","render_device_details":[ + {"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-ggg"}], + "nvidia_gpu_uuids":["GPU-ggg"]}`, + want: []string{"GPU-ggg"}, + }, + { + name: "unparseable uuid list yields no keys", + capabilities: `{"nvidia_gpu_uuids":"nope"}`, + want: nil, + }, + {name: "no capabilities stored", capabilities: "", want: nil}, + {name: "unparseable payload", capabilities: `not json`, want: nil}, + {name: "payload of the wrong shape", capabilities: `{"render_device_details":"nope"}`, want: nil}, + {name: "no render devices", capabilities: `{"boot_id":"boot-1","render_device_details":[]}`, want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := physicalGPUKeys([]byte(tt.capabilities)) + if !slices.Equal(got, tt.want) { + t.Fatalf("physicalGPUKeys() = %v, want %v", got, tt.want) + } + }) + } +} + +const gpuAAACapabilities = `{"boot_id":"boot-1","render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-aaa"}]}` + +// Keys must exist from the moment a stored row reaches a pool, not only after +// the next capability refetch: after an API restart the planner routes on the +// inventory the database already holds. +func TestPoolsDeriveGPUKeysOnLoad(t *testing.T) { + transcodes := NewTranscodePool() + transcodes.SetNodes([]*Node{{ + ID: 1, URL: "http://tc-1/", Enabled: true, Healthy: true, + Capabilities: json.RawMessage(gpuAAACapabilities), + }}) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("transcode pool load derived %v, want [GPU-aaa]", got) + } + // The URL normalization the same loop performs must still happen. + if got := transcodes.Nodes()[0].URL; got != "http://tc-1" { + t.Fatalf("transcode pool load left URL %q unnormalized", got) + } + + proxies := NewProxyPool() + proxies.SetNodes([]*Node{ + {ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true, Capabilities: json.RawMessage(gpuAAACapabilities)}, + {ID: 3, URL: "http://proxy-2", Enabled: true, Healthy: true}, + }) + if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("proxy pool load derived %v, want [GPU-aaa]", got) + } + if got := proxies.Nodes()[1].PhysicalGPUKeys; got != nil { + t.Fatalf("node without a stored report derived %v, want none", got) + } +} + +// A refetched report replaces the identities it describes; carrying the +// previous ones over would claim a GPU that the node no longer reports. +func TestApplyCapabilitiesDerivesGPUKeys(t *testing.T) { + refreshedAt := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + + transcodes := NewTranscodePool() + transcodes.SetNodes([]*Node{{ID: 1, URL: "http://tc-1", Enabled: true, Healthy: true}}) + transcodes.ApplyCapabilities(1, "http://tc-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil, nil) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("transcode ApplyCapabilities derived %v, want [GPU-aaa]", got) + } + + // The card was passed through to another host: the node now reports a + // device it cannot identify, and must stop claiming the old key. + transcodes.ApplyCapabilities(1, "http://tc-1", []byte(`{"boot_id":"boot-2","render_device_details":[{"path":"/dev/dri/renderD128"}]}`), + "sha256:bbb", refreshedAt, nil, nil) + if got := transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { + t.Fatalf("stale identities survived a new report: %v", got) + } + + proxies := NewProxyPool() + proxies.SetNodes([]*Node{{ID: 2, URL: "http://proxy-1", Enabled: true, Healthy: true}}) + proxies.ApplyCapabilities(2, "http://proxy-1", []byte(gpuAAACapabilities), "sha256:aaa", refreshedAt, nil, nil) + if got := proxies.Nodes()[0].PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("proxy ApplyCapabilities derived %v, want [GPU-aaa]", got) + } +} diff --git a/internal/nodepool/health.go b/internal/nodepool/health.go index 411263171..3defb724a 100644 --- a/internal/nodepool/health.go +++ b/internal/nodepool/health.go @@ -1,12 +1,18 @@ package nodepool import ( + "bytes" "context" "encoding/json" + "errors" + "io" "log/slog" "net/http" + "slices" + "strings" "sync" "time" + "unicode/utf8" ) // healthResponse is the JSON response from a node's /health endpoint. @@ -14,36 +20,176 @@ type healthResponse struct { Status string `json:"status"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash identifies the node's current hardware capability + // snapshot. A node that predates capability snapshots reports none, which + // is how the sweep tells "nothing changed" from "cannot say". + CapabilitiesHash string `json:"capabilities_hash"` + // System and GPU are the node's latest resource sample, carried opaquely. + // This package deliberately does not parse them: node metrics are display + // data, nothing here routes on them, and decoding them would make nodepool + // depend on the sampler's schema — so a node running a newer build can add + // fields without an API-side change. + System json.RawMessage `json:"system"` + GPU json.RawMessage `json:"gpu"` } +// maxHealthResponseBytes bounds a node's whole /health body. +// +// A node is a worker that may run on remote, less trusted hardware, and its +// health answer is the one node-controlled payload this process decodes every +// 30 seconds. An honest sample is under 2 KB; this leaves three orders of +// magnitude of headroom while keeping a buggy or hostile build from making the +// API allocate an arbitrary body on a fixed cadence. +const maxHealthResponseBytes = 256 << 10 + +// maxLastStatsBytes bounds the resource blob that is persisted and served. +// +// Past this, the stats are dropped and the health verdict is kept: whether a +// node is alive routes streams, while its dashboard numbers do not, and an +// oversized blob would otherwise be rewritten into a jsonb column every sweep +// and echoed to every admin listing nodes. +const maxLastStatsBytes = 32 << 10 + // CheckNode pings a node's /health endpoint and returns its health status, -// active job count, and reported egress bandwidth. -func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int) { +// active job count, reported egress bandwidth, capability hash, and the opaque +// resource-stats blob to persist (nil when the node reported none). +func CheckNode(ctx context.Context, n *Node) (healthy bool, activeJobs, egressKbps int, capabilitiesHash string, lastStats []byte) { client := &http.Client{Timeout: 5 * time.Second} ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, n.URL+"/api/v1/health", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, NodeEndpoint(n.URL, "/api/v1/health"), nil) if err != nil { - return false, 0, 0 + return false, 0, 0, "", nil } resp, err := client.Do(req) if err != nil { - return false, 0, 0 + return false, 0, 0, "", nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return false, 0, 0 + return false, 0, 0, "", nil + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxHealthResponseBytes+1)) + if err != nil { + return false, 0, 0, "", nil + } + if len(body) > maxHealthResponseBytes { + // Nothing in the body can be trusted to be well-formed at that size, so + // the node is treated as not answering rather than partially believed. + slog.WarnContext(ctx, "node health response too large to read", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, "limit_bytes", maxHealthResponseBytes) + return false, 0, 0, "", nil } var hr healthResponse - if err := json.NewDecoder(resp.Body).Decode(&hr); err != nil { - return false, 0, 0 + if err := json.Unmarshal(body, &hr); err != nil { + return false, 0, 0, "", nil } - return true, hr.ActiveJobs, hr.EgressKbps + return true, hr.ActiveJobs, hr.EgressKbps, hr.CapabilitiesHash, marshalLastStats(ctx, n, hr) +} + +// marshalLastStats packs a health response's resource fields into the blob +// stored on the node row, or nil when the node sent neither. +// +// nil is what a node predating resource sampling produces, and it must persist +// as SQL NULL rather than as an empty object: "this node cannot report" and +// "this node reported nothing in use" are different states, and only the second +// would justify drawing a zero on a dashboard. +func marshalLastStats(ctx context.Context, n *Node, hr healthResponse) []byte { + system := trimJSONNull(hr.System) + gpu := trimJSONNull(hr.GPU) + if system == nil && gpu == nil { + return nil + } + payload := struct { + System json.RawMessage `json:"system,omitempty"` + GPU json.RawMessage `json:"gpu,omitempty"` + }{System: system, GPU: gpu} + encoded, err := json.Marshal(payload) + if err != nil { + return nil + } + if len(encoded) > maxLastStatsBytes { + slog.WarnContext(ctx, "node resource sample too large to store", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, + "bytes", len(encoded), "limit_bytes", maxLastStatsBytes) + return nil + } + return encoded +} + +// trimJSONNull normalizes an absent field. encoding/json leaves a RawMessage +// nil when the key is missing but sets it to the literal "null" when the key is +// present and null, and both mean the node has nothing to say. +func trimJSONNull(raw json.RawMessage) json.RawMessage { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || string(trimmed) == "null" { + return nil + } + return trimmed +} + +// CapabilityFetcher retrieves one node's full capability report and the hash +// the payload identifies itself by. The payload is stored opaquely, so this +// package never has to understand — or import — the playback capability model. +// An empty hash means the payload cannot be tracked for change and must not be +// persisted. +// The whole node is passed rather than its URL because the answer's cost is a +// property of the node: a worker builds its probe matrix from its *effective* +// acceleration policy, so one carrying an hw_device override with two devices +// legitimately takes longer to answer than the cluster-wide setting predicts. A +// fetcher that sizes its request from the cluster setting alone cancels such a +// node mid-probe every sweep, and its inventory never lands. +type CapabilityFetcher func(ctx context.Context, node *Node) (payload []byte, hash string, err error) + +// capabilityFetchTimeout is the floor under the backstop on one capability +// fetch. It is not the budget. +// +// The budget belongs to the fetcher, which knows the configured hardware and +// sizes each request against the node's own advertised probe matrix. That matrix +// grows with the device count without bound — nine render devices legitimately +// ask for over five minutes — so no fixed number here can be both a real +// backstop and safe against cutting a node short. What this stops is the other +// failure: a fetcher that never returns, pinning a goroutine and a node's +// inventory forever. +// +// So the backstop is derived from the fetcher's own budget and this floor is +// only what applies when no budget is wired. A backstop that trips during +// ordinary operation is indistinguishable from the bug it was meant to catch. +const capabilityFetchTimeout = 5 * time.Minute + +// capabilityFetchSlack is how far the sweep's backstop sits above the budget the +// fetcher gave itself, so the fetcher's own deadline is always the one that +// fires first and the failure an operator sees names the probe rather than this. +const capabilityFetchSlack = time.Minute + +// CapabilityRefreshTimeout is the floor under that bound, and all a caller can +// assume when it has no health checker to ask. It is exported for the one caller +// that has to hold an HTTP connection open across a refresh and must therefore +// size its own write deadline to include it — see CapabilityRefreshBound, which +// is the number that caller should actually use. +const CapabilityRefreshTimeout = capabilityFetchTimeout + +// CapabilityRefreshBound is how long RefreshNodeCapabilities may take for this +// node, which is the same backstop the sweep's own fetches run under. +// +// It is exported because a caller holding an HTTP connection open across a +// refresh has to reserve the real number, not the floor: the backstop is derived +// from the node's advertised probe budget, and a node with a large device set +// asks for well past five minutes. Reserving the floor there means the +// connection's write deadline can fire after the refresh succeeded but before +// its response is written, and the operator is told an action failed that has +// already changed the node. +func (hc *HealthChecker) CapabilityRefreshBound(n *Node) time.Duration { + if hc == nil { + return capabilityFetchTimeout + } + return hc.capabilityFetchBackstop(n) } // HealthChecker runs periodic health checks on all nodes in both pools, @@ -53,6 +199,23 @@ type HealthChecker struct { transcodePool *TranscodePool repo *Repository // may be nil (proxy/transcode modes have no DB) interval time.Duration + + // mu guards the two injected hooks. Both are wired after construction — + // the capability-change callback because the playback handler that consumes + // it is built later, during router assembly — while the sweep may already + // be running. + mu sync.RWMutex + capFetch CapabilityFetcher + capFetchBudget func(*Node) time.Duration + onCapabilitiesChanged func(nodeURL string) + + // capabilityRefreshes tracks the detached capability fetches so shutdown — + // and tests — can wait for them. The sweep itself must never wait on one. + capabilityRefreshes sync.WaitGroup + // capabilityRefreshInFlight holds the node ids currently being fetched, so + // a fetch that outlives the sweep that started it is not started again by + // the next sweep. Node ids are unique across both pools (one table). + capabilityRefreshInFlight sync.Map } // NewHealthChecker creates a health checker for the given pools. @@ -65,6 +228,63 @@ func NewHealthChecker(proxyPool *ProxyPool, transcodePool *TranscodePool, repo * } } +// SetCapabilityFetchBudget wires how long the fetcher will allow itself for one +// node, so the sweep's backstop can be derived from it rather than guessed. +// +// Without it the backstop falls back to capabilityFetchTimeout, which is right +// for the common configuration and too tight for a node with many devices — +// hence this, supplied by the same wiring that supplies the fetcher. +func (hc *HealthChecker) SetCapabilityFetchBudget(budget func(*Node) time.Duration) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.capFetchBudget = budget +} + +// capabilityFetchBackstop bounds one fetch above whatever the fetcher allowed +// itself, so the fetcher's deadline always fires first. +func (hc *HealthChecker) capabilityFetchBackstop(n *Node) time.Duration { + hc.mu.RLock() + budget := hc.capFetchBudget + hc.mu.RUnlock() + if budget == nil { + return capabilityFetchTimeout + } + return max(capabilityFetchTimeout, budget(n)+capabilityFetchSlack) +} + +// SetCapabilityFetcher wires how the sweep retrieves a node's capability report +// once the node reports a hash it has not stored. Without one the sweep behaves +// exactly as it did before capability tracking. +func (hc *HealthChecker) SetCapabilityFetcher(fetch CapabilityFetcher) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.capFetch = fetch +} + +// SetCapabilitiesChangedCallback wires the notification fired after a node's +// capabilities were refetched and stored, so caches keyed on node capability +// can be invalidated without waiting for their own TTL. +func (hc *HealthChecker) SetCapabilitiesChangedCallback(fn func(nodeURL string)) { + if hc == nil { + return + } + hc.mu.Lock() + defer hc.mu.Unlock() + hc.onCapabilitiesChanged = fn +} + +func (hc *HealthChecker) hooks() (CapabilityFetcher, func(nodeURL string)) { + hc.mu.RLock() + defer hc.mu.RUnlock() + return hc.capFetch, hc.onCapabilitiesChanged +} + // Start runs health checks in a background goroutine. Stops when ctx is cancelled. func (hc *HealthChecker) Start(ctx context.Context) { go func() { @@ -82,15 +302,23 @@ func (hc *HealthChecker) Start(ctx context.Context) { }() } +// applyHealthFunc is a pool's copy-on-write health writer. +type applyHealthFunc func(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) + +// applyCapabilitiesFunc is a pool's copy-on-write capability writer. +type applyCapabilitiesFunc func(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) + func (hc *HealthChecker) checkAll(ctx context.Context) { var wg sync.WaitGroup - check := func(n *Node, applyHealth func(int, bool, int, int, time.Time)) { + check := func(n *Node, applyHealth applyHealthFunc, applyCapabilities applyCapabilitiesFunc) { wg.Go(func() { - healthy, activeJobs, egressKbps := CheckNode(ctx, n) + healthy, activeJobs, egressKbps, capabilitiesHash, lastStats := CheckNode(ctx, n) // Publish the result through the pool lock so readers never see - // a Node struct mutated in place (the pool swaps in a copy). - applyHealth(n.ID, healthy, activeJobs, egressKbps, time.Now()) + // a Node struct mutated in place (the pool swaps in a copy). Fenced + // on the checked URL, like the database write below: the pool can be + // reloaded with a different worker on this id while the check runs. + applyHealth(n.ID, n.URL, healthy, activeJobs, egressKbps, capabilitiesHash, lastStats, time.Now()) if n.Healthy && !healthy { slog.WarnContext(ctx, "stream node unhealthy", "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) @@ -99,17 +327,840 @@ func (hc *HealthChecker) checkAll(ctx context.Context) { } if hc.repo != nil { - if err := hc.repo.UpdateHealth(ctx, n.ID, healthy, activeJobs, egressKbps); err != nil { - slog.ErrorContext(ctx, "failed to persist node health", "component", "nodepool", "id", n.ID, "error", err) + // Fenced on the URL that was checked: last_stats feeds transcode + // admission, so one worker's disk reading must never land on a + // row an administrator has since repointed at another. + if err := hc.repo.UpdateHealth(ctx, n.ID, n.URL, healthy, activeJobs, egressKbps, lastStats); err != nil { + if errors.Is(err, ErrNodeMoved) { + slog.InfoContext(ctx, "discarded a health result for a node that changed identity mid-check", + "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) + } else { + slog.ErrorContext(ctx, "failed to persist node health", "component", "nodepool", "id", n.ID, "error", err) + } } } + + if healthy && capabilitiesHash != "" && capabilitiesHash != storedCapabilitiesHash(n) { + hc.startCapabilityRefresh(ctx, n, applyCapabilities) + } }) } for _, n := range hc.proxyPool.Nodes() { - check(n, hc.proxyPool.ApplyHealth) + check(n, hc.proxyPool.ApplyHealth, hc.proxyPool.ApplyCapabilities) } for _, n := range hc.transcodePool.Nodes() { - check(n, hc.transcodePool.ApplyHealth) + check(n, hc.transcodePool.ApplyHealth, hc.transcodePool.ApplyCapabilities) } wg.Wait() } + +// startCapabilityRefresh runs one node's capability fetch off the sweep's +// WaitGroup, deduplicated per node id. +// +// The fetch budget is larger than the sweep interval by design (a cold node +// runs ffmpeg probes to answer), so waiting for it inside the sweep would let +// one slow node stretch every other node's health cadence past that interval — +// and pool health is what routes streams away from a node that died. The +// in-flight guard is what keeps the detached fetches from stacking up: without +// it, a node that answers slower than the interval would collect one goroutine +// per sweep, since a fetch that has not completed cannot have moved the stored +// hash that triggered it. +func (hc *HealthChecker) startCapabilityRefresh(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) { + if fetch, _ := hc.hooks(); fetch == nil { + return + } + if _, loaded := hc.capabilityRefreshInFlight.LoadOrStore(n.ID, struct{}{}); loaded { + return + } + hc.capabilityRefreshes.Add(1) + go func() { + defer hc.capabilityRefreshes.Done() + defer hc.capabilityRefreshInFlight.Delete(n.ID) + // Errors are already logged inside; the sweep has no caller to report to. + _ = hc.refreshCapabilities(ctx, n, applyCapabilities) + }() +} + +// waitForCapabilityRefreshes blocks until every detached capability fetch +// started so far has finished. Callers must not hold the sweep open on it. +func (hc *HealthChecker) waitForCapabilityRefreshes() { + hc.capabilityRefreshes.Wait() +} + +// RefreshNodeCapabilities fetches, stores, and publishes one node's capability +// report immediately, on the caller's goroutine, using exactly the machinery the +// background sweep uses. +// +// It exists for the operator-triggered re-probe: the node has just recomputed +// its inventory, and waiting up to a sweep interval for the API to notice would +// make the action look like it did nothing. Every rule the sweep applies still +// applies here — drift is computed and persisted the same way, a report without +// a hash is refused, a failed fetch leaves the stored row alone — so there is no +// second, divergent persist path. +// +// It participates in the sweep's per-node in-flight guard, so a manual refresh +// and a sweep refresh of the same node cannot run at once; the loser reports +// ErrCapabilityRefreshInFlight rather than starting a duplicate fetch. +func (hc *HealthChecker) RefreshNodeCapabilities(ctx context.Context, n *Node) error { + if hc == nil || n == nil { + return errors.New("no node health checker configured") + } + if fetch, _ := hc.hooks(); fetch == nil { + return errors.New("no node capability fetcher configured") + } + if _, loaded := hc.capabilityRefreshInFlight.LoadOrStore(n.ID, struct{}{}); loaded { + return ErrCapabilityRefreshInFlight + } + defer hc.capabilityRefreshInFlight.Delete(n.ID) + return hc.refreshCapabilities(ctx, n, hc.applyCapabilitiesFor(n)) +} + +// ErrCapabilityRefreshInFlight reports that a capability refresh for the node +// was already running, so the caller's own refresh was not started. The report +// in flight is at least as fresh as the one the caller wanted. +var ErrCapabilityRefreshInFlight = errors.New("node capability refresh already in flight") + +// adoptStoredCapabilities brings this replica's in-memory node up to whatever +// is stored, after another writer won the capability compare-and-set. +// +// Without it the losing replica keeps the pre-fetch hash in memory, compares +// against it on every sweep, refetches, and loses the same write again — while +// its pools serve GPU identities and a drift note that the row has moved past. +// Reading the row is cheap next to the capability fetch that preceded it. +func (hc *HealthChecker) adoptStoredCapabilities( + ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc, onChanged func(string), +) { + if hc.repo == nil { + return + } + stored, err := hc.repo.GetByID(ctx, n.ID) + if err != nil || stored == nil || stored.CapabilitiesHash == nil { + return + } + refreshedAt := time.Now() + if stored.CapabilitiesRefreshedAt != nil { + refreshedAt = *stored.CapabilitiesRefreshedAt + } + if applyCapabilities != nil { + applyCapabilities(stored.ID, stored.URL, stored.Capabilities, *stored.CapabilitiesHash, + refreshedAt, stored.CapabilityDrift, stored.CapabilityDriftBaseline) + } + // The planning cache is per replica, so the winner's invalidation did not + // reach this one. A capability change is exactly what it exists to hear. + if onChanged != nil && !sameOptionalHash(n.CapabilitiesHash, stored.CapabilitiesHash) { + onChanged(stored.URL) + } +} + +// sameOptionalHash compares two stored capability hashes, treating absence as a +// value rather than as a match. +func sameOptionalHash(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +// applyCapabilitiesFor returns the pool writer for a node's type, or nil when +// this checker has no pool for it — which is the normal state for a node row +// that is disabled and therefore in no pool. +func (hc *HealthChecker) applyCapabilitiesFor(n *Node) applyCapabilitiesFunc { + switch n.Type { + case NodeTypeProxy: + if hc.proxyPool != nil { + return hc.proxyPool.ApplyCapabilities + } + case NodeTypeTranscode: + if hc.transcodePool != nil { + return hc.transcodePool.ApplyCapabilities + } + } + return nil +} + +// refreshCapabilities fetches and stores one node's capability report. A +// failure leaves the stored row alone and is retried on the next sweep, because +// a fetch that failed is no evidence about what the node has. +// +// The returned error is for a caller that asked for this refresh explicitly; the +// sweep ignores it, since every failure is already logged here. +func (hc *HealthChecker) refreshCapabilities(ctx context.Context, n *Node, applyCapabilities applyCapabilitiesFunc) error { + fetch, onChanged := hc.hooks() + if fetch == nil { + return nil + } + fetchCtx, cancel := context.WithTimeout(ctx, hc.capabilityFetchBackstop(n)) + defer cancel() + payload, hash, err := fetch(fetchCtx, n) + if err != nil { + slog.WarnContext(ctx, "node capability fetch failed", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, "error", err) + return err + } + if hash == "" || len(payload) == 0 { + // A hash is what makes the payload trackable; storing one without it + // would refetch every sweep forever. Never synthesize one here — the + // node is the only thing that knows what it hashed. + slog.WarnContext(ctx, "node capability report carries no hash", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + return errors.New("node capability report carries no hash") + } + + // Computed before the write, because the comparison is against the report + // this one replaces, and stored with it so a reader never sees a note + // describing a different payload. + // + // Not for a proxy. Drift is a statement about transcode hardware, and a + // proxy's report deliberately carries no hardware inventory at all: it + // relays streams and runs remux recipes, so it advertises transformations + // and nothing else. Comparing a hardware-free report against one an older + // build stored with render devices in it reads as every device disappearing + // at once, and nothing can ever clear that note — recovery is evidenced by + // probes the proxy will never run again. So drift is skipped and both + // columns are written nil, which also erases whatever an older build latched. + drift, parsed := computeCapabilityDrift(n.Capabilities, payload) + var note *string + var driftBaseline []byte + if n.Type != NodeTypeProxy { + note, driftBaseline = resolveDriftNote(n.CapabilityDrift, n.CapabilityDriftBaseline, drift, parsed, payload) + } + refreshedAt := time.Now() + if hc.repo != nil { + // Fenced on the URL this payload was fetched from — the fetch is + // detached and bounded, so the row may since have been repointed at a + // different worker — and on the report it is replacing, so a slower + // sweep on another replica cannot land an older report on top of a + // newer one. + if err := hc.repo.UpdateCapabilities(ctx, n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline, n.CapabilitiesHash); err != nil { + if errors.Is(err, ErrCapabilitiesSuperseded) { + // Another replica stored a report first. Its answer is the + // current one, so this replica adopts the row rather than + // discarding and retrying: left alone it would keep comparing + // against a hash the row no longer has, lose the same write + // every sweep, and serve stale GPU identities and drift state + // until something unrelated reloaded the pools. + hc.adoptStoredCapabilities(ctx, n, applyCapabilities, onChanged) + return err + } + if errors.Is(err, ErrNodeMoved) { + // Not a failure to report loudly: the node was edited or + // removed while this fetch ran, and the next sweep will fetch + // whatever the row addresses now. + slog.InfoContext(ctx, "discarded a capability report the row no longer expects", + "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL) + return err + } + slog.WarnContext(ctx, "failed to persist node capabilities", "component", "nodepool", + "id", n.ID, "name", n.Name, "error", err) + return err + } + } + logCapabilityChange(ctx, n, drift, parsed) + if applyCapabilities != nil { + applyCapabilities(n.ID, n.URL, payload, hash, refreshedAt, note, driftBaseline) + } + if onChanged != nil { + onChanged(n.URL) + } + return nil +} + +func storedCapabilitiesHash(n *Node) string { + if n == nil || n.CapabilitiesHash == nil { + return "" + } + return *n.CapabilitiesHash +} + +// capabilityDriftView is the minimal projection this package parses out of an +// otherwise opaque capability payload. It is deliberately local and partial: +// nodepool must not depend on playback, and drift only needs to know which +// backends were verified and which render devices existed. +type capabilityDriftView struct { + Resolved string `json:"resolved"` + DetectedBackends []driftBackendView `json:"detected_backends"` + RenderDevices []string `json:"render_devices"` + // RenderDeviceDetails carries each device's stable identity. Comparing + // enumeration paths alone reports a GPU as gone whenever DRM hands out a + // different renderD number, which a reboot is free to do; the uuid and the + // PCI slot survive that. + RenderDeviceDetails []struct { + Path string `json:"path"` + PCIAddress string `json:"pci_address"` + GPUUUID string `json:"gpu_uuid"` + } `json:"render_device_details"` + // NVIDIAGPUUUIDs is where an NVIDIA card's identity lives when it has no + // readable DRM node — the ordinary NVENC container: /dev/nvidia* and the + // toolkit, no /dev/dri. Such a node reports no render devices at all, so + // without this a card disappearing from it moves nothing this comparison + // looks at, and the backend comparison does not cover it either: NVENC stops + // being a candidate the moment /dev/nvidia* goes away, and an absent backend + // is deliberately not a lost one. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` +} + +// driftBackendView is one backend's probe outcome as drift reads it. +type driftBackendView struct { + Backend string `json:"backend"` + Verified bool `json:"verified"` + // Skipped reports that no probe was attempted because the node cannot open + // any of the backend's candidate devices. That is a statement about access, + // not about hardware, so it never counts as a failure. + Skipped bool `json:"skipped"` +} + +// nvidiaBackend is the backend name whose presence in a report means the node +// still has NVIDIA device nodes; detection drops it as a candidate when they go +// away. +const nvidiaBackend = "nvenc" + +// nvidiaIdentityBlind reports that this report cannot name the node's NVIDIA +// cards even though the node still has them. +// +// nvidia-smi is queried behind a circuit breaker and can be absent from an +// image entirely, so an empty uuid list is not by itself evidence that anything +// is gone — the same "identity strength is not constant" problem renderDeviceAliases +// exists for. What separates the two is the backend list: NVENC is only probed +// where /dev/nvidia* can be opened, so a report that still carries NVENC and no +// uuids is a node whose cards are present and whose query tool is not. +func (v capabilityDriftView) nvidiaIdentityBlind() bool { + if len(v.NVIDIAGPUUUIDs) > 0 { + return false + } + return slices.ContainsFunc(v.DetectedBackends, func(backend driftBackendView) bool { + return backend.Backend == nvidiaBackend + }) +} + +// renderDeviceAliases lists every stable name each device in a report answers +// to, alongside the path an operator recognizes it by. +// +// All of them, not just the strongest: a report's identity strength is not +// constant. nvidia-smi is queried behind a circuit breaker and may be missing +// on one pass and present on the next, so the same NVIDIA card alternates +// between publishing a PCI address alone and publishing a uuid as well. Keeping +// only the strongest name would make those two reports describe different +// devices and persist a "render device gone" note for a card that never moved — +// the same false positive as comparing enumeration paths, one level up. Two +// reports describe the same device when they share any alias. +type renderDeviceAliases struct { + // path is what the note names the device by; it is the least stable of the + // aliases, which is why it is display only. + path string + // uuid is the card's permanent identity where it published one. It is held + // apart from the aliases because it is the only name that can prove two + // devices are *different*: a slot and a render path are properties of the + // machine and outlive the card in them. + uuid string + aliases []string + // nvidiaOnly marks a card known only through nvidia-smi, with no render + // node behind it. Its identity depends on a tool that comes and goes, so a + // report that lost it has to be read differently — see nvidiaIdentityBlind. + nvidiaOnly bool +} + +// sameDevice reports whether two reports describe one card. +// +// Two permanent uuids that disagree settle it on their own — a replacement card +// in the same slot keeps the slot's PCI address and usually the same render +// path, and letting those weaker names match would hide the old card's +// disappearance entirely. Only when at least one side published no uuid does a +// shared weaker alias stand in for one. +func (a renderDeviceAliases) sameDevice(b renderDeviceAliases) bool { + if a.uuid != "" && b.uuid != "" { + return a.uuid == b.uuid + } + return slices.ContainsFunc(a.aliases, func(alias string) bool { + return slices.Contains(b.aliases, alias) + }) +} + +func renderDeviceAliasSets(view capabilityDriftView) []renderDeviceAliases { + devices := make([]renderDeviceAliases, 0, len(view.RenderDevices)+len(view.NVIDIAGPUUUIDs)) + covered := make(map[string]bool, len(view.RenderDeviceDetails)) + coveredUUIDs := make(map[string]bool, len(view.RenderDeviceDetails)+len(view.NVIDIAGPUUUIDs)) + for _, device := range view.RenderDeviceDetails { + entry := renderDeviceAliases{path: device.Path, uuid: device.GPUUUID} + for _, alias := range []string{device.GPUUUID, device.PCIAddress, device.Path} { + if alias != "" && !slices.Contains(entry.aliases, alias) { + entry.aliases = append(entry.aliases, alias) + } + } + if len(entry.aliases) == 0 { + continue + } + covered[device.Path] = true + if device.GPUUUID != "" { + coveredUUIDs[device.GPUUUID] = true + } + devices = append(devices, entry) + } + // A report that lists paths without details (a node predating them) still + // has to be comparable, so any uncovered path stands for itself. + for _, path := range view.RenderDevices { + if path == "" || covered[path] { + continue + } + devices = append(devices, renderDeviceAliases{path: path, aliases: []string{path}}) + } + // A card nvidia-smi named and no render node covers. Deduplicated against + // the details above by uuid, because a card with both a DRM node and an + // nvidia-smi entry is one card, not two. It gets no path: the uuid is the + // only name it has, and lostRenderDevices falls back to naming it by that. + for _, uuid := range view.NVIDIAGPUUUIDs { + if uuid == "" || coveredUUIDs[uuid] { + continue + } + coveredUUIDs[uuid] = true + devices = append(devices, renderDeviceAliases{uuid: uuid, aliases: []string{uuid}, nvidiaOnly: true}) + } + return devices +} + +// lostRenderDeviceEntries returns the devices in previous that nothing in +// current answers to, with their full alias sets: the note displays the path, +// while the drift baseline keeps every identity so the device can be recognized +// when it comes back under a different one. +func lostRenderDeviceEntries(previous, current capabilityDriftView) []renderDeviceAliases { + currentDevices := renderDeviceAliasSets(current) + blind := current.nvidiaIdentityBlind() + var lost []renderDeviceAliases + for _, device := range renderDeviceAliasSets(previous) { + if slices.ContainsFunc(currentDevices, device.sameDevice) { + continue + } + if blind && device.nvidiaOnly { + // The node still has its NVIDIA devices; only the tool that names + // them is missing. Latching a note here would demand a uuid come + // back that nothing on the node can currently produce. + continue + } + lost = append(lost, device) + } + return lost +} + +// lostRenderDevices names the devices in previous that nothing in current +// answers to. +func lostRenderDevices(previous, current capabilityDriftView) []string { + entries := lostRenderDeviceEntries(previous, current) + lost := make([]string, 0, len(entries)) + for _, device := range entries { + name := device.path + if name == "" { + name = device.aliases[0] + } + lost = append(lost, name) + } + // The order devices are reported in is incidental, and this note is + // persisted and compared against the one it replaces. + slices.Sort(lost) + return slices.Compact(lost) +} + +// capabilityDrift is what a refetch lost relative to the report it replaces. +// The zero value means nothing was lost, which is also what a node's very first +// report produces. +type capabilityDrift struct { + // first reports that there was no stored report to compare against. + first bool + // lostBackends are backends that used to pass their probe and no longer do. + lostBackends []string + // lostDevices are render devices present in the previous report and absent + // from this one. + lostDevices []string + // lostDeviceAliases carries every identity each lost device answered to, so + // the drift baseline can recognize it if it comes back under another one. + lostDeviceAliases []renderDeviceAliases + // previousResolved and resolved are the backend each report resolved to; + // carried for the log line, which is where an operator reads the effect. + previousResolved string + resolved string +} + +// regressed reports whether this refetch lost something worth telling an +// operator about. +func (d capabilityDrift) regressed() bool { + return len(d.lostBackends) > 0 || len(d.lostDevices) > 0 +} + +// maxCapabilityDriftNoteBytes bounds the stored note. The inputs are a node's +// own device and backend lists, so an honest note is a line long; the bound is +// only there because the lists come from a worker that may run elsewhere, and a +// text column echoed to every admin listing nodes is not the place to trust +// them. +const maxCapabilityDriftNoteBytes = 512 + +// persistedNote renders this refetch's regression for the +// stream_nodes.capability_drift column, or nil when this refetch lost nothing. +// nil is not by itself a reason to clear a note the node already carries — see +// resolveDriftNote, which owns that decision. +func (d capabilityDrift) persistedNote() *string { + if !d.regressed() { + return nil + } + parts := make([]string, 0, 3) + if len(d.lostBackends) > 0 { + parts = append(parts, "verified hardware backends lost: "+strings.Join(d.lostBackends, ", ")) + } + if len(d.lostDevices) > 0 { + parts = append(parts, "render devices gone: "+strings.Join(d.lostDevices, ", ")) + } + if d.previousResolved != d.resolved { + parts = append(parts, "resolved backend "+d.previousResolved+" -> "+d.resolved) + } + note := truncateDriftNote(strings.Join(parts, "; ")) + return ¬e +} + +// truncateDriftNote bounds a note to maxCapabilityDriftNoteBytes without ever +// cutting a rune in half. +// +// The column is Postgres text, which rejects invalid UTF-8 outright, and a +// rejected UPDATE takes capabilities and capabilities_hash down with it — the +// stored hash then never advances, so every later sweep refetches and fails +// again. Slicing bytes is exactly how the untrusted device and backend names +// this bound exists to contain would produce that. +func truncateDriftNote(note string) string { + if len(note) <= maxCapabilityDriftNoteBytes { + return note + } + cut := note[:maxCapabilityDriftNoteBytes] + // A trailing partial sequence decodes as RuneError with size 1; a real + // U+FFFD in the input decodes with size 3 and is left alone. + for len(cut) > 0 { + r, size := utf8.DecodeLastRuneInString(cut) + if r != utf8.RuneError || size > 1 { + break + } + cut = cut[:len(cut)-1] + } + return cut + "..." +} + +// resolveDriftNote decides what stream_nodes.capability_drift should say after +// this refetch, given the note it already carries. +// +// Setting the note is a delta — a regression against the report being replaced. +// Clearing it must not be, because a delta against an already-degraded report +// finds nothing newly lost and would erase a standing regression on the next +// unrelated hash change: a reboot moves boot_id, a reworded FFmpeg failure moves +// the probe reason, and the operator-triggered re-probe refetches +// unconditionally. That would tell an operator the node recovered while its +// backend is still failing its probe, which is the one reading this column must +// never produce. So the note is latched, and only a report whose probes all pass +// clears it. +func resolveDriftNote(stored *string, storedBaseline []byte, drift capabilityDrift, parsed bool, payload []byte) (*string, []byte) { + outstanding := mergeDriftBaseline(storedBaseline, drift) + if drift.regressed() { + // The note names everything still outstanding, not just this pass's + // delta. Both accumulate, and if only the baseline did, two GPUs going + // one at a time would leave the operator reading about the second while + // the note stayed latched — after that one returned — for a first loss + // nothing on screen ever mentioned. + note := outstanding.note(drift) + return ¬e, marshalDriftBaseline(outstanding) + } + if stored == nil || strings.TrimSpace(*stored) == "" { + return nil, nil + } + if !parsed || !hardwareProbesEvidenced(payload) { + // Nothing new was lost, but this report is not evidence of recovery. + return stored, marshalDriftBaseline(outstanding) + } + if outstanding.empty() && !hardwareProbesClean(payload) { + // A note written before baselines existed names nothing to wait for, so + // a wholly clean report is the only evidence available for it. + return stored, marshalDriftBaseline(outstanding) + } + if !outstanding.recoveredBy(payload) { + // Recovery is the *originally lost* hardware coming back, not the + // inventory merely looking healthy. A surviving sibling probes just as + // cleanly with its partner still missing, and an unrelated GPU added + // later is growth without repair. Only the baseline can tell those from + // a genuine return, which is why it is kept rather than re-derived: once + // the degraded report is stored, every later comparison is + // degraded-to-degraded and finds nothing at all. + return stored, marshalDriftBaseline(outstanding) + } + return nil, nil +} + +// driftBaseline is the hardware a standing capability_drift note is waiting on. +type driftBaseline struct { + // Backends must verify again. + Backends []string `json:"backends,omitempty"` + // Devices are the cards that must reappear, each carrying every name it + // answered to so a renumbered render node — or a pass where nvidia-smi did + // not answer — still matches it. + Devices []driftBaselineDevice `json:"devices,omitempty"` +} + +// driftBaselineDevice is one lost card's identity, in the same shape the loss +// comparison uses so both sides apply the same matching rule. +type driftBaselineDevice struct { + // UUID is the card's permanent identity where it published one. It is held + // apart from Aliases because it is the only name that can prove two devices + // are *different*: a replacement in the same slot inherits the slot and + // usually the render path, and matching on those would read as the lost + // card returning. + UUID string `json:"uuid,omitempty"` + Aliases []string `json:"aliases,omitempty"` +} + +func (d driftBaselineDevice) matches(candidate renderDeviceAliases) bool { + return renderDeviceAliases{uuid: d.UUID, aliases: d.Aliases}.sameDevice(candidate) +} + +func (b driftBaseline) empty() bool { return len(b.Backends) == 0 && len(b.Devices) == 0 } + +// note renders what this baseline is still waiting on, plus the resolution +// change this pass saw. +// +// The hardware half comes from the baseline so the visible text and the latch +// agree: an operator watching the note disappear should be watching the same +// thing the clearing rule is watching. The resolution transition comes from the +// delta, because "qsv -> none" describes this refresh rather than a standing +// debt, and the previous pass's transition is stale once another has happened. +func (b driftBaseline) note(drift capabilityDrift) string { + parts := make([]string, 0, 3) + if len(b.Backends) > 0 { + parts = append(parts, "verified hardware backends lost: "+strings.Join(b.Backends, ", ")) + } + if names := b.deviceNames(); len(names) > 0 { + parts = append(parts, "render devices gone: "+strings.Join(names, ", ")) + } + if drift.previousResolved != drift.resolved { + parts = append(parts, "resolved backend "+drift.previousResolved+" -> "+drift.resolved) + } + return truncateDriftNote(strings.Join(parts, "; ")) +} + +// deviceNames picks one readable name per outstanding device: the render path an +// operator would recognize, else the uuid, else whatever alias there is. +func (b driftBaseline) deviceNames() []string { + names := make([]string, 0, len(b.Devices)) + for _, device := range b.Devices { + name := device.UUID + for _, alias := range device.Aliases { + if strings.HasPrefix(alias, "/dev/") { + name = alias + break + } + if name == "" { + name = alias + } + } + if name != "" { + names = append(names, name) + } + } + return names +} + +// recoveredBy reports whether every backend and device the note is waiting on is +// accounted for in this report. +func (b driftBaseline) recoveredBy(payload []byte) bool { + if b.empty() { + // Nothing recorded to wait for — a note written before baselines + // existed. A clean report is the best evidence available, and holding + // such a note forever would strand it. + return true + } + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false + } + verified := make(map[string]bool, len(current.DetectedBackends)) + for _, backend := range current.DetectedBackends { + verified[backend.Backend] = backend.Verified + } + for _, backend := range b.Backends { + if !verified[backend] { + return false + } + } + // Matched with sameDevice, not by any shared alias: a replacement card in + // the same slot shares the PCI address and usually the render path, and + // treating that as the lost card returning is exactly the false recovery + // the loss side already refuses to call a match. + currentDevices := renderDeviceAliasSets(current) + for _, device := range b.Devices { + if !slices.ContainsFunc(currentDevices, device.matches) { + return false + } + } + return true +} + +// mergeDriftBaseline adds this refetch's losses to whatever the note was already +// waiting on. +func mergeDriftBaseline(stored []byte, drift capabilityDrift) driftBaseline { + var baseline driftBaseline + if len(stored) > 0 { + // An unreadable baseline is treated as absent rather than as a reason to + // discard the losses this pass found. + _ = json.Unmarshal(stored, &baseline) + } + for _, backend := range drift.lostBackends { + if !slices.Contains(baseline.Backends, backend) { + baseline.Backends = append(baseline.Backends, backend) + } + } + for _, device := range drift.lostDeviceAliases { + if slices.ContainsFunc(baseline.Devices, func(existing driftBaselineDevice) bool { + return existing.matches(device) + }) { + continue + } + baseline.Devices = append(baseline.Devices, driftBaselineDevice{ + UUID: device.uuid, + Aliases: slices.Clone(device.aliases), + }) + } + slices.Sort(baseline.Backends) + return baseline +} + +func marshalDriftBaseline(baseline driftBaseline) []byte { + if baseline.empty() { + return nil + } + encoded, err := json.Marshal(baseline) + if err != nil { + return nil + } + return encoded +} + +// hardwareProbesClean reports whether the node probed at least one backend and +// every backend it probed passed. A skipped backend is not a failure — it means +// the node cannot open the devices, which is the normal reading for a proxy +// pointed at a cluster-wide hw_device — and a report that cannot be parsed is +// not evidence of anything. +// +// The "at least one" half matters as much as the "every" half. A GPU that +// disappears completely leaves a report with no detected_backends at all, and a +// loop over an empty list finds no failure: taking that as clean would clear a +// standing drift note on the next unrelated hash change — a reboot moving +// boot_id is enough — and tell an operator the node recovered while its card is +// still gone. Recovery has to be evidenced by hardware that was actually +// probed, not by the absence of anything to probe. +func hardwareProbesClean(payload []byte) bool { + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false + } + probed := false + for _, backend := range current.DetectedBackends { + if backend.Skipped { + continue + } + if !backend.Verified { + return false + } + probed = true + } + return probed +} + +// hardwareProbesEvidenced reports whether this report contains probe evidence at +// all: at least one backend that was actually probed rather than skipped. +// +// It is the weaker of the two, and it is what a note with a baseline is judged +// against. Requiring every backend to verify latches a note forever on a node +// that carries one which has never worked — VAAPI failing beside a working QSV +// is an ordinary mixed host — even after the render device the note is about +// comes back and the backend that used it verifies again. Which backends had to +// return is the baseline's question, and recoveredBy answers it precisely; this +// only rules out the empty report, where a loop over no backends finds no +// failure and would otherwise read as recovery. +func hardwareProbesEvidenced(payload []byte) bool { + var current capabilityDriftView + if json.Unmarshal(payload, ¤t) != nil { + return false + } + for _, backend := range current.DetectedBackends { + if !backend.Skipped { + return true + } + } + return false +} + +// computeCapabilityDrift compares the report a node just served against the one +// it replaces. parsed is false when either payload cannot be read, in which case +// the drift is unknown rather than empty and callers must not treat it as +// evidence of recovery. +func computeCapabilityDrift(stored, payload []byte) (drift capabilityDrift, parsed bool) { + if len(stored) == 0 { + return capabilityDrift{first: true}, true + } + var previous, current capabilityDriftView + if json.Unmarshal(stored, &previous) != nil || json.Unmarshal(payload, ¤t) != nil { + return capabilityDrift{}, false + } + drift.previousResolved = previous.Resolved + drift.resolved = current.Resolved + // Skipped is carried, not flattened into "not verified": it means no probe + // ran because the node cannot open the backend's configured devices, which + // is a statement about access rather than about hardware. Treating it as a + // loss also contradicted hardwareProbesClean, which counts a skipped + // backend as clean — the note would be set by one rule and cleared by the + // other on the next hash change, flapping without anything having changed. + type probeOutcome struct{ verified, skipped bool } + now := make(map[string]probeOutcome, len(current.DetectedBackends)) + for _, backend := range current.DetectedBackends { + now[backend.Backend] = probeOutcome{verified: backend.Verified, skipped: backend.Skipped} + } + for _, backend := range previous.DetectedBackends { + if !backend.Verified { + continue + } + outcome, reported := now[backend.Backend] + // Absent is not lost. Detection only probes the backends the configured + // hw_device gives it candidates for, so a backend vanishes from the + // report both when its hardware went away *and* when an operator simply + // pointed the node at something else — moving hw_device from a QSV + // render path to an NVENC index legitimately stops QSV being reported. + // Latching drift on that demands a backend verify again that the node is + // deliberately no longer configured for, which nothing can ever satisfy. + // Hardware actually disappearing shows up in render_devices, which is + // the host's own inventory and owes nothing to the configuration, and is + // compared separately below. + if !reported || outcome.verified || outcome.skipped { + continue + } + drift.lostBackends = append(drift.lostBackends, backend.Backend) + } + drift.lostDevices = lostRenderDevices(previous, current) + drift.lostDeviceAliases = lostRenderDeviceEntries(previous, current) + return drift, true +} + +// logCapabilityChange records what changed between the stored report and the +// new one. Losing a verified backend or a render device is the case worth +// waking an operator for: it means a node that was picked for hardware work +// silently became less capable, which otherwise only shows up as slow or +// failing transcodes. +func logCapabilityChange(ctx context.Context, n *Node, drift capabilityDrift, parsed bool) { + if !parsed { + return + } + if drift.first { + slog.InfoContext(ctx, "node capabilities stored", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + return + } + if !drift.regressed() || n.Type == NodeTypeProxy { + // See refreshCapabilities: a proxy reports no hardware, so a "lost" + // backend or device here is the old report's inventory going away rather + // than the node's, and warning about it would page an operator for an + // upgrade. + return + } + slog.WarnContext(ctx, "node capability drift", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL, + "lost_verified_backends", drift.lostBackends, "lost_render_devices", drift.lostDevices, + "previous_resolved", drift.previousResolved, "resolved", drift.resolved) +} diff --git a/internal/nodepool/health_drift_proxy_test.go b/internal/nodepool/health_drift_proxy_test.go new file mode 100644 index 000000000..a5909f829 --- /dev/null +++ b/internal/nodepool/health_drift_proxy_test.go @@ -0,0 +1,67 @@ +package nodepool + +import ( + "context" + "encoding/json" + "testing" +) + +// newProxyCapabilityFixture wires one proxy node into a checker with a fake +// fetcher, the proxy-pool mirror of newCapabilityCheckerFixture. +func newProxyCapabilityFixture(t *testing.T, node *Node, fetcher *fakeCapabilityFetcher) (*HealthChecker, *ProxyPool) { + t.Helper() + proxyPool := NewProxyPool() + proxyPool.SetNodes([]*Node{node}) + checker := NewHealthChecker(proxyPool, NewTranscodePool(), nil) + checker.SetCapabilityFetcher(fetcher.fetch) + return checker, proxyPool +} + +// proxyCapabilityPayload is what a proxy reports now: what its ffmpeg can do, +// and no hardware inventory at all. +const proxyCapabilityPayload = `{"resolved":"none","source":"local",` + + `"transformations":[{"name":"audio_to_aac","recipe_version":"2"}],` + + `"capability_hash":"sha256:proxy-new"}` + +// A proxy never executes a hardware transcode, so its report deliberately +// carries no backends and no render devices. Comparing one against a report an +// older build stored — which walked the host and listed its GPU — finds every +// device gone at once, and nothing could ever clear that note: recovery is +// evidenced by probes this proxy will never run again, so hardwareProbesEvidenced +// is false forever. Drift is a statement about transcode hardware; on a proxy it +// must not be computed at all, and the upgrade must erase what an older build +// latched rather than freeze it. +func TestProxyCapabilityRefreshNeverLatchesDrift(t *testing.T) { + const previousProxyPayload = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}],"capability_hash":"sha256:proxy-old"}` + stale := "render devices gone: /dev/dri/renderD128" + url := newHealthNode(t, "sha256:proxy-new") + fetcher := &fakeCapabilityFetcher{payload: []byte(proxyCapabilityPayload), hash: "sha256:proxy-new"} + node := &Node{ + ID: 1, Name: "proxy-1", Type: NodeTypeProxy, URL: url, Enabled: true, + Capabilities: json.RawMessage(previousProxyPayload), + CapabilitiesHash: stringPtr("sha256:proxy-old"), + CapabilityDrift: &stale, + CapabilityDriftBaseline: json.RawMessage(`{"devices":[{"aliases":["/dev/dri/renderD128"]}]}`), + } + checker, pool := newProxyCapabilityFixture(t, node, fetcher) + + checker.checkAll(context.Background()) + checker.waitForCapabilityRefreshes() + + nodes := pool.Nodes() + if len(nodes) != 1 { + t.Fatalf("pool holds %d nodes, want 1", len(nodes)) + } + stored := nodes[0] + if stored.CapabilityDrift != nil { + t.Fatalf("capability_drift = %q on a proxy, want nothing: a proxy reports no hardware to lose", + *stored.CapabilityDrift) + } + if len(stored.CapabilityDriftBaseline) != 0 { + t.Fatalf("capability_drift_baseline = %s on a proxy, want it cleared", stored.CapabilityDriftBaseline) + } + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:proxy-new" { + t.Fatalf("capabilities_hash = %v, want the fetched report to have landed", stored.CapabilitiesHash) + } +} diff --git a/internal/nodepool/health_drift_test.go b/internal/nodepool/health_drift_test.go new file mode 100644 index 000000000..83f1aa050 --- /dev/null +++ b/internal/nodepool/health_drift_test.go @@ -0,0 +1,855 @@ +package nodepool + +import ( + "context" + "encoding/json" + "strings" + "testing" + "unicode/utf8" +) + +const degradedCapabilityPayload = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false}],"capability_hash":"sha256:degraded"}` + +// The drift log line only reaches an operator who is reading logs. Persisting +// the same finding is what puts it on the node list, so the pool copy has to +// carry it the moment the report is applied. +func TestHealthCheckerCarriesCapabilityDriftIntoThePool(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilityDrift == nil { + t.Fatal("pool copy carries no capability_drift after a regression") + } + note := *stored.CapabilityDrift + for _, want := range []string{"nvenc", "/dev/dri/renderD128", "none"} { + if !strings.Contains(note, want) { + t.Fatalf("drift note %q does not name %q", note, want) + } + } +} + +// A repaired node must stop being flagged. The note describes the last +// comparison, not a latched incident, so a clean report clears it — otherwise a +// one-off driver hiccup would mark a node broken forever. +func TestHealthCheckerClearsCapabilityDriftOnRecovery(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + previousNote := "verified hardware backends lost: nvenc" + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + }, fetcher) + + fixture.sweep() + + if stored := fixture.storedNode(t); stored.CapabilityDrift != nil { + t.Fatalf("capability_drift = %q after recovery, want it cleared", *stored.CapabilityDrift) + } +} + +// A still-degraded report is not a recovery. Drift is a delta, so the refetch +// after a regression finds nothing *newly* lost — and clearing the note there +// would tell an operator the node is fine while its backend is still failing its +// probe. Anything that moves the hash reaches this path: a reboot moves boot_id, +// a reworded FFmpeg failure moves the probe reason. +func TestHealthCheckerKeepsCapabilityDriftWhileTheReportIsStillDegraded(t *testing.T) { + const rebootedPayload = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false,"reason":"no such device"}],` + + `"boot_id":"after-reboot","capability_hash":"sha256:rebooted"}` + url := newHealthNode(t, "sha256:rebooted") + fetcher := &fakeCapabilityFetcher{payload: []byte(rebootedPayload), hash: "sha256:rebooted"} + previousNote := "verified hardware backends lost: nvenc" + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + }, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilityDrift == nil { + t.Fatal("capability_drift cleared by a refetch that found the node still degraded") + } + if *stored.CapabilityDrift != previousNote { + t.Fatalf("capability_drift = %q, want the standing note %q", *stored.CapabilityDrift, previousNote) + } +} + +// The operator-triggered re-probe refetches unconditionally, with no hash gate. +// It is the action the docs and the UI tooltip prescribe for checking whether a +// drift note is still true, so it must be able to answer "yes" — clearing the +// badge on the first click regardless of what the probe found would make the +// only tool for confirming the note the tool that destroys it. +func TestRefreshNodeCapabilitiesKeepsDriftWhenNothingRecovered(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + previousNote := "verified hardware backends lost: nvenc" + node := &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(degradedCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:degraded"), + CapabilityDrift: &previousNote, + } + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err != nil { + t.Fatalf("RefreshNodeCapabilities: %v", err) + } + + if stored := fixture.storedNode(t); stored.CapabilityDrift == nil { + t.Fatal("a re-probe that found the same failing probe cleared the drift note") + } +} + +// Clearing the note requires evidence that hardware came back, and a report in +// which nothing was probed carries none. +// +// The two shapes that produce no passing probe are the two ways hardware goes +// away: a device the node can no longer open reports the backend `skipped`, and +// a card that is gone entirely leaves no candidate backend to report at all. +// Both used to read as clean — the first because a skipped backend is not a +// failure, the second because a loop over an empty list finds none — so a +// standing regression was erased by the next unrelated hash change (a reboot +// moving boot_id is enough), telling an operator a still-broken node had +// recovered. +// +// A proxy pointed at a cluster-wide hw_device does not get stuck behind this: +// it never verified those backends in the first place, so computeCapabilityDrift +// never gives it a note to hold open. +func TestResolveDriftNoteKeepsNoteWhenNoProbePassed(t *testing.T) { + tests := []struct { + name string + payload string + }{ + { + name: "every candidate device is inaccessible", + payload: `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"vaapi","verified":false,"skipped":true}]}`, + }, + { + name: "the gpu is gone, so nothing was a candidate", + payload: `{"resolved":"none","render_devices":[],"detected_backends":[]}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + standing := "verified hardware backends lost: vaapi" + payload := []byte(test.payload) + // Both sides degraded: the delta finds nothing newly lost, which is + // exactly the state in which clearing has to be refused. + drift, parsed := computeCapabilityDrift(payload, payload) + got, _ := resolveDriftNote(&standing, nil, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift was cleared by a report in which no probe passed") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } + }) + } +} + +// The complement: a report that gains back what the stored one lacked is the +// evidence recovery needs, and clears the note. +func TestResolveDriftNoteClearsWhenHardwareComesBack(t *testing.T) { + const degraded = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"vaapi","verified":false,"reason":"no such device"}]}` + const recovered = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true},` + + `{"backend":"qsv","verified":false,"skipped":true}]}` + + standing := "verified hardware backends lost: vaapi" + // The baseline the loss recorded: vaapi has to verify again. + baseline := []byte(`{"backends":["vaapi"]}`) + drift, parsed := computeCapabilityDrift([]byte(degraded), []byte(recovered)) + got, gotBaseline := resolveDriftNote(&standing, baseline, drift, parsed, []byte(recovered)) + if got != nil { + t.Fatalf("capability_drift = %q, want the recovered backend to clear it", *got) + } + if gotBaseline != nil { + t.Fatalf("baseline = %s, want it cleared with the note", gotBaseline) + } +} + +// Growth is not repair. A node standing on a lost GPU that gains an unrelated +// one has a bigger, perfectly clean inventory and still has not got its card +// back — which is why the note keeps what it is waiting for rather than +// re-deriving it from the stored report. +func TestResolveDriftNoteKeepsNoteWhenAnUnrelatedGPUIsAdded(t *testing.T) { + const gained = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD130","pci_address":"0000:09:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + // The lost card, by every identity it answered to. + baseline := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(gained) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, gotBaseline := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared because an unrelated GPU appeared") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } + if len(gotBaseline) == 0 { + t.Fatal("the baseline was dropped while the note still stands") + } +} + +// The lost card coming back under a renumbered render node still counts: the +// baseline keeps every identity it answered to, and any one of them matching +// identifies it. +func TestResolveDriftNoteClearsWhenTheLostCardReturnsRenumbered(t *testing.T) { + const back = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(back) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want the card at the same slot to clear it", *got) + } +} + +// Two cards going one at a time must both have to return: the second loss +// extends the baseline rather than replacing it. +func TestResolveDriftNoteAccumulatesSuccessiveLosses(t *testing.T) { + const twoCards = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128","/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"},` + + `{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + const oneCard = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + const noCards = `{"resolved":"none","render_devices":[],"detected_backends":[]}` + + firstLoss, parsed := computeCapabilityDrift([]byte(twoCards), []byte(oneCard)) + note, baseline := resolveDriftNote(nil, nil, firstLoss, parsed, []byte(oneCard)) + if note == nil || len(baseline) == 0 { + t.Fatalf("first loss produced note=%v baseline=%s", note, baseline) + } + + secondLoss, parsed := computeCapabilityDrift([]byte(oneCard), []byte(noCards)) + note, baseline = resolveDriftNote(note, baseline, secondLoss, parsed, []byte(noCards)) + if note == nil || len(baseline) == 0 { + t.Fatal("second loss dropped the standing note or its baseline") + } + + // Only the first card returns; the note must stand for the second. + if got, _ := resolveDriftNote(note, baseline, capabilityDrift{}, true, []byte(oneCard)); got == nil { + t.Fatal("capability_drift cleared with one of two lost cards still missing") + } + // Both back clears it. + if got, _ := resolveDriftNote(note, baseline, capabilityDrift{}, true, []byte(twoCards)); got != nil { + t.Fatalf("capability_drift = %q, want both cards returning to clear it", *got) + } +} + +// A multi-GPU node that lost one card keeps probing the survivor cleanly, and +// once the degraded report is stored the delta finds nothing lost ever again. +// Clearing on that generic success told an operator the node recovered while +// one of its cards was still missing. +func TestResolveDriftNoteKeepsNoteWhileASiblingGPUIsStillMissing(t *testing.T) { + // Two cards before the loss, one after — and the survivor verifies, so + // every probe that ran passes. + const degraded = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD129" + baseline := []byte(`{"devices":[{"aliases":["0000:04:00.0","/dev/dri/renderD129"]}]}`) + payload := []byte(degraded) + // The next refetch is degraded-to-degraded: nothing newly lost, and every + // probe that ran passed, because the survivor is fine. + drift, parsed := computeCapabilityDrift(payload, payload) + if !hardwareProbesClean(payload) { + t.Fatal("the surviving card should probe cleanly; that is the point") + } + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared while the lost card was still missing") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } +} + +// A node's very first report has nothing to compare against, so it must not be +// flagged. +func TestHealthCheckerStoresNoDriftOnFirstReport(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + }, fetcher) + + fixture.sweep() + + if stored := fixture.storedNode(t); stored.CapabilityDrift != nil { + t.Fatalf("capability_drift = %q on a first report, want none", *stored.CapabilityDrift) + } +} + +// An unreadable payload means the drift is unknown, not that the node recovered. +func TestComputeCapabilityDriftReportsUnparseablePayloads(t *testing.T) { + if _, parsed := computeCapabilityDrift([]byte(testCapabilityPayload), []byte(`not json`)); parsed { + t.Fatal("an unreadable new report parsed") + } + if _, parsed := computeCapabilityDrift([]byte(`not json`), []byte(testCapabilityPayload)); parsed { + t.Fatal("an unreadable stored report parsed") + } + drift, parsed := computeCapabilityDrift(nil, []byte(testCapabilityPayload)) + if !parsed || !drift.first || drift.regressed() { + t.Fatalf("first report drift = %+v, parsed = %v", drift, parsed) + } +} + +// The note is echoed to every admin listing nodes, and its inputs come from a +// worker that may run on remote hardware. +func TestCapabilityDriftNoteIsBounded(t *testing.T) { + drift := capabilityDrift{} + for range 400 { + drift.lostDevices = append(drift.lostDevices, "/dev/dri/renderD128") + } + note := drift.persistedNote() + if note == nil { + t.Fatal("a regression produced no note") + } + if len(*note) > maxCapabilityDriftNoteBytes+3 { + t.Fatalf("note is %d bytes, want it bounded at %d", len(*note), maxCapabilityDriftNoteBytes) + } +} + +// capability_drift is a Postgres text column, which rejects invalid UTF-8 +// outright — and the rejected UPDATE takes capabilities and capabilities_hash +// with it, so the stored hash never advances and every later sweep refetches and +// fails again. Device names come from a worker that may run elsewhere, so the +// bound has to cut on a rune boundary at every alignment, not just the lucky +// ones. +func TestCapabilityDriftNoteStaysValidUTF8AtEveryTruncationOffset(t *testing.T) { + for pad := range 8 { + drift := capabilityDrift{} + for range 40 { + drift.lostDevices = append(drift.lostDevices, + "/dev/dri/"+strings.Repeat("x", pad)+strings.Repeat("é", 12)) + } + note := drift.persistedNote() + if note == nil { + t.Fatalf("pad %d: a regression produced no note", pad) + } + if len(*note) <= maxCapabilityDriftNoteBytes { + t.Fatalf("pad %d: note is %d bytes, the fixture must exceed the bound", pad, len(*note)) + } + if !utf8.ValidString(*note) { + t.Fatalf("pad %d: truncated note is not valid UTF-8: %q", pad, *note) + } + } +} + +// The operator-triggered re-probe stores the node's new report immediately, and +// must go through the same fetch, drift, and publish path the sweep uses rather +// than a second implementation. +func TestRefreshNodeCapabilitiesStoresImmediately(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + fetcher := &fakeCapabilityFetcher{payload: []byte(degradedCapabilityPayload), hash: "sha256:degraded"} + node := &Node{ + ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + } + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err != nil { + t.Fatalf("RefreshNodeCapabilities: %v", err) + } + + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:degraded" { + t.Fatalf("stored hash = %v, want the refetched report", stored.CapabilitiesHash) + } + if stored.CapabilityDrift == nil { + t.Fatal("an immediate refresh did not persist the drift the sweep would have") + } + // The capability cache must be told, exactly as on a sweep refresh. + if notifications := fixture.notifications(); len(notifications) != 1 || notifications[0] != url { + t.Fatalf("notifications = %v, want one for %s", notifications, url) + } + if got := fetcher.callCount(); got != 1 { + t.Fatalf("fetch calls = %d, want exactly one", got) + } +} + +// A refresh already running is at least as fresh as the one being asked for, so +// the second caller is told rather than starting a duplicate fetch. +func TestRefreshNodeCapabilitiesRefusesWhenAlreadyInFlight(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + node := &Node{ID: 1, Name: "gpu-1", Type: NodeTypeTranscode, URL: url, Enabled: true} + fixture := newCapabilityCheckerFixture(t, node, fetcher) + + fixture.checker.capabilityRefreshInFlight.Store(node.ID, struct{}{}) + defer fixture.checker.capabilityRefreshInFlight.Delete(node.ID) + + if err := fixture.checker.RefreshNodeCapabilities(context.Background(), node); err == nil { + t.Fatal("a duplicate refresh was allowed to start") + } + if got := fetcher.callCount(); got != 0 { + t.Fatalf("fetch calls = %d, want none while a refresh is in flight", got) + } +} + +// DRM is free to hand the same card a different renderD number across a reboot. +// Comparing enumeration paths alone then reports a GPU as gone, and because the +// reboot moves boot_id it also triggers the refetch that persists the note — so +// an operator sees a hardware regression for a card that never moved. +func TestComputeCapabilityDriftMatchesRenumberedRenderDevices(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"qsv","render_devices":["/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if drift.regressed() { + t.Fatalf("drift = %+v, want a renumbered path at the same PCI slot to be no regression", drift) + } +} + +// A card that genuinely goes away has neither its path nor its slot in the new +// report, and must still be caught. +func TestComputeCapabilityDriftStillCatchesARemovedDevice(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128","/dev/dri/renderD129"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"},` + + `{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostDevices) != 1 || drift.lostDevices[0] != "/dev/dri/renderD129" { + t.Fatalf("lostDevices = %v, want the card at 0000:04:00.0 reported gone", drift.lostDevices) + } +} + +// An NVIDIA uuid outranks the slot, so a card moved between slots is still the +// same card. +func TestComputeCapabilityDriftMatchesAMovedCardByUUID(t *testing.T) { + const before = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-abc"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD130","pci_address":"0000:07:00.0","gpu_uuid":"GPU-abc"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed || drift.regressed() { + t.Fatalf("drift = %+v (parsed=%v), want the same uuid to be the same card", drift, parsed) + } +} + +// A node that predates render_device_details reports paths only, and must still +// be comparable. +func TestComputeCapabilityDriftFallsBackToPathsWithoutDetails(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"qsv","verified":false}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostDevices) != 1 || drift.lostDevices[0] != "/dev/dri/renderD128" { + t.Fatalf("lostDevices = %v, want the path-only device reported gone", drift.lostDevices) + } +} + +// nvidia-smi is queried behind a circuit breaker, so the same NVIDIA card +// publishes a uuid on one pass and only its PCI address on another. Keeping +// just the strongest identity made those two reports describe different +// devices, persisting a "render device gone" note for a card that never moved. +func TestComputeCapabilityDriftMatchesAcrossIdentityStrength(t *testing.T) { + const withUUID = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-abc"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + const withoutUUID = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + for _, test := range []struct{ name, before, after string }{ + {"uuid disappears", withUUID, withoutUUID}, + {"uuid appears", withoutUUID, withUUID}, + } { + t.Run(test.name, func(t *testing.T) { + drift, parsed := computeCapabilityDrift([]byte(test.before), []byte(test.after)) + if !parsed { + t.Fatal("both reports should parse") + } + if drift.regressed() { + t.Fatalf("drift = %+v, want a shared PCI alias to identify the same card", drift) + } + }) + } +} + +// A replacement card in the same slot keeps the slot's PCI address and usually +// the render path too, so matching on any shared alias would hide the old card's +// disappearance entirely. Two permanent uuids that disagree settle it. +func TestComputeCapabilityDriftReportsAReplacedCardInTheSameSlot(t *testing.T) { + const before = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-old"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-new"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostDevices) != 1 || drift.lostDevices[0] != "/dev/dri/renderD128" { + t.Fatalf("lostDevices = %v, want the replaced card reported gone", drift.lostDevices) + } +} + +// Skipped means no probe ran, because the node cannot open the backend's +// configured devices — a statement about access, not about hardware. Counting +// it as a loss also contradicted hardwareProbesClean, which treats a skipped +// backend as clean: the note would be set by one rule and cleared by the other +// on the next hash change, flapping with nothing having changed. +func TestComputeCapabilityDriftDoesNotTreatASkippedBackendAsLost(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":false,"skipped":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostBackends) != 0 { + t.Fatalf("lostBackends = %v, want a skipped backend not counted as lost", drift.lostBackends) + } + // And the pair round-trips: what does not set the note must not be held + // open by it either. + standing := "verified hardware backends lost: qsv" + if got, _ := resolveDriftNote(&standing, nil, drift, parsed, []byte(after)); got == nil || *got != standing { + t.Fatalf("resolveDriftNote = %v, want a skipped report to leave a standing note alone", got) + } +} + +// A backend that fails its probe outright is still a loss — the distinction is +// "could not try" versus "tried and the driver said no". +func TestComputeCapabilityDriftStillCatchesAFailedBackend(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":false,"reason":"h264_qsv smoke encode failed"}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if len(drift.lostBackends) != 1 || drift.lostBackends[0] != "qsv" { + t.Fatalf("lostBackends = %v, want the failing backend reported", drift.lostBackends) + } +} + +// A GPU that actually disappears is caught by the device comparison, which reads +// the host's own inventory and owes nothing to the configuration. The backend +// going unreported alongside it is a consequence, not separate evidence. +func TestComputeCapabilityDriftCatchesAVanishedGPUAsADeviceLoss(t *testing.T) { + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"none","render_devices":[],"detected_backends":[]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if !drift.regressed() { + t.Fatalf("drift = %+v, want a vanished GPU recorded", drift) + } + if len(drift.lostDevices) != 1 || drift.lostDevices[0] != "/dev/dri/renderD128" { + t.Fatalf("lostDevices = %v, want the vanished card reported", drift.lostDevices) + } +} + +// Detection only probes the backends the configured hw_device gives it +// candidates for, so repointing a node from a QSV render path to an NVENC index +// legitimately stops QSV being reported. Treating that as a disappeared backend +// latched a warning demanding QSV verify again on a node deliberately configured +// for NVENC — which nothing could ever satisfy, so the false incident could +// never clear. +func TestComputeCapabilityDriftIgnoresABackendThePolicyStoppedProbing(t *testing.T) { + // The host's inventory is unchanged; only what was probed moved. + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("both reports should parse") + } + if drift.regressed() { + t.Fatalf("drift = %+v, want a policy change not recorded as hardware loss", drift) + } +} + +// A note written before the baseline column existed has nothing recorded to wait +// for. Holding it forever would strand it on an upgraded deployment, so a clean +// report clears it — the best evidence available for a note whose subject was +// never captured. +func TestResolveDriftNoteClearsALegacyNoteWithNoBaseline(t *testing.T) { + const clean = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"vaapi","verified":true}]}` + + standing := "verified hardware backends lost: vaapi" + payload := []byte(clean) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, nil, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want a baseline-less note cleared by a clean report", *got) + } +} + +// A replacement card in the same slot inherits the PCI address and usually the +// render path. Clearing on that would report the lost card as returned when a +// different one arrived — the same false match the loss comparison already +// refuses, applied to the recovery side. +func TestResolveDriftNoteKeepsNoteWhenAReplacementCardTakesTheSlot(t *testing.T) { + const replaced = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0","gpu_uuid":"GPU-new"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"uuid":"GPU-old","aliases":["GPU-old","0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(replaced) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared because a different card took the slot") + } + if *got != standing { + t.Fatalf("capability_drift = %q, want the standing note %q", *got, standing) + } +} + +// The original card returning does clear it, even under a renumbered path. +func TestResolveDriftNoteClearsWhenTheSameCardReturnsByUUID(t *testing.T) { + const back = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD131"],` + + `"render_device_details":[{"path":"/dev/dri/renderD131","pci_address":"0000:09:00.0","gpu_uuid":"GPU-old"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"uuid":"GPU-old","aliases":["GPU-old","0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(back) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want the same uuid in a new slot to clear it", *got) + } +} + +// A mixed host can carry a backend that has never worked — VAAPI failing beside +// a working QSV is ordinary. Requiring every backend to verify before clearing +// meant a note about a lost render device latched forever, long after that +// device came back and the backend that used it verified again. +func TestResolveDriftNoteClearsDespiteAnUnrelatedFailingBackend(t *testing.T) { + const recovered = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true},{"backend":"vaapi","verified":false,"reason":"no driver"}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"backends":["qsv"],"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + payload := []byte(recovered) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got != nil { + t.Fatalf("capability_drift = %q, want it cleared: the lost device is back and qsv verifies", *got) + } +} + +// The baseline still decides. A backend the note is waiting on that has not come +// back keeps it latched, however healthy the rest of the report looks. +func TestResolveDriftNoteKeepsNoteWhenTheBaselineBackendIsStillFailing(t *testing.T) { + const stillBroken = `{"resolved":"vaapi","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":false,"reason":"no driver"},{"backend":"vaapi","verified":true}]}` + + standing := "backends no longer verifying: qsv" + baseline := []byte(`{"backends":["qsv"]}`) + payload := []byte(stillBroken) + drift, parsed := computeCapabilityDrift(payload, payload) + + got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload) + if got == nil { + t.Fatal("capability_drift cleared while the backend it names is still failing") + } +} + +// A report with nothing probed is not evidence of anything. Every backend +// skipped means the node could not open its configured devices, which says +// nothing about the hardware the note is waiting on. +func TestResolveDriftNoteKeepsNoteWhenEveryBackendWasSkipped(t *testing.T) { + const allSkipped = `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"qsv","skipped":true},{"backend":"vaapi","skipped":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + baseline := []byte(`{"devices":[{"aliases":["/dev/dri/renderD128"]}]}`) + payload := []byte(allSkipped) + drift, parsed := computeCapabilityDrift(payload, payload) + + if got, _ := resolveDriftNote(&standing, baseline, drift, parsed, payload); got == nil { + t.Fatal("capability_drift cleared on a report where nothing was probed") + } +} + +// Two GPUs going one at a time: the note and the latch have to name the same +// thing. Built from the delta alone, the second loss replaced the first in the +// text while the baseline still waited for both — so after the visible GPU came +// back the warning stayed, naming hardware the operator could no longer see. +func TestResolveDriftNoteNamesEveryOutstandingLoss(t *testing.T) { + const both = `{"resolved":"qsv","render_devices":["/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + + standing := "render devices gone: /dev/dri/renderD128" + firstLoss := []byte(`{"devices":[{"aliases":["0000:03:00.0","/dev/dri/renderD128"]}]}`) + // A second card goes while the first is still missing. + const before = `{"resolved":"qsv","render_devices":["/dev/dri/renderD129","/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD129","pci_address":"0000:04:00.0"},` + + `{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + drift, parsed := computeCapabilityDrift([]byte(before), []byte(both)) + if !parsed { + t.Fatal("drift not parsed") + } + + note, baseline := resolveDriftNote(&standing, firstLoss, drift, parsed, []byte(both)) + if note == nil { + t.Fatal("capability_drift cleared while two cards are missing") + } + for _, want := range []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} { + if !strings.Contains(*note, want) { + t.Fatalf("capability_drift = %q, want it to name %s — the baseline is still waiting on it", *note, want) + } + } + + // And the note keeps agreeing with the latch: the first card returning is + // not enough, and what remains is still named. + const firstBack = `{"resolved":"qsv","render_devices":["/dev/dri/renderD128","/dev/dri/renderD130"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"},` + + `{"path":"/dev/dri/renderD130","pci_address":"0000:05:00.0"}],` + + `"detected_backends":[{"backend":"qsv","verified":true}]}` + drift, parsed = computeCapabilityDrift([]byte(firstBack), []byte(firstBack)) + still, _ := resolveDriftNote(note, baseline, drift, parsed, []byte(firstBack)) + if still == nil { + t.Fatal("capability_drift cleared with the second card still missing") + } + if !strings.Contains(*still, "/dev/dri/renderD129") { + t.Fatalf("capability_drift = %q, want the still-missing card named", *still) + } +} + +// The ordinary NVENC container has /dev/nvidia* and the toolkit and no /dev/dri, +// so its cards exist only in nvidia_gpu_uuids. Losing one moves nothing in +// render_devices, and the backend comparison does not cover it either: NVENC +// stops being a candidate the moment the device nodes go away, and an absent +// backend is deliberately not a lost one. +func TestCapabilityDriftNotesAnNVIDIAOnlyCardGoingAway(t *testing.T) { + const before = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-aaa","GPU-bbb"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + // One card left. NVENC still verifies on it, so nothing else in the report + // says anything was lost. + const after = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-bbb"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if !drift.regressed() { + t.Fatal("a card that disappeared produced no drift") + } + if got := drift.lostDevices; len(got) != 1 || got[0] != "GPU-aaa" { + t.Fatalf("lost devices = %v, want the missing GPU-aaa", got) + } + if len(drift.lostBackends) != 0 { + t.Fatalf("lost backends = %v, want none — NVENC still verifies on the remaining card", drift.lostBackends) + } +} + +// nvidia-smi sits behind a circuit breaker and can be missing from an image +// outright, so an empty uuid list is not evidence a card is gone. NVENC is only +// probed where /dev/nvidia* opens, so a report still carrying it describes a +// node whose cards are present and whose query tool is not — latching drift +// there would demand a uuid come back that nothing on the node can produce. +func TestCapabilityDriftIgnoresNVIDIAUUIDsLostWithTheQueryTool(t *testing.T) { + const before = `{"resolved":"nvenc","nvidia_gpu_uuids":["GPU-aaa"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + const after = `{"resolved":"nvenc","detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if drift.regressed() { + t.Fatalf("nvidia-smi going quiet was read as hardware loss: %+v", drift) + } +} + +// A card with both a render node and an nvidia-smi entry is one card. Counting +// it twice would report a device gone whenever the two sources disagree about +// which of them can currently see it. +func TestCapabilityDriftCountsOneCardOnceAcrossBothIdentitySources(t *testing.T) { + const before = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128","gpu_uuid":"GPU-aaa"}],` + + `"nvidia_gpu_uuids":["GPU-aaa"],"detected_backends":[{"backend":"nvenc","verified":true}]}` + // nvidia-smi went quiet; the render node still reports the same card. + const after = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"render_device_details":[{"path":"/dev/dri/renderD128"}],` + + `"detected_backends":[{"backend":"nvenc","verified":true}]}` + + drift, parsed := computeCapabilityDrift([]byte(before), []byte(after)) + if !parsed { + t.Fatal("drift did not parse") + } + if drift.regressed() { + t.Fatalf("one card read as two: %+v", drift) + } +} diff --git a/internal/nodepool/health_stats_test.go b/internal/nodepool/health_stats_test.go new file mode 100644 index 000000000..e2dbed8c9 --- /dev/null +++ b/internal/nodepool/health_stats_test.go @@ -0,0 +1,222 @@ +package nodepool + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// newStatsHealthNode starts a stand-in node whose /health serves a verbatim +// body, so a test can express exactly what an old or new node puts on the wire. +func newStatsHealthNode(t *testing.T, body string) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server.URL +} + +// Node resource stats pass through this package untouched. Parsing them here +// would couple the health sweep to the sampler's schema for no benefit — +// nothing in nodepool routes on them. +func TestCheckNodePassesResourceStatsThroughOpaquely(t *testing.T) { + url := newStatsHealthNode(t, `{ + "status":"ok", + "active_jobs":2, + "egress_kbps":17, + "capabilities_hash":"sha256:abc", + "system":{"cpu_pct":41,"load1":3.2,"cores":16,"mem_used_mb":9011,"mem_total_mb":32768, + "disks":[{"path":"/transcode","used_gb":210,"total_gb":500}], + "net_rx_bps":1200000,"net_tx_bps":98000000}, + "gpu":[{"device":"/dev/dri/renderD128","vendor":"intel","sessions":2, + "video_busy_pct":63,"render_busy_pct":12,"source":"fdinfo", + "future_field_we_do_not_know":true}] + }`) + + healthy, activeJobs, egressKbps, hash, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy || activeJobs != 2 || egressKbps != 17 || hash != "sha256:abc" { + t.Fatalf("check = %v/%d/%d/%q, want the existing fields unchanged", healthy, activeJobs, egressKbps, hash) + } + if len(lastStats) == 0 { + t.Fatal("lastStats is empty; the node reported a sample") + } + + var decoded struct { + System struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []map[string]any `json:"gpu"` + } + if err := json.Unmarshal(lastStats, &decoded); err != nil { + t.Fatalf("lastStats is not valid JSON: %v (%s)", err, lastStats) + } + if decoded.System.CPUPct != 41 { + t.Fatalf("system.cpu_pct = %d, want 41", decoded.System.CPUPct) + } + if len(decoded.GPU) != 1 { + t.Fatalf("gpu = %v, want one device", decoded.GPU) + } + // A newer node adding a field must survive the round trip, which is the + // point of keeping the payload opaque. + if _, ok := decoded.GPU[0]["future_field_we_do_not_know"]; !ok { + t.Fatalf("unknown gpu field was dropped: %s", lastStats) + } +} + +// A node predating resource sampling — and a node that reports an explicit null +// — must both produce nil, which persists as SQL NULL. Writing an empty object +// instead would draw zeros on a dashboard for a node that measured nothing. +func TestCheckNodeReportsNoStatsForOlderNodes(t *testing.T) { + for name, body := range map[string]string{ + "fields absent": `{"status":"ok","active_jobs":1,"egress_kbps":0,"capabilities_hash":""}`, + "fields null": `{"status":"ok","active_jobs":1,"system":null,"gpu":null}`, + "gpu empty only": `{"status":"ok","active_jobs":1,"gpu":null}`, + } { + t.Run(name, func(t *testing.T) { + url := newStatsHealthNode(t, body) + healthy, _, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy { + t.Fatal("node reported unhealthy") + } + if lastStats != nil { + t.Fatalf("lastStats = %s, want nil so the column is written NULL", lastStats) + } + }) + } +} + +// A node that reports only one half still persists that half. +func TestCheckNodeKeepsAPartialSample(t *testing.T) { + url := newStatsHealthNode(t, `{"status":"ok","gpu":[{"device":"cuda:0","source":"nvidia-smi"}]}`) + _, _, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + var decoded map[string]json.RawMessage + if err := json.Unmarshal(lastStats, &decoded); err != nil { + t.Fatalf("lastStats invalid: %v (%s)", err, lastStats) + } + if _, ok := decoded["system"]; ok { + t.Fatalf("system emitted when the node sent none: %s", lastStats) + } + if _, ok := decoded["gpu"]; !ok { + t.Fatalf("gpu missing: %s", lastStats) + } +} + +// A node is a worker that may run on hardware this process does not control, +// and last_stats is the one thing it dictates that gets written to the control +// plane database every 30 seconds. An oversized sample is dropped, not stored — +// but whether the node is alive still routes streams, so the verdict survives. +func TestCheckNodeDropsAnOversizedResourceSample(t *testing.T) { + padding := strings.Repeat("x", maxLastStatsBytes) + url := newStatsHealthNode(t, `{"status":"ok","active_jobs":3,"egress_kbps":9, + "system":{"cpu_pct":41,"junk":"`+padding+`"}}`) + + healthy, activeJobs, egressKbps, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if !healthy || activeJobs != 3 || egressKbps != 9 { + t.Fatalf("check = %v/%d/%d, want the health verdict kept", healthy, activeJobs, egressKbps) + } + if lastStats != nil { + t.Fatalf("lastStats = %d bytes, want the oversized sample dropped", len(lastStats)) + } +} + +// Past the body cap nothing in the response can be trusted to be well formed, +// so the node reads as not answering rather than as partially believed. +func TestCheckNodeRejectsAnOversizedHealthBody(t *testing.T) { + url := newStatsHealthNode(t, `{"status":"ok","active_jobs":3,"junk":"`+ + strings.Repeat("x", maxHealthResponseBytes)+`"}`) + + healthy, activeJobs, _, _, lastStats := CheckNode(context.Background(), &Node{URL: url}) + if healthy || activeJobs != 0 || lastStats != nil { + t.Fatalf("check = %v/%d/%s, want an unreadable body treated as no answer", healthy, activeJobs, lastStats) + } +} + +// An unreachable node reports nothing, and its stats must be cleared rather +// than left behind: a dead node's five-minute-old CPU number reads as live. +func TestApplyHealthClearsStatsWhenACheckCarriesNone(t *testing.T) { + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) + + pool.ApplyHealth(1, "http://node", true, 1, 0, "", []byte(`{"system":{"cpu_pct":41}}`), time.Now()) + stored := pool.Nodes()[0] + if len(stored.LastStats) == 0 { + t.Fatal("stats were not published to the pool") + } + + pool.ApplyHealth(1, "http://node", false, 0, 0, "", nil, time.Now()) + if got := pool.Nodes()[0].LastStats; got != nil { + t.Fatalf("LastStats = %s after a failed check, want nil", got) + } +} + +// The pool publishes immutable copies. A caller's buffer (a decoded HTTP body) +// must not stay aliased into a node other goroutines are reading. +func TestApplyHealthClonesStats(t *testing.T) { + pool := NewProxyPool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://node", Enabled: true}}) + + buffer := []byte(`{"system":{"cpu_pct":41}}`) + pool.ApplyHealth(1, "http://node", true, 0, 0, "", buffer, time.Now()) + copy(buffer, []byte(`{"system":{"cpu_pct":99}}`)) + + if got := string(pool.Nodes()[0].LastStats); got != `{"system":{"cpu_pct":41}}` { + t.Fatalf("LastStats = %s, want the published copy to be independent of the caller's buffer", got) + } +} + +// The database fence cannot undo a pool write that already happened. A health +// request is bounded at five seconds, which is ample time for an administrator +// to repoint a row and reload the pools; publishing by id alone would then put +// one worker's health — and the scratch fill transcode admission reads — onto +// the replacement, and it would stay there until a later sweep. +func TestApplyHealthIgnoresAResultForAReplacedWorker(t *testing.T) { + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://replacement", Enabled: true}}) + + pool.ApplyHealth(1, "http://original", true, 7, 0, "", []byte(`{"system":{"cpu_pct":41}}`), time.Now()) + + stored := pool.Nodes()[0] + if stored.ActiveJobs != 0 || len(stored.LastStats) != 0 || stored.LastHealthCheck != nil { + t.Fatalf("pool took the old worker's result: %+v", stored) + } +} + +// The fence must not reject the ordinary case: the pools normalize URLs and the +// database column does not, so a trailing slash on one side is the same worker. +func TestApplyHealthAcceptsATrailingSlashDifference(t *testing.T) { + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://node/", Enabled: true}}) + + pool.ApplyHealth(1, "http://node", true, 3, 0, "", nil, time.Now()) + + if stored := pool.Nodes()[0]; stored.ActiveJobs != 3 { + t.Fatalf("active jobs = %d, want the result applied despite the trailing slash", stored.ActiveJobs) + } +} + +// Capability reports carry the GPU identities the planner places work on, and +// their fetch is bounded at two minutes — an even wider window for the row to +// be repointed underneath them. +func TestApplyCapabilitiesIgnoresAReportForAReplacedWorker(t *testing.T) { + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, URL: "http://replacement", Enabled: true}}) + + pool.ApplyCapabilities(1, "http://original", + []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`), + "sha256:stale", time.Now(), nil, nil) + + stored := pool.Nodes()[0] + if len(stored.Capabilities) != 0 || stored.CapabilitiesHash != nil || len(stored.PhysicalGPUKeys) != 0 { + t.Fatalf("pool took the old worker's capability report: %+v", stored) + } +} diff --git a/internal/nodepool/health_test.go b/internal/nodepool/health_test.go new file mode 100644 index 000000000..42aae6b98 --- /dev/null +++ b/internal/nodepool/health_test.go @@ -0,0 +1,580 @@ +package nodepool + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeCapabilityFetcher records what the sweep asked for and answers with a +// canned report, standing in for a real node's /hw-capabilities. +type fakeCapabilityFetcher struct { + mu sync.Mutex + calls []string + payload []byte + hash string + err error +} + +func (f *fakeCapabilityFetcher) fetch(_ context.Context, node *Node) ([]byte, string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, node.URL) + return f.payload, f.hash, f.err +} + +func (f *fakeCapabilityFetcher) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +// newHealthNode starts a stand-in node whose /health advertises capabilitiesHash. +func newHealthNode(t *testing.T, capabilitiesHash string) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(healthResponse{ + Status: "ok", + ActiveJobs: 2, + EgressKbps: 17, + CapabilitiesHash: capabilitiesHash, + }) + })) + t.Cleanup(server.Close) + return server.URL +} + +func stringPtr(s string) *string { return &s } + +// capabilityCheckerFixture wires one transcode node into a checker with a fake +// fetcher and a callback recorder, which is the whole capability flow minus the +// database (repo stays nil, as it is in proxy/transcode modes). +type capabilityCheckerFixture struct { + checker *HealthChecker + pool *TranscodePool + fetcher *fakeCapabilityFetcher + node *Node + + mu sync.Mutex + notified []string +} + +func newCapabilityCheckerFixture(t *testing.T, node *Node, fetcher *fakeCapabilityFetcher) *capabilityCheckerFixture { + t.Helper() + transcodePool := NewTranscodePool() + transcodePool.SetNodes([]*Node{node}) + fixture := &capabilityCheckerFixture{ + checker: NewHealthChecker(NewProxyPool(), transcodePool, nil), + pool: transcodePool, + fetcher: fetcher, + node: node, + } + fixture.checker.SetCapabilityFetcher(fetcher.fetch) + fixture.checker.SetCapabilitiesChangedCallback(func(nodeURL string) { + fixture.mu.Lock() + defer fixture.mu.Unlock() + fixture.notified = append(fixture.notified, nodeURL) + }) + return fixture +} + +// sweep runs one health sweep and then waits for the capability fetches it +// detached, which is what a test needs to observe. Production deliberately does +// not wait: see startCapabilityRefresh. +func (f *capabilityCheckerFixture) sweep() { + f.checker.checkAll(context.Background()) + f.checker.waitForCapabilityRefreshes() +} + +func (f *capabilityCheckerFixture) notifications() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.notified...) +} + +// storedNode returns the pool's current copy, which is what the next sweep +// compares against. +func (f *capabilityCheckerFixture) storedNode(t *testing.T) *Node { + t.Helper() + nodes := f.pool.Nodes() + if len(nodes) != 1 { + t.Fatalf("pool holds %d nodes, want 1", len(nodes)) + } + return nodes[0] +} + +const testCapabilityPayload = `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"],` + + `"detected_backends":[{"backend":"nvenc","verified":true}],"capability_hash":"sha256:new"}` + +// A node whose hardware changed advertises a new hash; the sweep must fetch the +// report once, publish it to the pool, and tell the capability cache to drop +// what it had. +func TestHealthCheckerFetchesCapabilitiesOnHashChange(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 1 { + t.Fatalf("capability fetches = %d, want exactly 1 per sweep", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("pool hash = %v, want sha256:new", stored.CapabilitiesHash) + } + if string(stored.Capabilities) != testCapabilityPayload { + t.Fatalf("pool payload = %s", stored.Capabilities) + } + if stored.CapabilitiesRefreshedAt == nil || stored.CapabilitiesRefreshedAt.IsZero() { + t.Fatal("pool copy carries no capability refresh time") + } + if got := fixture.notifications(); len(got) != 1 || got[0] != url { + t.Fatalf("capability change notifications = %v, want one for %s", got, url) + } +} + +// Nothing changed: the sweep must cost one health request, not a probe of every +// node every 30 seconds. +func TestHealthCheckerSkipsFetchWhenHashUnchanged(t *testing.T) { + url := newHealthNode(t, "sha256:same") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:same"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:same"), + }, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for an unchanged hash", got) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none", got) + } +} + +// A node from before capability snapshots reports no hash. It gets exactly the +// old behavior: no fetch, and nothing invented on its behalf. +func TestHealthCheckerSkipsFetchWhenNodeReportsNoHash(t *testing.T) { + url := newHealthNode(t, "") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "old-node", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for a node that reports no hash", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash != nil || stored.Capabilities != nil { + t.Fatalf("stored capabilities were synthesized: hash=%v payload=%s", stored.CapabilitiesHash, stored.Capabilities) + } + if !stored.Healthy { + t.Fatal("node without a capability hash was not marked healthy") + } +} + +// A node that is down tells us nothing about its hardware, so its stored +// inventory must survive rather than be cleared or refetched. +func TestHealthCheckerSkipsFetchWhenNodeUnhealthy(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "down", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: server.URL, Enabled: true, Healthy: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + + if got := fetcher.callCount(); got != 0 { + t.Fatalf("capability fetches = %d, want none for an unhealthy node", got) + } + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:old" { + t.Fatalf("stored hash = %v, want the previous report kept", stored.CapabilitiesHash) + } +} + +// A failed fetch is not evidence about the hardware: keep what is stored and +// retry on the next sweep. +func TestHealthCheckerKeepsStoredCapabilitiesOnFetchFailure(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{err: errors.New("connection refused")} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + fixture.sweep() + stored := fixture.storedNode(t) + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:old" { + t.Fatalf("stored hash = %v, want the previous report kept", stored.CapabilitiesHash) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none after a failed fetch", got) + } + + // The next sweep retries rather than backing off forever. + fixture.sweep() + if got := fetcher.callCount(); got != 2 { + t.Fatalf("capability fetches = %d, want a retry on the next sweep", got) + } +} + +// A payload without its own hash cannot be tracked for change; storing it would +// refetch every sweep forever. +func TestHealthCheckerRefusesUnhashedCapabilityPayload(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(`{"resolved":"nvenc"}`), hash: ""} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + fixture.sweep() + + stored := fixture.storedNode(t) + if stored.CapabilitiesHash != nil || stored.Capabilities != nil { + t.Fatalf("unhashed payload was stored: hash=%v payload=%s", stored.CapabilitiesHash, stored.Capabilities) + } + if got := fixture.notifications(); len(got) != 0 { + t.Fatalf("notifications = %v, want none", got) + } +} + +// Proxy nodes carry capabilities too (they execute remux recipes), so the sweep +// must publish through the proxy pool's copy-on-write path as well. +func TestHealthCheckerAppliesCapabilitiesToProxyPool(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + proxyPool := NewProxyPool() + proxyPool.SetNodes([]*Node{{ID: 9, Name: "proxy-1", URL: url, Enabled: true}}) + checker := NewHealthChecker(proxyPool, NewTranscodePool(), nil) + checker.SetCapabilityFetcher(fetcher.fetch) + + checker.checkAll(context.Background()) + checker.waitForCapabilityRefreshes() + + stored := proxyPool.Nodes()[0] + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("proxy pool hash = %v, want sha256:new", stored.CapabilitiesHash) + } +} + +// Without a fetcher wired (proxy/transcode modes, or before wiring) the sweep +// must behave exactly as it did before capability tracking. +func TestHealthCheckerWithoutFetcherStillChecksHealth(t *testing.T) { + url := newHealthNode(t, "sha256:new") + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, Name: "gpu-1", URL: url, Enabled: true}}) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + + checker.checkAll(context.Background()) + + stored := pool.Nodes()[0] + if !stored.Healthy || stored.ActiveJobs != 2 || stored.EgressKbps != 17 { + t.Fatalf("health was not applied: %+v", stored) + } + if stored.Capabilities != nil { + t.Fatalf("capabilities = %s, want none without a fetcher", stored.Capabilities) + } +} + +// Losing a verified backend or a render device is the case worth warning about: +// the node still answers health, so nothing else would surface the loss. +func TestHealthCheckerWarnsOnCapabilityDrift(t *testing.T) { + url := newHealthNode(t, "sha256:degraded") + degraded := `{"resolved":"none","render_devices":[],` + + `"detected_backends":[{"backend":"nvenc","verified":false}],"capability_hash":"sha256:degraded"}` + fetcher := &fakeCapabilityFetcher{payload: []byte(degraded), hash: "sha256:degraded"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + logged := captureSlog(t, slog.LevelWarn) + fixture.sweep() + + output := logged.String() + if !bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("no drift warning logged; log was:\n%s", output) + } + for _, want := range []string{"nvenc", "/dev/dri/renderD128"} { + if !bytes.Contains(logged.Bytes(), []byte(want)) { + t.Fatalf("drift warning does not name %q; log was:\n%s", want, output) + } + } +} + +// The first report is not drift — there is nothing to compare it against — so +// it must not warn. +func TestHealthCheckerDoesNotWarnOnFirstCapabilityStore(t *testing.T) { + url := newHealthNode(t, "sha256:new") + fetcher := &fakeCapabilityFetcher{payload: []byte(testCapabilityPayload), hash: "sha256:new"} + fixture := newCapabilityCheckerFixture(t, &Node{ID: 1, Name: "gpu-1", URL: url, Enabled: true}, fetcher) + + logged := captureSlog(t, slog.LevelInfo) + fixture.sweep() + + if bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("first capability store logged drift:\n%s", logged.String()) + } + if !bytes.Contains(logged.Bytes(), []byte("node capabilities stored")) { + t.Fatalf("first capability store was not logged:\n%s", logged.String()) + } +} + +// Gaining hardware is not drift either: only a loss is worth an operator's +// attention. +func TestHealthCheckerDoesNotWarnWhenCapabilitiesImprove(t *testing.T) { + url := newHealthNode(t, "sha256:better") + improved := `{"resolved":"nvenc","render_devices":["/dev/dri/renderD128","/dev/dri/renderD129"],` + + `"detected_backends":[{"backend":"nvenc","verified":true},{"backend":"vaapi","verified":true}],` + + `"capability_hash":"sha256:better"}` + fetcher := &fakeCapabilityFetcher{payload: []byte(improved), hash: "sha256:better"} + fixture := newCapabilityCheckerFixture(t, &Node{ + ID: 1, Name: "gpu-1", URL: url, Enabled: true, + Capabilities: json.RawMessage(testCapabilityPayload), + CapabilitiesHash: stringPtr("sha256:old"), + }, fetcher) + + logged := captureSlog(t, slog.LevelWarn) + fixture.sweep() + + if bytes.Contains(logged.Bytes(), []byte("node capability drift")) { + t.Fatalf("added hardware logged as drift:\n%s", logged.String()) + } +} + +// captureSlog redirects the default logger for one test and returns its output. +func captureSlog(t *testing.T, level slog.Level) *lockedBuffer { + t.Helper() + buffer := &lockedBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(buffer, &slog.HandlerOptions{Level: level}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return buffer +} + +// lockedBuffer collects log output written from the sweep's per-node goroutines. +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(p) +} + +func (b *lockedBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + return append([]byte(nil), b.buffer.Bytes()...) +} + +func (b *lockedBuffer) String() string { return string(b.Bytes()) } + +// The fetch budget must not inherit the 5s health timeout: a cold node runs +// ffmpeg probes to answer a capability request. +func TestCapabilityFetchTimeoutExceedsHealthTimeout(t *testing.T) { + if capabilityFetchTimeout <= 5*time.Second { + t.Fatalf("capabilityFetchTimeout = %s, want more than the health request budget", capabilityFetchTimeout) + } +} + +// blockedSweepTimeout only bounds a failure: on a correct sweep the wait below +// is satisfied by the sweep returning, not by elapsed time. +const blockedSweepTimeout = 30 * time.Second + +// blockingFetcher answers only after it is released, so a test can hold one +// node's capability fetch open and observe what the sweep does meanwhile. +type blockingFetcher struct { + started chan struct{} + release chan struct{} + calls atomic.Int64 +} + +func newBlockingFetcher() *blockingFetcher { + return &blockingFetcher{started: make(chan struct{}, 1), release: make(chan struct{})} +} + +func (f *blockingFetcher) fetch(ctx context.Context, _ *Node) ([]byte, string, error) { + f.calls.Add(1) + select { + case f.started <- struct{}{}: + default: + } + select { + case <-f.release: + return []byte(testCapabilityPayload), "sha256:new", nil + case <-ctx.Done(): + return nil, "", ctx.Err() + } +} + +// The capability fetch budget is larger than the sweep interval, so a fetch +// that ran inside the sweep would stretch every other node's liveness update +// past that interval — and pool health is what routes streams away from a node +// that just died. The sweep must return while the fetch is still outstanding. +func TestHealthCheckerSweepDoesNotWaitForCapabilityFetch(t *testing.T) { + slowURL := newHealthNode(t, "sha256:new") + // The second node advertises no hash, so its own health is all the sweep + // has to do for it. + steadyURL := newHealthNode(t, "") + pool := NewTranscodePool() + pool.SetNodes([]*Node{ + {ID: 1, Name: "slow-gpu", URL: slowURL, Enabled: true}, + {ID: 2, Name: "steady-gpu", URL: steadyURL, Enabled: true}, + }) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + fetcher := newBlockingFetcher() + checker.SetCapabilityFetcher(fetcher.fetch) + + sweepDone := make(chan struct{}) + go func() { + defer close(sweepDone) + checker.checkAll(context.Background()) + }() + + <-fetcher.started + select { + case <-sweepDone: + case <-time.After(blockedSweepTimeout): + t.Fatal("sweep is still running while one node's capability fetch is outstanding") + } + + for _, n := range pool.Nodes() { + if !n.Healthy || n.LastHealthCheck == nil { + t.Fatalf("node %s health was not applied during the blocked fetch: %+v", n.Name, n) + } + } + + close(fetcher.release) + checker.waitForCapabilityRefreshes() + stored := pool.Nodes()[0] + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:new" { + t.Fatalf("detached fetch did not publish its report: hash = %v", stored.CapabilitiesHash) + } +} + +// A fetch that outlives its sweep has not moved the stored hash, so the next +// sweep sees the same mismatch. Without a per-node guard that would start a +// second fetch of the same node on every sweep until the first one answers. +func TestHealthCheckerDoesNotStackCapabilityFetchesForOneNode(t *testing.T) { + url := newHealthNode(t, "sha256:new") + pool := NewTranscodePool() + pool.SetNodes([]*Node{{ID: 1, Name: "slow-gpu", URL: url, Enabled: true}}) + checker := NewHealthChecker(NewProxyPool(), pool, nil) + fetcher := newBlockingFetcher() + checker.SetCapabilityFetcher(fetcher.fetch) + + checker.checkAll(context.Background()) + <-fetcher.started + checker.checkAll(context.Background()) + + if got := fetcher.calls.Load(); got != 1 { + t.Fatalf("capability fetches = %d while the first is still outstanding, want 1", got) + } + + close(fetcher.release) + checker.waitForCapabilityRefreshes() + + // Once the report is stored the hash matches, so no further fetch is due. + checker.checkAll(context.Background()) + checker.waitForCapabilityRefreshes() + if got := fetcher.calls.Load(); got != 1 { + t.Fatalf("capability fetches = %d after the report was stored, want 1", got) + } +} + +// The probe matrix a cold node runs grows with its device count without bound — +// nine render devices legitimately ask for over five minutes — so a fixed outer +// deadline cancels a node operating inside its published contract, and its +// inventory never populates. The backstop is derived from the budget the fetcher +// gave itself so the fetcher's own deadline always fires first. +func TestCapabilityFetchBackstopSitsAboveTheFetcherBudget(t *testing.T) { + checker := NewHealthChecker(NewProxyPool(), NewTranscodePool(), nil) + node := &Node{ID: 1, URL: "http://gpu-1"} + + if got := checker.capabilityFetchBackstop(node); got != capabilityFetchTimeout { + t.Fatalf("backstop with no budget wired = %v, want the %v floor", got, capabilityFetchTimeout) + } + + // A budget under the floor leaves the floor standing: it already clears the + // fetcher by a wide margin. + checker.SetCapabilityFetchBudget(func(*Node) time.Duration { return 2 * time.Minute }) + if got := checker.capabilityFetchBackstop(node); got != capabilityFetchTimeout { + t.Fatalf("backstop for a small budget = %v, want the %v floor", got, capabilityFetchTimeout) + } + + // A budget above it carries the backstop with it, always by the slack, so + // the fetch is never cut short by this. + big := 311 * time.Second + checker.SetCapabilityFetchBudget(func(*Node) time.Duration { return big }) + got := checker.capabilityFetchBackstop(node) + if got != big+capabilityFetchSlack { + t.Fatalf("backstop for a %v budget = %v, want %v", big, got, big+capabilityFetchSlack) + } + if got <= big { + t.Fatalf("backstop %v does not clear the %v the fetcher allowed itself", got, big) + } +} + +// A stored node URL may carry a trailing slash — pasting a base URL is the usual +// way an operator enters one, and everything here already treats the two forms +// as the same worker. Concatenating a route onto it produces "//admin/…", which +// no node's router has: the request 404s against a node that is running and +// reachable, and the operator's action fails for a reason nothing reports. +func TestNodeEndpointJoinsATrailingSlashBaseURL(t *testing.T) { + const want = "http://gpu-1:8082/admin/reprobe-capabilities" + for _, base := range []string{ + "http://gpu-1:8082", + "http://gpu-1:8082/", + "http://gpu-1:8082///", + } { + if got := NodeEndpoint(base, "/admin/reprobe-capabilities"); got != want { + t.Errorf("NodeEndpoint(%q) = %q, want %q", base, got, want) + } + } +} + +// The health check is the first thing that runs against a node, so a base URL +// this could not address would leave the node permanently unhealthy. +func TestCheckNodeReachesATrailingSlashBaseURL(t *testing.T) { + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/health" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"active_jobs": 1, "capabilities_hash": "sha256:x"}) + })) + defer node.Close() + + healthy, activeJobs, _, hash, _ := CheckNode(context.Background(), &Node{ID: 1, URL: node.URL + "/"}) + if !healthy { + t.Fatal("a node stored with a trailing slash was reported unhealthy") + } + if activeJobs != 1 || hash != "sha256:x" { + t.Fatalf("active jobs = %d, hash = %q; want the node's own answer", activeJobs, hash) + } +} diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 62d332135..894a91f97 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -2,6 +2,7 @@ package nodepool import ( "context" + "log/slog" "sync" "time" @@ -85,16 +86,31 @@ type Planner struct { rr map[string]int // per-group round-robin cursor; "" = global reserved map[string]*reservation // keyed by playback session ID now func() time.Time // overridable for tests + // scratchPressed latches, by node id, which nodes were under scratch + // pressure the last time a transcode selection looked. It exists only to + // make the operator warning fire once per transition into pressure: the + // eligibility path runs on every session start and every quality switch, so + // logging there unlatched would produce a line per playback event for as + // long as a disk stays full. Guarded by mu, which every selection path + // already holds. Entries are pruned as nodes leave the pool. + scratchPressed map[int]bool + // scratchGuardDropped latches whether the last transcode selection had to + // ignore the scratch guard because every eligible candidate was pressured. + // Latched for the same reason as scratchPressed, and separately from it + // because a cluster reaches "no headroom anywhere" on its own schedule. + // Guarded by mu. + scratchGuardDropped bool } // NewPlanner creates a planner over the given pools. func NewPlanner(proxies *ProxyPool, transcodes *TranscodePool) *Planner { return &Planner{ - proxies: proxies, - transcodes: transcodes, - rr: make(map[string]int), - reserved: make(map[string]*reservation), - now: time.Now, + proxies: proxies, + transcodes: transcodes, + rr: make(map[string]int), + reserved: make(map[string]*reservation), + now: time.Now, + scratchPressed: make(map[int]bool), } } @@ -125,6 +141,37 @@ func (p *Planner) TranscodeNode(nodeID int) (*Node, bool) { return nil, false } +// TranscodeNodeByURL returns the pooled record for a transcode node URL. It +// gives a dispatch path that node's own acceleration override, from the URL it +// is about to send a job to. Enabled or not: a caller holding the URL has +// already selected it. +func (p *Planner) TranscodeNodeByURL(nodeURL string) (*Node, bool) { + if p == nil || p.transcodes == nil || nodeURL == "" { + return nil, false + } + node := p.transcodes.FindByURL(normalizeNodeURL(nodeURL)) + if node == nil { + return nil, false + } + return node, true +} + +// ProxyNodeByURL returns the pooled record for a proxy node URL, under the +// same contract as TranscodeNodeByURL. Capability-budget pricing needs it: +// a proxy's stored report and overrides say how long its own cold probe may +// take, and a caller that can only resolve transcode nodes prices every proxy +// from the cluster policy instead. +func (p *Planner) ProxyNodeByURL(nodeURL string) (*Node, bool) { + if p == nil || p.proxies == nil || nodeURL == "" { + return nil, false + } + node := p.proxies.FindByURL(normalizeNodeURL(nodeURL)) + if node == nil { + return nil, false + } + return node, true +} + // TranscodeNodeHealthy reports whether the pooled transcode node serving a URL // is currently healthy and enabled. Remote-start adoption gates its redirect // on this: a recipe another API server published is only trustworthy while @@ -206,17 +253,19 @@ func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTr estBitrateKbps = 0 } proxies := p.proxies.Nodes() - transcodes := p.transcodes.Nodes() + pooledTranscodes := p.transcodes.Nodes() + transcodes := pooledTranscodes // Group health is computed over the full pool before any narrowing: // eligibility restricts what may be selected, not co-location semantics. - groupHealthy := groupHealth(proxies, transcodes) + // Shared-GPU load is summed over the same full pool, for the same reason. + groupHealthy := groupHealth(proxies, pooledTranscodes) var plan Plan if needsTranscode { if eligible != nil { transcodes = filterNodes(transcodes, eligible) } - plan.TranscodeNode = p.pickTranscode(transcodes, proxies, groupHealthy, currentTranscodeURL, estBitrateKbps, now) + plan.TranscodeNode = p.pickTranscode(transcodes, pooledTranscodes, proxies, groupHealthy, currentTranscodeURL, estBitrateKbps, now) if plan.TranscodeNode != nil { plan.ProxyNode = p.pickProxy(proxies, groupHealthy, plan.TranscodeNode.Group, estBitrateKbps, now) } @@ -262,12 +311,13 @@ func (p *Planner) PlanTranscodeSessionWithLocalEgress(sessionID, currentTranscod p.pruneReservations(now) delete(p.reserved, sessionID) - transcodes := p.transcodes.Nodes() - groupHealthy := groupHealth(nil, transcodes) + pooledTranscodes := p.transcodes.Nodes() + transcodes := pooledTranscodes + groupHealthy := groupHealth(nil, pooledTranscodes) if eligible != nil { transcodes = filterNodes(transcodes, eligible) } - node := p.pickLocalEgressTranscode(transcodes, groupHealthy, currentTranscodeURL, now) + node := p.pickLocalEgressTranscode(transcodes, pooledTranscodes, groupHealthy, currentTranscodeURL, now) if node == nil { return Plan{} } @@ -449,8 +499,13 @@ func groupHealth(proxies, transcodes []*Node) map[string]bool { // the session on currentURL unless a candidate has at least two fewer jobs // (the historical soft-affinity rule). Shared by pickTranscode and // pickLocalEgressTranscode, which differ only in their eligibility predicate. -func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, eligible func(*Node) bool) *Node { +// +// tieBreak, when non-nil, scores candidates that are level on effective jobs; +// the lower score wins. It never overrides the job count or the affinity rule, +// so a caller that passes nil gets exactly the historical selection. +func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, eligible func(*Node) bool, tieBreak func(*Node) int) *Node { var best, current *Node + bestJobs := 0 for _, n := range nodes { if !eligible(n) { continue @@ -458,7 +513,11 @@ func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, elig if n.URL == currentURL { current = n } - if best == nil || p.effectiveJobs(n, now) < p.effectiveJobs(best, now) { + jobs := p.effectiveJobs(n, now) + switch { + case best == nil, jobs < bestJobs: + best, bestJobs = n, jobs + case jobs == bestJobs && tieBreak != nil && tieBreak(n) < tieBreak(best): best = n } } @@ -473,11 +532,202 @@ func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, elig // pickTranscode returns the eligible transcode node with the fewest effective // jobs, keeping the session on currentURL unless a candidate has at least two -// fewer jobs (the historical soft-affinity rule). -func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { - return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { +// fewer jobs (the historical soft-affinity rule). Candidates level on job count +// are separated by the load on the physical GPU behind them: two pooled nodes +// can be two containers on one card, and spreading jobs across node records +// that share silicon does not spread the work. +// +// pool is the full transcode pool the candidates were drawn from; shared-GPU +// load is summed over all of it, since a job on a node this plan may not select +// still occupies the same GPU. +func (p *Planner) pickTranscode(transcodes, pool, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { + return p.pickWithScratchGuard(transcodes, pool, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, proxies, groupHealthy, estKbps, now) - }) + }, p.physicalGPULoadScore(pool, now)) +} + +// pickWithScratchGuard is pickNode with the scratch-pressure exclusion applied +// as a *soft* filter: a node whose transcode scratch volume is at or past +// scratchPressureFillPercent is skipped, unless skipping leaves no candidate at +// all, in which case the guard is ignored and the ordinary pick stands. +// +// Soft is the whole point. The guard's job is to steer sessions away from a node +// that will die mid-stream while a healthy sibling exists; it is not a license +// to refuse playback. A cluster whose scratch volumes have all filled — one shared +// NFS export, a retention setting that is too generous everywhere — would +// otherwise go from degraded to dark on a threshold nobody chose for that +// purpose. Degraded service beats no service, and the WARN below is what tells +// an operator which it is. +// +// pool is the full transcode pool the candidates were drawn from; it scopes the +// warning latch, which tracks disks rather than the narrowed candidate set. +// +// Callers must hold mu. +func (p *Planner) pickWithScratchGuard(nodes, pool []*Node, currentURL string, now time.Time, eligible func(*Node) bool, tieBreak func(*Node) int) *Node { + excluded := false + guarded := func(n *Node) bool { + if !eligible(n) { + return false + } + if scratchPressured(n) { + excluded = true + return false + } + return true + } + picked := p.pickNode(nodes, currentURL, now, guarded, tieBreak) + guardDropped := false + if picked == nil && excluded { + // Without an exclusion the unguarded pick would return the same nil: + // nothing was eligible in the first place. + picked = p.pickNode(nodes, currentURL, now, eligible, tieBreak) + guardDropped = picked != nil + } + // Logged after the fallback decides, because what an operator needs to know + // is whether the pressured node was actually kept out of selection. + p.noteScratchPressure(pool, guardDropped) + return picked +} + +// noteScratchPressure logs each pooled node's transition into and out of +// scratch pressure exactly once, and separately latches the cluster-wide state +// where the guard had to give way. +// +// guardDropped reports that this pick found every eligible candidate pressured +// and admitted one anyway. It has to reach the log, because the two states the +// guard can be in are opposites for an operator: "sessions are being steered +// away from this disk" is a warning, while "sessions are landing on a disk that +// is about to fail mid-stream because nothing else is eligible" is an outage in +// progress. One invariant message asserting an exclusion cannot say which, and +// the pressure latch alone would keep asserting the exclusion long after the +// fallback started ignoring it. +// +// There is no context to log against: selection is synchronous inside +// PlanSession and friends, which take no ctx because they are called from +// several request paths and from reservation code that has none. The latch, +// not a context, is what keeps this out of the hot path's log volume — the +// eligibility check runs on every session start and every quality switch. +// +// Callers must hold mu. +func (p *Planner) noteScratchPressure(pool []*Node, guardDropped bool) { + seen := make(map[int]struct{}, len(pool)) + for _, n := range pool { + if n == nil { + continue + } + seen[n.ID] = struct{}{} + pressured := scratchPressured(n) + if pressured == p.scratchPressed[n.ID] { + continue + } + if pressured { + pct, _ := scratchDiskFillPercent(n) + attrs := []any{ + "component", "nodepool", "id", n.ID, "name", n.Name, "url", n.URL, + "scratch_fill_pct", pct, "threshold_pct", scratchPressureFillPercent, + "scratch_guard_dropped", guardDropped, + } + if guardDropped { + slog.Warn("transcode node scratch volume nearly full, still selected because no eligible node has scratch headroom", attrs...) + } else { + slog.Warn("transcode node scratch volume nearly full, excluded from selection", attrs...) + } + p.scratchPressed[n.ID] = true + continue + } + slog.Info("transcode node scratch volume recovered", "component", "nodepool", + "id", n.ID, "name", n.Name, "url", n.URL) + delete(p.scratchPressed, n.ID) + } + // A node that left the pool must not keep a latch entry: it would suppress + // the warning if the same id came back still full. + for id := range p.scratchPressed { + if _, ok := seen[id]; !ok { + delete(p.scratchPressed, id) + } + } + p.noteScratchGuardDropped(guardDropped) +} + +// noteScratchGuardDropped reports the transition into and out of the state where +// the scratch guard has nothing left to steer towards. +// +// It is latched separately from the per-node pressure warnings because the two +// move independently: a cluster can cross into "no headroom anywhere" long after +// its nodes' pressure transitions were logged, and without this the operator's +// only signal would be an exclusion message written before the exclusion stopped +// happening. +// +// Callers must hold mu. +func (p *Planner) noteScratchGuardDropped(dropped bool) { + if dropped == p.scratchGuardDropped { + return + } + p.scratchGuardDropped = dropped + if dropped { + slog.Warn("transcode scratch guard ignored: every eligible node is over the scratch threshold", + "component", "nodepool", "threshold_pct", scratchPressureFillPercent) + return + } + slog.Info("transcode scratch guard back in force: an eligible node has scratch headroom again", + "component", "nodepool") +} + +// physicalGPULoadScore precomputes, once per pick, the total effective jobs +// running on each pooled node's physical GPU group: itself plus every pooled +// node sharing at least one GPU identity with it, healthy or not, because a job +// occupies a card regardless of whether the node running it may take another. A +// node with no derived identities is a group of one. +// +// Pools only hold enabled nodes, so a node disabled while its transcodes are +// still running leaves the pool and stops counting against its group — the same +// blind spot the primary least-jobs rule already has, since there is no drain +// state between enabled and gone. +// +// Job counts only. Live GPU utilization is a richer signal but a much worse +// tie-breaker: it lags a newly admitted job by a sampling interval, so a burst +// of starts would all see the same idle card. +func (p *Planner) physicalGPULoadScore(pool []*Node, now time.Time) func(*Node) int { + jobs := make(map[*Node]int, len(pool)) + byKey := make(map[string][]*Node) + for _, n := range pool { + if n == nil { + continue + } + jobs[n] = p.effectiveJobs(n, now) + for _, key := range n.PhysicalGPUKeys { + byKey[key] = append(byKey[key], n) + } + } + + loads := make(map[*Node]int, len(pool)) + for _, n := range pool { + if n == nil { + continue + } + total := jobs[n] + counted := map[*Node]struct{}{n: {}} + for _, key := range n.PhysicalGPUKeys { + for _, peer := range byKey[key] { + // A peer sharing several keys with n must only be counted once. + if _, seen := counted[peer]; seen { + continue + } + counted[peer] = struct{}{} + total += jobs[peer] + } + } + loads[n] = total + } + + return func(n *Node) int { + if load, ok := loads[n]; ok { + return load + } + // A candidate the pool snapshot does not contain (a concurrent pool + // swap) scores as its own load, which is the no-sharing answer. + return p.effectiveJobs(n, now) + } } // pickLocalEgressTranscode applies the transcode half of normal session @@ -486,10 +736,10 @@ func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[st // suppress an otherwise healthy transcode executor. Passing nil proxies to // transcodeEligible reduces it to exactly that: healthy, enabled, under cap, // and group-healthy, with no proxy partner required. -func (p *Planner) pickLocalEgressTranscode(transcodes []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { - return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { +func (p *Planner) pickLocalEgressTranscode(transcodes, pool []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { + return p.pickWithScratchGuard(transcodes, pool, currentURL, now, func(n *Node) bool { return p.transcodeEligible(n, nil, groupHealthy, 0, now) - }) + }, p.physicalGPULoadScore(pool, now)) } // transcodeEligible reports whether a transcode node may take a new session: @@ -497,6 +747,12 @@ func (p *Planner) pickLocalEgressTranscode(transcodes []*Node, groupHealthy map[ // its whole group healthy and — when the group contains proxies — at least // one of them with job and bandwidth headroom (a group's capacity is bounded // by its proxies). +// +// Scratch-volume pressure is deliberately *not* checked here. It is a soft +// exclusion applied by pickWithScratchGuard, which can drop it when it would +// empty the candidate set; a hard predicate could not. Callers that use this +// directly (ReserveTranscodeWorkWith's non-streaming reservations) intentionally +// keep the historical behavior. func (p *Planner) transcodeEligible(n *Node, proxies []*Node, groupHealthy map[string]bool, estKbps int, now time.Time) bool { if !n.Healthy || !n.Enabled || !p.underCap(n, now) { return false diff --git a/internal/nodepool/planner_scratch_test.go b/internal/nodepool/planner_scratch_test.go new file mode 100644 index 000000000..659b56469 --- /dev/null +++ b/internal/nodepool/planner_scratch_test.go @@ -0,0 +1,237 @@ +package nodepool + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +// captureScratchLogs redirects the default logger, which is what selection logs +// against: PlanSession takes no context, so there is no handler to inject. +func captureScratchLogs(t *testing.T) *bytes.Buffer { + t.Helper() + buffer := &bytes.Buffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(buffer, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return buffer +} + +// A transcode writes segments to its scratch volume for the life of the session, +// so admitting one onto a nearly full node buys a stream that dies mid-playback. +// With a healthy sibling available the pressured node must lose, even though it +// carries fewer jobs and would otherwise win outright. +func TestPlanSessionSkipsScratchPressuredTranscodeNode(t *testing.T) { + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(96, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 3) + roomy.LastStats = scratchStats(20, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{full, roomy}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("no transcode node selected") + } + if plan.TranscodeNode.URL != "http://tc-roomy" { + t.Fatalf("selected %s, want the node with scratch headroom", plan.TranscodeNode.URL) + } +} + +// The guard is soft on purpose. If every otherwise-eligible node is over the +// threshold, refusing to plan would turn a degraded cluster into a dark one, and +// the threshold was never chosen to be a kill switch. +func TestPlanSessionIgnoresScratchGuardWhenEveryNodeIsPressured(t *testing.T) { + first := transcodeNode(1, "http://tc-a", nil, 5) + first.LastStats = scratchStats(99, 100) + second := transcodeNode(2, "http://tc-b", nil, 1) + second.LastStats = scratchStats(97, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{first, second}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("scratch pressure emptied the pool; degraded service must beat no service") + } + // With the guard dropped the ordinary least-jobs rule decides. + if plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("selected %s, want the least-loaded node once the guard is ignored", plan.TranscodeNode.URL) + } + if plan.ProxyNode == nil { + t.Fatal("no proxy paired with the fallback transcode node") + } +} + +// The WARN is the operator's only signal about scratch pressure, so it must not +// claim an exclusion that did not happen. When every eligible node is over the +// threshold the guard gives way and the pressured node takes the session — the +// opposite of "excluded from selection", and a far more urgent state. +func TestScratchPressureWarningReportsTheDegradedAdmission(t *testing.T) { + logs := captureScratchLogs(t) + first := transcodeNode(1, "http://tc-a", nil, 5) + first.LastStats = scratchStats(99, 100) + second := transcodeNode(2, "http://tc-b", nil, 1) + second.LastStats = scratchStats(97, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{first, second}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil { + t.Fatal("scratch pressure emptied the pool; degraded service must beat no service") + } + + output := logs.String() + if strings.Contains(output, "excluded from selection") { + t.Fatalf("logged an exclusion for a node it then selected (%s):\n%s", plan.TranscodeNode.URL, output) + } + if !strings.Contains(output, "still selected because no eligible node has scratch headroom") { + t.Fatalf("no degraded-admission warning:\n%s", output) + } + if !strings.Contains(output, "transcode scratch guard ignored") { + t.Fatalf("the cluster-wide guard-dropped state was never reported:\n%s", output) + } +} + +// With a sibling that has headroom the guard really does exclude, and the +// message has to say so — the two states are told apart by wording, so both have +// to be exercised. +func TestScratchPressureWarningReportsARealExclusion(t *testing.T) { + logs := captureScratchLogs(t) + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(99, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 3) + roomy.LastStats = scratchStats(20, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{full, roomy}) + + f.planner.PlanSession("session-1", "", true, 0) + + output := logs.String() + if !strings.Contains(output, "excluded from selection") { + t.Fatalf("no exclusion warning while a node with headroom existed:\n%s", output) + } + if strings.Contains(output, "transcode scratch guard ignored") { + t.Fatalf("reported the guard as dropped while it was honored:\n%s", output) + } +} + +// The guard-dropped state is latched like the per-node pressure warnings, and +// reports its way back out: an operator who saw the outage warning needs to know +// when steering resumed. +func TestScratchGuardDroppedStateLatchesAndRecovers(t *testing.T) { + logs := captureScratchLogs(t) + only := transcodeNode(1, "http://tc-a", nil, 0) + only.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{only}) + + for range 3 { + f.planner.PlanSession("session-1", "", true, 0) + } + if got := strings.Count(logs.String(), "transcode scratch guard ignored"); got != 1 { + t.Fatalf("guard-dropped warnings = %d across three plans, want exactly one", got) + } + + recovered := transcodeNode(1, "http://tc-a", nil, 0) + recovered.LastStats = scratchStats(10, 100) + f.transcodes.SetNodes([]*Node{recovered}) + f.planner.PlanSession("session-2", "", true, 0) + + if !strings.Contains(logs.String(), "transcode scratch guard back in force") { + t.Fatalf("the guard coming back into force was never reported:\n%s", logs.String()) + } +} + +// A node whose sample cannot be read must not be excluded: the guard fires on +// measured pressure only, and a node predating the scratch flag reports none. +func TestPlanSessionAdmitsNodeWithoutScratchStats(t *testing.T) { + unknown := transcodeNode(1, "http://tc-unknown", nil, 0) + pressured := transcodeNode(2, "http://tc-full", nil, 0) + pressured.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{unknown, pressured}) + + plan := f.planner.PlanSession("session-1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-unknown" { + t.Fatalf("selected %v, want the node with no scratch reading", plan.TranscodeNode) + } +} + +// Soft affinity keeps a session on its current node unless a candidate is two +// jobs better. Scratch pressure has to beat that: staying is what kills the +// stream, and the affinity rule exists to avoid gratuitous switches, not to +// defend a full disk. +func TestPlanSessionMovesOffAScratchPressuredCurrentNode(t *testing.T) { + current := transcodeNode(1, "http://tc-current", nil, 0) + current.LastStats = scratchStats(99, 100) + other := transcodeNode(2, "http://tc-other", nil, 0) + other.LastStats = scratchStats(10, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{current, other}) + + plan := f.planner.PlanSession("session-1", "http://tc-current", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-other" { + t.Fatalf("selected %v, want the session moved off the full-scratch node", plan.TranscodeNode) + } +} + +// The local-egress path admits transcodes without a proxy partner and must apply +// the same guard, since it is the same scratch volume being written. +func TestPlanTranscodeSessionWithLocalEgressAppliesScratchGuard(t *testing.T) { + full := transcodeNode(1, "http://tc-full", nil, 0) + full.LastStats = scratchStats(96, 100) + roomy := transcodeNode(2, "http://tc-roomy", nil, 2) + roomy.LastStats = scratchStats(20, 100) + f := newFixture(nil, []*Node{full, roomy}) + + plan := f.planner.PlanTranscodeSessionWithLocalEgress("session-1", "", nil) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-roomy" { + t.Fatalf("selected %v, want the node with scratch headroom", plan.TranscodeNode) + } +} + +// The warning is latched so a full disk does not produce a log line per session +// start. The latch is internal state, so this asserts the transitions it records +// rather than the log output. +func TestScratchPressureWarningLatchesPerTransition(t *testing.T) { + node := transcodeNode(1, "http://tc-a", nil, 0) + node.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{node}) + + for range 3 { + f.planner.PlanSession("session-1", "", true, 0) + } + f.planner.mu.Lock() + latched := len(f.planner.scratchPressed) + pressed := f.planner.scratchPressed[1] + f.planner.mu.Unlock() + if latched != 1 || !pressed { + t.Fatalf("latch = %v, want node 1 latched once", f.planner.scratchPressed) + } + + // Recovery clears the latch, so the next episode warns again. + recovered := transcodeNode(1, "http://tc-a", nil, 0) + recovered.LastStats = scratchStats(10, 100) + f.transcodes.SetNodes([]*Node{recovered}) + f.planner.PlanSession("session-2", "", true, 0) + + f.planner.mu.Lock() + latched = len(f.planner.scratchPressed) + f.planner.mu.Unlock() + if latched != 0 { + t.Fatalf("latch = %v after recovery, want it cleared", f.planner.scratchPressed) + } +} + +// A node that leaves the pool must not leave a latch entry behind: the same id +// coming back still full would then never warn. +func TestScratchPressureLatchIsPrunedWhenANodeLeavesThePool(t *testing.T) { + node := transcodeNode(1, "http://tc-a", nil, 0) + node.LastStats = scratchStats(99, 100) + f := newFixture([]*Node{proxyNode(10, "http://proxy", nil)}, []*Node{node}) + f.planner.PlanSession("session-1", "", true, 0) + + f.transcodes.SetNodes([]*Node{transcodeNode(2, "http://tc-b", nil, 0)}) + f.planner.PlanSession("session-2", "", true, 0) + + f.planner.mu.Lock() + _, stillLatched := f.planner.scratchPressed[1] + f.planner.mu.Unlock() + if stillLatched { + t.Fatal("latch entry survived the node leaving the pool") + } +} diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index fe32baf12..f45173d9b 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -1,7 +1,10 @@ package nodepool import ( + "encoding/json" + "fmt" "strconv" + "strings" "testing" "time" ) @@ -556,7 +559,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // Unlike job reservations, bandwidth bridges ignore health freshness — // a report right after admission would not reflect the streams yet. newer := f.now.Add(5 * time.Second) - f.proxies.ApplyHealth(1, true, 0, 0, newer) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 0, "", nil, newer) f.now = f.now.Add(10 * time.Second) if got := f.planner.PlanSession("s4", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("stream should still be rejected during bridge window, got %+v", got) @@ -566,7 +569,7 @@ func TestBandwidthReservationsCountDuringBridge(t *testing.T) { // meter now reports 8 Mbps, so one more 4 Mbps stream still won't fit, // but a 2 Mbps one will. f.now = f.now.Add(bandwidthBridgeAge) - f.proxies.ApplyHealth(1, true, 0, 8_000, f.now) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 8_000, "", nil, f.now) if got := f.planner.PlanSession("s5", "", false, 4_000).ProxyNode; got != nil { t.Fatalf("4 Mbps stream should not fit at 8/10 Mbps, got %+v", got) } @@ -618,7 +621,7 @@ func TestUnknownBitrateAdmittedBelowCap(t *testing.T) { t.Fatal("unknown-bitrate stream should be admitted below cap") } - f.proxies.ApplyHealth(1, true, 0, 10_000, f.now) + f.proxies.ApplyHealth(1, f.proxies.Nodes()[0].URL, true, 0, 10_000, "", nil, f.now) if got := f.planner.PlanSession("s2", "", false, 0).ProxyNode; got != nil { t.Fatalf("unknown-bitrate stream should be rejected at cap, got %+v", got) } @@ -784,3 +787,239 @@ func TestProxyNodeURLsListsEnabledProxies(t *testing.T) { t.Fatalf("proxy urls = %v, want both pooled proxies", urls) } } + +// gpuCapabilities is a capability report naming one render device per uuid, so +// pool loading derives exactly those identities. Written as a payload rather +// than by setting the derived field directly: the pool re-derives it on load, +// which is the behavior these tests depend on. +func gpuCapabilities(uuids ...string) json.RawMessage { + devices := make([]string, 0, len(uuids)) + for i, uuid := range uuids { + devices = append(devices, fmt.Sprintf(`{"path":"/dev/dri/renderD%d","gpu_uuid":%q}`, 128+i, uuid)) + } + return json.RawMessage(`{"boot_id":"boot-1","render_device_details":[` + strings.Join(devices, ",") + `]}`) +} + +// gpuTranscodeNode is transcodeNode carrying a capability report for the named +// GPUs. +func gpuTranscodeNode(id int, url string, activeJobs int, uuids ...string) *Node { + n := transcodeNode(id, url, nil, activeJobs) + n.Capabilities = gpuCapabilities(uuids...) + return n +} + +// Two pooled nodes can be two containers on one card. Spreading jobs evenly +// across node records that share silicon does not spread the work, so equal job +// counts are broken toward the node whose physical GPU is doing less. +func TestEqualJobsPrefersTheIdlePhysicalGPU(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 2, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + // tc-b and tc-c are level on jobs and tc-b comes first in pool order, so + // only the shared-GPU tie-break can select tc-c. + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("expected tc-c on the idle GPU, got %+v", plan.TranscodeNode) + } + + // The same rule applies to the API-relayed route, which shares pickNode. + // s1's reservation is released first so the second pick starts from the + // same job counts rather than from tc-c already charged for s1. + f.planner.ReleaseSession("s1") + local := f.planner.PlanTranscodeSessionWithLocalEgress("s2", "", nil) + if local.TranscodeNode == nil || local.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("local-egress expected tc-c, got %+v", local.TranscodeNode) + } +} + +// Jobs occupy a card whether or not the node running them may take another, so +// an unhealthy sharer — which stays pooled, only ineligible — still counts +// against its group. +func TestSharedGPULoadCountsUnusableSharers(t *testing.T) { + busy := gpuTranscodeNode(2, "http://tc-a", 3, "GPU-shared") + busy.Healthy = false + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + busy, + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-c" { + t.Fatalf("expected tc-c, got %+v", plan.TranscodeNode) + } +} + +// Boot id detection is best-effort, so two unrelated hosts can each report an +// iGPU at the near-universal slot 0000:00:02.0 with no boot id to scope it. +// Those are two cards, and planning them as one would steer work away from a +// genuinely idle GPU. +func TestUnscopedSlotsDoNotShareAGroup(t *testing.T) { + iGPU := json.RawMessage(`{"render_device_details":[` + + `{"path":"/dev/dri/renderD128","pci_address":"0000:00:02.0"}]}`) + busy := transcodeNode(2, "http://tc-a", nil, 2) + busy.Capabilities = iGPU + idle := transcodeNode(3, "http://tc-b", nil, 0) + idle.Capabilities = iGPU + f := newFixture([]*Node{proxyNode(1, "http://proxy-1", nil)}, []*Node{busy, idle}) + + if got := f.transcodes.Nodes()[0].PhysicalGPUKeys; got != nil { + t.Fatalf("an unscoped slot derived %v, want no key", got) + } + loads := f.planner.physicalGPULoadScore(f.transcodes.Nodes(), f.now) + if got := loads(f.transcodes.Nodes()[1]); got != 0 { + t.Fatalf("idle host's shared-GPU load = %d, want 0 (its own jobs only)", got) + } +} + +// A node sharing several keys with the same peer must not have that peer's jobs +// counted once per key; the group is a set of nodes, not of keys. +func TestSharedGPULoadCountsEachSharerOnce(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 1, "GPU-x", "GPU-y"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-x", "GPU-y"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-p", "GPU-q"), + }, + ) + // tc-b's group load is 1 (its own 0 plus tc-a's single job counted once), + // tc-c's is 0, so tc-c wins — but only by one job, which double counting + // would inflate without changing the winner. The assertion that matters is + // the score itself. + loads := f.planner.physicalGPULoadScore(f.transcodes.Nodes(), f.now) + if got := loads(f.transcodes.Nodes()[1]); got != 1 { + t.Fatalf("tc-b shared-GPU load = %d, want 1", got) + } + if got := loads(f.transcodes.Nodes()[0]); got != 1 { + t.Fatalf("tc-a shared-GPU load = %d, want 1", got) + } + if got := loads(f.transcodes.Nodes()[2]); got != 0 { + t.Fatalf("tc-c shared-GPU load = %d, want 0", got) + } +} + +// The tie-break only ranks candidates that are already level; a node with fewer +// jobs still wins outright, even when its GPU is the busier one. +func TestFewerJobsBeatsAnIdlePhysicalGPU(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 5, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 1, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("expected the least-loaded tc-b, got %+v", plan.TranscodeNode) + } +} + +// Soft affinity outranks the tie-break: moving a running session to another +// node costs a restart, which a shared GPU is not on its own reason enough for. +func TestSharedGPUTieBreakDoesNotBreakSoftAffinity(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + gpuTranscodeNode(2, "http://tc-a", 2, "GPU-shared"), + gpuTranscodeNode(3, "http://tc-b", 0, "GPU-shared"), + gpuTranscodeNode(4, "http://tc-c", 0, "GPU-own"), + }, + ) + + plan := f.planner.PlanSession("s1", "http://tc-b", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("expected affinity to keep tc-b, got %+v", plan.TranscodeNode) + } +} + +// A node with no identifiable GPU is a group of itself, so a pool that reports +// none selects exactly as it did before this rule existed: least jobs, then +// pool order. +func TestNodesWithoutGPUKeysKeepPoolOrderOnTies(t *testing.T) { + f := newFixture( + []*Node{proxyNode(1, "http://proxy-1", nil)}, + []*Node{ + transcodeNode(2, "http://tc-a", nil, 0), + transcodeNode(3, "http://tc-b", nil, 0), + }, + ) + + plan := f.planner.PlanSession("s1", "", true, 0) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-a" { + t.Fatalf("expected the first pooled node, got %+v", plan.TranscodeNode) + } +} + +// Proxies are picked round-robin, and egress is not GPU work: a shared card +// must not perturb that rotation. +func TestSharedGPUDoesNotAffectProxySelection(t *testing.T) { + busyPeer := proxyNode(1, "http://proxy-a", nil) + busyPeer.Capabilities = gpuCapabilities("GPU-shared") + busyPeer.ActiveJobs = 9 + quiet := proxyNode(2, "http://proxy-b", nil) + quiet.Capabilities = gpuCapabilities("GPU-shared") + f := newFixture([]*Node{busyPeer, quiet}, nil) + + first := f.planner.PlanSession("s1", "", false, 0).ProxyNode + second := f.planner.PlanSession("s2", "", false, 0).ProxyNode + if first == nil || second == nil || first.URL == second.URL { + t.Fatalf("expected round-robin across both proxies, got %+v then %+v", first, second) + } +} + +// A proxy URL stored with a trailing slash must still resolve: URLs are +// normalized where they enter the pool, so the normalized lookup key +// ProxyNodeByURL builds compares equal. Without the SetNodes normalization the +// lookup always missed and capability pricing fell back to the cluster policy +// for exactly the proxies an operator had configured by hand. +func TestProxyNodeByURLNormalizesStoredAndLookupURLs(t *testing.T) { + proxies := NewProxyPool() + proxies.SetNodes([]*Node{{ID: 1, Name: "p1", URL: "https://proxy.example.com/", Enabled: true, Healthy: true}}) + planner := NewPlanner(proxies, NewTranscodePool()) + + for _, lookup := range []string{"https://proxy.example.com", "https://proxy.example.com/"} { + node, ok := planner.ProxyNodeByURL(lookup) + if !ok || node == nil { + t.Fatalf("ProxyNodeByURL(%q) = %v, %v; want the pooled node", lookup, node, ok) + } + if node.ID != 1 { + t.Fatalf("ProxyNodeByURL(%q) resolved node %d, want 1", lookup, node.ID) + } + } +} + +// ClientURL is what every client-facing URL builder joins paths onto: the +// public URL when set, the backend URL otherwise. The fallback is what keeps +// every deployment registered before the split — and every flat network — +// byte-identical. +func TestClientURLPrefersThePublicURL(t *testing.T) { + public := "https://cdn.example.com/" + blank := " " + cases := []struct { + name string + node *Node + want string + }{ + {"nil node", nil, ""}, + {"no public url", &Node{URL: "http://10.0.0.5:8083/"}, "http://10.0.0.5:8083"}, + {"public url set", &Node{URL: "http://10.0.0.5:8083", PublicURL: &public}, "https://cdn.example.com"}, + {"blank public url falls back", &Node{URL: "http://10.0.0.5:8083", PublicURL: &blank}, "http://10.0.0.5:8083"}, + } + for _, tc := range cases { + if got := tc.node.ClientURL(); got != tc.want { + t.Fatalf("%s: ClientURL() = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/internal/nodepool/proxy_pool.go b/internal/nodepool/proxy_pool.go index 95c0480ad..355e1d168 100644 --- a/internal/nodepool/proxy_pool.go +++ b/internal/nodepool/proxy_pool.go @@ -19,10 +19,22 @@ func NewProxyPool() *ProxyPool { return &ProxyPool{} } -// SetNodes replaces the node list. Called on startup and when admin changes nodes. +// SetNodes replaces the node list. Called on startup and when admin changes +// nodes. Physical GPU identities are derived from each node's stored capability +// report here, so they exist from the first load rather than only after a +// capability refetch. func (p *ProxyPool) SetNodes(nodes []*Node) { p.mu.Lock() defer p.mu.Unlock() + for _, n := range nodes { + if n != nil { + // Same rule as the transcode pool: URLs are normalized where they + // enter the pool and where lookups are made, so a URL stored with + // a trailing slash still matches FindByURL's exact comparison. + n.URL = normalizeNodeURL(n.URL) + applyPhysicalGPUKeys(n) + } + } p.nodes = nodes } @@ -46,6 +58,20 @@ func (p *ProxyPool) Pick() *Node { return nil } +// FindByURL returns the node with the given URL, or nil if not found. Same +// contract as the transcode pool's: the caller has already selected the URL, +// so enabled and healthy are not filtered here. +func (p *ProxyPool) FindByURL(url string) *Node { + p.mu.RLock() + defer p.mu.RUnlock() + for _, n := range p.nodes { + if n.URL == url { + return n + } + } + return nil +} + // Nodes returns a copy of the current node list. func (p *ProxyPool) Nodes() []*Node { p.mu.RLock() @@ -57,8 +83,16 @@ func (p *ProxyPool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *ProxyPool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +func (p *ProxyPool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, advertisedHash, lastStats, checkedAt) +} + +// ApplyCapabilities records a freshly fetched capability report by swapping the +// node for an updated copy, keeping published *Node values immutable. +func (p *ProxyPool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) + applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) } diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index 21a48e9a6..93415b36f 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -1,9 +1,12 @@ package nodepool import ( + "bytes" "context" + "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -20,11 +23,18 @@ const ( // Node represents a stream node in the database. type Node struct { - ID int `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - URL string `json:"url"` - Enabled bool `json:"enabled"` + ID int `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + Enabled bool `json:"enabled"` + // PublicURL is the base URL streaming clients are given for this node, + // when it differs from URL. URL is the backend address — what the server + // and other nodes dial — so on a split network a proxy carries its + // private address in URL and its client-facing one here. Nil means + // clients use URL. Only ever read for proxy nodes: clients never talk to + // transcode nodes. + PublicURL *string `json:"public_url,omitempty"` Healthy bool `json:"healthy"` ActiveJobs int `json:"active_jobs"` Group *string `json:"group"` // co-location group; nil = ungrouped @@ -33,13 +43,139 @@ type Node struct { EgressKbps int `json:"egress_kbps"` // health-reported rolling egress average LastHealthCheck *time.Time `json:"last_health_check"` CreatedAt time.Time `json:"created_at"` + // Capabilities is the node's last stored capability report, verbatim as the + // node served it. Kept opaque here: nodepool must not depend on playback, + // and readers that need fields parse the ones they need. + Capabilities json.RawMessage `json:"capabilities,omitempty"` + // CapabilitiesHash identifies Capabilities. The health sweep compares it + // against the hash a node reports to decide whether to refetch. + CapabilitiesHash *string `json:"capabilities_hash,omitempty"` + // CapabilitiesRefreshedAt is when Capabilities was last fetched — the age of + // the inventory, not of the last health check. + CapabilitiesRefreshedAt *time.Time `json:"capabilities_refreshed_at,omitempty"` + // LastStats is the node's resource sample from the last health check — + // {"system":…,"gpu":…} — kept opaque for the same reason as Capabilities. + // It is written by the same 30s health update that writes ActiveJobs, so it + // is exactly as fresh as LastHealthCheck and never fresher. Absent for a + // node that reports no sample. + LastStats json.RawMessage `json:"last_stats,omitempty"` + // HWAccelOverride and HWDeviceOverride are this node's own acceleration + // policy. nil means the node inherits the cluster-wide playback.hw_accel / + // playback.hw_device settings, which is the normal case; a value here is + // what the node itself resolves against once it has reloaded its config. + HWAccelOverride *string `json:"hw_accel_override,omitempty"` + HWDeviceOverride *string `json:"hw_device_override,omitempty"` + // CapabilityDrift is an operator-facing note describing how this node's + // hardware got worse at the last capability refetch: a backend that used to + // pass its probe and now fails, or a render device that is gone. nil means + // the last refetch found no regression, which is also how a recovered node + // reads — the note is rewritten by every refetch and a clean report clears + // it. It is written beside Capabilities in one statement, so it always + // describes the report stored with it, and nothing routes on it. + CapabilityDrift *string `json:"capability_drift,omitempty"` + // CapabilityDriftBaseline records, machine-readably, the backends and device + // identities CapabilityDrift is waiting on. Recovery cannot be derived from + // the stored report alone — once a degraded report is stored every later + // comparison is degraded-to-degraded — so the note keeps what it is standing + // for. Non-nil exactly when CapabilityDrift is, except on a note written + // before this column existed. + CapabilityDriftBaseline json.RawMessage `json:"capability_drift_baseline,omitempty"` + // AdvertisedCapabilitiesHash is the hash the node named on its last health + // check, which is not always the one stored beside it: the sweep refetches + // on a mismatch, and a refetch that keeps failing leaves the two apart while + // the health check goes on succeeding every 30 seconds. Derived per sweep + // rather than persisted — it is an observation about right now, and a + // restarted API re-learns it on its first check. + // + // A pointer for three states, not two. nil is "nobody has checked this node + // yet", which every node reads as until the first sweep after a restart, and + // which says nothing about the stored report. A pointer to "" is the health + // check answering with no hash at all — a node downgraded to a build that + // predates capability reports — and that is a node no longer standing behind + // what is stored for it, which a reader must be able to tell from silence. + AdvertisedCapabilitiesHash *string `json:"advertised_capabilities_hash,omitempty"` + // PhysicalGPUKeys identifies the actual GPUs behind this node, derived from + // Capabilities rather than stored: it is a pure function of that payload, so + // a column would only be a second copy that can disagree with it. Two nodes + // sharing a key are backed by the same card — the case that makes + // independent per-node capacity accounting wrong, and which no single node's + // report can express. Last in the struct so it stays last on the wire, where + // the admin node list has always carried it. + PhysicalGPUKeys []string `json:"physical_gpu_keys,omitempty"` +} + +// EffectiveHWAccel is the acceleration backend this node runs under: its own +// override when it carries one, and otherwise the cluster-wide setting passed +// in. It is what a dispatch path names in a start request so the request, the +// recipe card, and what the node actually runs agree. +// +// Deliberately not derived from the node's stored capability report: that +// report is a snapshot up to a capability-refresh interval old, and naming its +// resolved backend would pin a stale answer *and* suppress the node's own +// start-time resolution — a node honors a named backend verbatim, so "auto" +// has to survive this far to reach live device enumeration on the node. +func (n *Node) EffectiveHWAccel(clusterHWAccel string) string { + if n == nil || n.HWAccelOverride == nil { + return clusterHWAccel + } + if override := strings.TrimSpace(*n.HWAccelOverride); override != "" { + return override + } + return clusterHWAccel +} + +// ClientURL is the base URL to hand streaming clients for this node: the +// public URL when one is set, otherwise the backend URL — which is every node +// registered before the split and every deployment with one flat network. +// Normalized like every other node URL so builders can join paths directly. +func (n *Node) ClientURL() string { + if n == nil { + return "" + } + if n.PublicURL != nil { + if public := strings.TrimSpace(*n.PublicURL); public != "" { + return normalizeNodeURL(public) + } + } + return normalizeNodeURL(n.URL) +} + +// StoredCapabilities returns this node's last stored capability report, nil-safe +// like the Effective* accessors so a caller whose lookup came up empty prices a +// missing node and a missing report through one path. +func (n *Node) StoredCapabilities() json.RawMessage { + if n == nil { + return nil + } + return n.Capabilities +} + +// EffectiveHWDevice is the device set this node runs under, resolved the same +// way as EffectiveHWAccel. +// +// Its readers care about the size of the set, not the paths: how many devices a +// node walks is what decides how long its cold capability probe takes, and a +// node overridden onto four devices needs several times the budget the cluster +// setting would price for it. +func (n *Node) EffectiveHWDevice(clusterHWDevice string) string { + if n == nil || n.HWDeviceOverride == nil { + return clusterHWDevice + } + if override := strings.TrimSpace(*n.HWDeviceOverride); override != "" { + return override + } + return clusterHWDevice } // CreateNodeInput holds the fields for creating a new node. type CreateNodeInput struct { - Name string `json:"name"` - Type string `json:"type"` - URL string `json:"url"` + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + // PublicURL is meaningful for proxy nodes only; see Node.PublicURL. + // Accepted on any node for symmetry with the acceleration overrides, + // which are likewise scoped by what reads them rather than rejected. + PublicURL string `json:"public_url"` Group string `json:"group"` // empty = ungrouped MaxJobs *int `json:"max_jobs"` // nil or <= 0 = unlimited MaxBandwidthKbps *int `json:"max_bandwidth_kbps"` // nil or <= 0 = unlimited @@ -61,17 +197,98 @@ func (i CreateNodeInput) Validate() error { // UpdateNodeInput holds the fields for updating a node. // The optional fields distinguish "leave unchanged" (nil) from "clear": -// an empty-string Group clears the group, and a non-positive MaxJobs or -// MaxBandwidthKbps clears that cap. +// an empty-string Group clears the group, an empty-string HWAccelOverride or +// HWDeviceOverride restores inheritance of the cluster-wide setting, and a +// non-positive MaxJobs or MaxBandwidthKbps clears that cap. type UpdateNodeInput struct { - Name *string `json:"name,omitempty"` - URL *string `json:"url,omitempty"` + Name *string `json:"name,omitempty"` + URL *string `json:"url,omitempty"` + // PublicURL follows the override convention: empty string (or JSON null) + // clears the column, so clients go back to using the backend URL. + PublicURL *string `json:"public_url,omitempty"` Enabled *bool `json:"enabled,omitempty"` Group *string `json:"group,omitempty"` MaxJobs *int `json:"max_jobs,omitempty"` MaxBandwidthKbps *int `json:"max_bandwidth_kbps,omitempty"` + HWAccelOverride *string `json:"hw_accel_override,omitempty"` + HWDeviceOverride *string `json:"hw_device_override,omitempty"` +} + +// UnmarshalJSON decodes an update body, mapping an explicit JSON null on the +// two acceleration overrides onto the empty-string clear sentinel the rest of +// this type uses. Plain decoding leaves a *string nil for an omitted field and +// for an explicit null alike, which would silently turn "go back to inheriting +// the cluster setting" into a no-op. Every other field decodes normally. +func (i *UpdateNodeInput) UnmarshalJSON(data []byte) error { + type plain UpdateNodeInput + var decoded plain + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + *i = UpdateNodeInput(decoded) + if isJSONNull(raw["hw_accel_override"]) { + i.HWAccelOverride = new(string) + } + if isJSONNull(raw["hw_device_override"]) { + i.HWDeviceOverride = new(string) + } + if isJSONNull(raw["public_url"]) { + i.PublicURL = new(string) + } + return nil +} + +// isJSONNull reports whether a field was present in the body with the literal +// value null (an absent field decodes to a nil RawMessage instead). +func isJSONNull(raw json.RawMessage) bool { + return string(bytes.TrimSpace(raw)) == "null" +} + +// Validate checks the values an update may set. Only the acceleration override +// has a closed set of values; the database CHECK enforces the same list, and +// rejecting here turns a constraint violation into an operator-readable error. +func (i UpdateNodeInput) Validate() error { + if i.HWAccelOverride == nil { + return nil + } + value := normalizeHWAccelOverride(*i.HWAccelOverride) + if value == nil { + // The clear sentinel: inherit the cluster-wide setting again. + return nil + } + if !slices.Contains(hwAccelOverrideValues, *value) { + return fmt.Errorf("%w: hw_accel_override must be one of %s", ErrInvalidNodeInput, + strings.Join(hwAccelOverrideValues, ", ")) + } + return nil } +// hwAccelOverrideValues mirrors the playback.hw_accel enum in +// internal/config/admin_settings.go and the CHECK constraint on +// stream_nodes.hw_accel_override: a per-node override may only name a backend +// the cluster-wide setting could also name. +var hwAccelOverrideValues = []string{hwAccelAuto, hwAccelQSV, hwAccelVAAPI, hwAccelNVENC, hwAccelNone} + +const ( + // hwAccelAuto asks the node to resolve its own backend against live + // hardware at session start; dispatch passes it through untouched for + // exactly that reason. + hwAccelAuto = "auto" + hwAccelQSV = "qsv" + hwAccelVAAPI = "vaapi" + hwAccelNVENC = "nvenc" + hwAccelNone = "none" +) + +// sameURL is true when an update leaves the row addressing the same worker. +// Trailing slashes are ignored on both sides because the pools normalize URLs +// and the column does not, so "http://n1/" becoming "http://n1" is not a move. +const sameURL = `rtrim(COALESCE($3, url), '/') = rtrim(url, '/')` + // normalizeGroup trims a group label and converts empty to NULL. func normalizeGroup(group string) *string { g := strings.TrimSpace(group) @@ -81,6 +298,26 @@ func normalizeGroup(group string) *string { return &g } +// normalizeOverride trims an override value and converts empty to NULL, which +// is how a node goes back to inheriting the cluster-wide setting. Case is +// preserved: a render device path is a filesystem path, not an enum. +func normalizeOverride(value string) *string { + v := strings.TrimSpace(value) + if v == "" { + return nil + } + return &v +} + +// normalizeHWAccelOverride is normalizeOverride for the acceleration enum, +// which is also lowercased. The cluster-wide playback.hw_accel accepts any +// casing (config.normalizeAdminEnum lowercases before comparing), and +// docs/admin-api.md promises the override takes the same values, so "QSV" from +// a third-party admin client must not be a 400 here when it is a 200 there. +func normalizeHWAccelOverride(value string) *string { + return normalizeOverride(strings.ToLower(value)) +} + // normalizeCap converts non-positive capacity values to NULL (unlimited). func normalizeCap(v *int) *int { if v == nil || *v <= 0 { @@ -99,37 +336,52 @@ func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} } -const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at` +const nodeColumns = `id, name, type, url, public_url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at, capabilities, capabilities_hash, capabilities_refreshed_at, last_stats, hw_accel_override, hw_device_override, capability_drift, capability_drift_baseline` func scanNode(row pgx.Row) (*Node, error) { var n Node + // jsonb is scanned as raw bytes rather than into json.RawMessage directly so + // a NULL column stays nil instead of decoding through the JSON codec. + var capabilities, lastStats, driftBaselineBytes []byte err := row.Scan( - &n.ID, &n.Name, &n.Type, &n.URL, + &n.ID, &n.Name, &n.Type, &n.URL, &n.PublicURL, &n.Enabled, &n.Healthy, &n.ActiveJobs, &n.Group, &n.MaxJobs, &n.MaxBandwidthKbps, &n.EgressKbps, &n.LastHealthCheck, &n.CreatedAt, + &capabilities, &n.CapabilitiesHash, &n.CapabilitiesRefreshedAt, + &lastStats, + &n.HWAccelOverride, &n.HWDeviceOverride, + &n.CapabilityDrift, &driftBaselineBytes, ) if err != nil { return nil, err } + if len(capabilities) > 0 { + n.Capabilities = json.RawMessage(capabilities) + } + if len(lastStats) > 0 { + n.LastStats = json.RawMessage(lastStats) + } + if len(driftBaselineBytes) > 0 { + n.CapabilityDriftBaseline = json.RawMessage(driftBaselineBytes) + } + // Derived here so every reader of a stored row — the admin listing as much + // as a pool load — sees the same identities without parsing the payload + // again for itself. + applyPhysicalGPUKeys(&n) return &n, nil } func scanNodes(rows pgx.Rows) ([]*Node, error) { var nodes []*Node for rows.Next() { - var n Node - if err := rows.Scan( - &n.ID, &n.Name, &n.Type, &n.URL, - &n.Enabled, &n.Healthy, &n.ActiveJobs, - &n.Group, &n.MaxJobs, - &n.MaxBandwidthKbps, &n.EgressKbps, - &n.LastHealthCheck, &n.CreatedAt, - ); err != nil { + // pgx.Rows satisfies pgx.Row, so both paths share one column list. + n, err := scanNode(rows) + if err != nil { return nil, err } - nodes = append(nodes, &n) + nodes = append(nodes, n) } return nodes, rows.Err() } @@ -177,18 +429,21 @@ func (r *Repository) Create(ctx context.Context, input CreateNodeInput) (*Node, return nil, err } row := r.pool.QueryRow(ctx, - `INSERT INTO stream_nodes (name, type, url, node_group, max_jobs, max_bandwidth_kbps) - VALUES ($1, $2, $3, $4, $5, $6) + `INSERT INTO stream_nodes (name, type, url, public_url, node_group, max_jobs, max_bandwidth_kbps) + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING `+nodeColumns, - input.Name, input.Type, input.URL, normalizeGroup(input.Group), + input.Name, input.Type, input.URL, normalizeOverride(input.PublicURL), normalizeGroup(input.Group), normalizeCap(input.MaxJobs), normalizeCap(input.MaxBandwidthKbps)) return scanNode(row) } // Update modifies a node's mutable fields. The optional fields use sentinel -// values to clear: an empty-string group and non-positive caps set the -// column to NULL (see UpdateNodeInput). +// values to clear: an empty-string group, an empty-string acceleration +// override, and non-positive caps set the column to NULL (see UpdateNodeInput). func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) (*Node, error) { + if err := input.Validate(); err != nil { + return nil, err + } var group *string if input.Group != nil { group = normalizeGroup(*input.Group) @@ -200,6 +455,17 @@ func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) if input.MaxBandwidthKbps != nil { maxBandwidth = normalizeCap(input.MaxBandwidthKbps) } + var hwAccelOverride, hwDeviceOverride *string + if input.HWAccelOverride != nil { + hwAccelOverride = normalizeHWAccelOverride(*input.HWAccelOverride) + } + if input.HWDeviceOverride != nil { + hwDeviceOverride = normalizeOverride(*input.HWDeviceOverride) + } + var publicURL *string + if input.PublicURL != nil { + publicURL = normalizeOverride(*input.PublicURL) + } row := r.pool.QueryRow(ctx, `UPDATE stream_nodes SET name = COALESCE($2, name), @@ -207,13 +473,34 @@ func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) enabled = COALESCE($4, enabled), node_group = CASE WHEN $5::boolean THEN $6::text ELSE node_group END, max_jobs = CASE WHEN $7::boolean THEN $8::integer ELSE max_jobs END, - max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END + max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END, + hw_accel_override = CASE WHEN $11::boolean THEN $12::text ELSE hw_accel_override END, + hw_device_override = CASE WHEN $13::boolean THEN $14::text ELSE hw_device_override END, + public_url = CASE WHEN $15::boolean THEN $16::text ELSE public_url END, + -- Everything below describes the worker the old URL addressed, so + -- repointing the row at a different machine has to drop it. The + -- caller publishes the returned row to the pools immediately, and + -- these are exactly the fields placement reads: the GPU identities + -- behind physical_gpu_keys and the scratch fill behind admission. + -- Keeping them would route work onto the replacement using its + -- predecessor's hardware until a health check and a capability + -- fetch caught up. NULL is the same state a freshly registered node + -- is in, which is the truth here. + capabilities = CASE WHEN `+sameURL+` THEN capabilities END, + capabilities_hash = CASE WHEN `+sameURL+` THEN capabilities_hash END, + capabilities_refreshed_at = CASE WHEN `+sameURL+` THEN capabilities_refreshed_at END, + last_stats = CASE WHEN `+sameURL+` THEN last_stats END, + capability_drift = CASE WHEN `+sameURL+` THEN capability_drift END, + capability_drift_baseline = CASE WHEN `+sameURL+` THEN capability_drift_baseline END WHERE id = $1 RETURNING `+nodeColumns, id, input.Name, input.URL, input.Enabled, input.Group != nil, group, input.MaxJobs != nil, maxJobs, - input.MaxBandwidthKbps != nil, maxBandwidth) + input.MaxBandwidthKbps != nil, maxBandwidth, + input.HWAccelOverride != nil, hwAccelOverride, + input.HWDeviceOverride != nil, hwDeviceOverride, + input.PublicURL != nil, publicURL) n, err := scanNode(row) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNodeNotFound @@ -236,21 +523,103 @@ func (r *Repository) Delete(ctx context.Context, id int) error { return nil } -// UpdateHealth updates a node's health status, active job count, and -// reported egress bandwidth. -func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int) error { +// UpdateHealth updates a node's health status, active job count, reported +// egress bandwidth, and last resource sample. +// +// A nil lastStats writes NULL, which is what a node that reports no sample — +// an older build, or a non-Linux host — must produce. Passing the previous +// value through instead would leave a dead node's numbers on screen looking +// current. +// checkedURL fences the write the same way UpdateCapabilities does. The window +// is smaller — a health request is bounded at five seconds — but the +// consequence is not: last_stats carries the scratch fill that transcode +// admission reads, so one worker's disk reading landing on a row that now +// addresses another can exclude a healthy node or admit a full one. +func (r *Repository) UpdateHealth(ctx context.Context, id int, checkedURL string, healthy bool, activeJobs, egressKbps int, lastStats []byte) error { tag, err := r.pool.Exec(ctx, - `UPDATE stream_nodes SET healthy = $2, active_jobs = $3, egress_kbps = $4, last_health_check = NOW() - WHERE id = $1`, - id, healthy, activeJobs, egressKbps) + `UPDATE stream_nodes SET healthy = $2, active_jobs = $3, egress_kbps = $4, last_stats = $5, last_health_check = NOW() + WHERE id = $1 AND rtrim(url, '/') = rtrim($6, '/')`, + id, healthy, activeJobs, egressKbps, lastStats, checkedURL) if err != nil { return fmt.Errorf("update node health: %w", err) } if tag.RowsAffected() == 0 { - return ErrNodeNotFound + return ErrNodeMoved } return nil } +// UpdateCapabilities persists a freshly fetched capability report together with +// the hash that identifies it and the drift note comparing it against the +// previous one. The four columns are written in one statement so a reader never +// sees a payload beside a hash — or a drift note — from a different report. +// +// A nil drift writes NULL, which is how a node that has recovered stops being +// flagged: the note describes the last comparison, not a latched incident, so +// carrying the previous value forward would keep a repaired driver on screen as +// broken. +// fetchedFrom fences the write against the row having been repointed while the +// fetch was in flight. A capability fetch runs detached from the sweep and is +// bounded at two minutes, which is ample time for an administrator to edit the +// node's URL; an id-only write would then store one worker's GPU identities on +// a row that now addresses a different machine, and the planner would place +// shared-GPU work on that reading until another sweep corrected it. Trailing +// slashes are ignored on both sides because the pools normalize URLs and the +// column does not. +// It is also fenced on the report it believes it is replacing. Every API +// replica runs its own health sweep, so two can fetch successive reports from +// one node concurrently; without this a slower fetch of an older report lands +// after a newer one and overwrites it, taking the durable GPU identities and +// drift state back with it until some later sweep repairs them. Comparing +// against the hash the caller read before fetching makes the write a +// compare-and-set: whichever replica gets there first wins, and the loser +// discards a report that no longer describes the row it was derived from. +// Clock skew between replicas does not enter into it. +func (r *Repository) UpdateCapabilities(ctx context.Context, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte, replacing *string) error { + tag, err := r.pool.Exec(ctx, + `UPDATE stream_nodes SET capabilities = $2, capabilities_hash = $3, capabilities_refreshed_at = $4, capability_drift = $5, capability_drift_baseline = $7 + WHERE id = $1 AND rtrim(url, '/') = rtrim($6, '/') AND capabilities_hash IS NOT DISTINCT FROM $8`, + id, capabilities, hash, refreshedAt, drift, fetchedFrom, driftBaseline, replacing) + if err != nil { + return fmt.Errorf("update node capabilities: %w", err) + } + if tag.RowsAffected() == 0 { + // Three ways to get here, and they do not mean the same thing. The row + // being gone or repointed is terminal for this payload; another replica + // having stored a different report is not — that replica's answer is the + // current one, and this one's in-memory copy is now behind it. Telling + // them apart costs one read and saves a replica sweeping forever against + // a hash the row no longer has. + current, err := r.GetByID(ctx, id) + if err != nil || current == nil || !sameStoredURL(current.URL, fetchedFrom) { + return ErrNodeMoved + } + return ErrCapabilitiesSuperseded + } + return nil +} + +// sameStoredURL compares a stored node URL with the one a payload was fetched +// from, ignoring the trailing slash the pools normalize and the column does not. +func sameStoredURL(stored, fetchedFrom string) bool { + return strings.TrimRight(stored, "/") == strings.TrimRight(fetchedFrom, "/") +} + // Sentinel errors. -var ErrNodeNotFound = errors.New("stream node not found") +var ( + ErrNodeNotFound = errors.New("stream node not found") + // ErrInvalidNodeInput marks a caller-supplied value the store refuses, so + // an API layer can answer 400 without string-matching the message. + ErrInvalidNodeInput = errors.New("invalid node input") + // ErrNodeMoved reports that a row no longer matches the identity a + // long-running fetch was made against — it was deleted, or its URL was + // edited to address a different worker. The result must be discarded rather + // than published against whatever the row is now. + ErrNodeMoved = errors.New("stream node no longer matches the fetched identity") + + // ErrCapabilitiesSuperseded reports that the row still addresses this + // worker but another writer stored a different report first. The payload is + // discarded; unlike ErrNodeMoved the caller has something to learn from the + // row, because the report now on it is newer than the one it started from. + ErrCapabilitiesSuperseded = errors.New("stream node capabilities were superseded by another writer") +) diff --git a/internal/nodepool/repository_capabilities_test.go b/internal/nodepool/repository_capabilities_test.go new file mode 100644 index 000000000..d3922f026 --- /dev/null +++ b/internal/nodepool/repository_capabilities_test.go @@ -0,0 +1,338 @@ +package nodepool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// newNodeTestPool follows the repository-wide convention for tests that need a +// real database: skip unless one is configured, and skip again if it predates +// the migration under test rather than failing on a missing column. +func newNodeTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + // Every read here selects the full column list, so the newest column is the + // one worth probing: a database missing it fails every test in the package. + var columns int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM information_schema.columns + WHERE table_name = 'stream_nodes' + AND column_name IN ('capabilities_hash', 'hw_accel_override', 'capability_drift')`).Scan(&columns); err != nil || columns < 3 { + t.Skip("test database has not applied the stream_nodes capability/override migrations") + } + return pool +} + +// Stored capabilities are what makes GPU inventory survive an API restart, so +// the payload, its hash, and its age must all come back through an ordinary +// List — the same read the pools and the admin API use. +func TestRepositoryUpdateCapabilitiesRoundTrip(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("capability-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://capability-test-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + if node.Capabilities != nil || node.CapabilitiesHash != nil || node.CapabilitiesRefreshedAt != nil { + t.Fatalf("new node already carries capabilities: %+v", node) + } + + payload := json.RawMessage(`{"resolved":"nvenc","render_devices":["/dev/dri/renderD128"]}`) + refreshedAt := time.Now().UTC().Truncate(time.Millisecond) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:abc", refreshedAt, nil, nil, nil); err != nil { + t.Fatalf("update capabilities: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + var stored, want map[string]any + if err := json.Unmarshal(reloaded.Capabilities, &stored); err != nil { + t.Fatalf("stored capabilities are not json: %v", err) + } + if err := json.Unmarshal(payload, &want); err != nil { + t.Fatal(err) + } + if fmt.Sprint(stored) != fmt.Sprint(want) { + t.Fatalf("stored capabilities = %v, want %v", stored, want) + } + if reloaded.CapabilitiesHash == nil || *reloaded.CapabilitiesHash != "sha256:abc" { + t.Fatalf("stored hash = %v", reloaded.CapabilitiesHash) + } + if reloaded.CapabilitiesRefreshedAt == nil || !reloaded.CapabilitiesRefreshedAt.UTC().Equal(refreshedAt) { + t.Fatalf("stored refresh time = %v, want %v", reloaded.CapabilitiesRefreshedAt, refreshedAt) + } +} + +// The drift note is what puts a silent hardware regression on the node list, so +// it has to survive a write and come back through an ordinary read — and a later +// clean refetch has to clear it, or a repaired node stays flagged forever. +func TestRepositoryUpdateCapabilitiesDriftRoundTripAndClear(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("drift-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://drift-test-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + if node.CapabilityDrift != nil { + t.Fatalf("new node already carries drift: %q", *node.CapabilityDrift) + } + + note := "verified hardware backends lost: nvenc; resolved backend nvenc -> none" + payload := json.RawMessage(`{"resolved":"none"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:degraded", time.Now(), ¬e, nil, nil); err != nil { + t.Fatalf("update capabilities with drift: %v", err) + } + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.CapabilityDrift == nil || *reloaded.CapabilityDrift != note { + t.Fatalf("stored drift = %v, want %q", reloaded.CapabilityDrift, note) + } + + recovered := json.RawMessage(`{"resolved":"nvenc"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, recovered, "sha256:recovered", time.Now(), nil, nil, ptrString("sha256:degraded")); err != nil { + t.Fatalf("update capabilities without drift: %v", err) + } + reloaded, err = repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.CapabilityDrift != nil { + t.Fatalf("drift = %q after recovery, want NULL", *reloaded.CapabilityDrift) + } +} + +func TestRepositoryUpdateCapabilitiesUnknownNode(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + err := repo.UpdateCapabilities(context.Background(), -1, "http://gone", []byte(`{}`), "sha256:abc", time.Now(), nil, nil, nil) + if !errors.Is(err, ErrNodeMoved) { + t.Fatalf("err = %v, want ErrNodeMoved", err) + } +} + +// A capability fetch is detached from the sweep and bounded at two minutes, +// which is ample time for an administrator to repoint the row at a different +// machine. Writing by id alone would store one worker's GPU identities against +// another's URL, and the planner would place shared-GPU work on that reading +// until a later sweep corrected it. +func TestRepositoryUpdateCapabilitiesRefusesAfterAURLEdit(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("moved-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://moved-test-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + moved := node.URL + "-elsewhere" + if _, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &moved}); err != nil { + t.Fatalf("repoint node: %v", err) + } + + err = repo.UpdateCapabilities(ctx, node.ID, node.URL, []byte(`{"resolved":"qsv"}`), "sha256:stale", time.Now(), nil, nil, nil) + if !errors.Is(err, ErrNodeMoved) { + t.Fatalf("err = %v, want ErrNodeMoved after the row was repointed", err) + } + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if len(reloaded.Capabilities) != 0 { + t.Fatalf("capabilities = %s, want the stale payload discarded", reloaded.Capabilities) + } +} + +// The trailing slash a pool trims is not a different node: the pools normalize +// URLs and the column does not, so an exact match would fence out every +// legitimate write for a row registered with one. +func TestRepositoryUpdateCapabilitiesIgnoresATrailingSlash(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("slash-test-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://slash-test-%d/", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + normalized := strings.TrimSuffix(node.URL, "/") + if err := repo.UpdateCapabilities(ctx, node.ID, normalized, []byte(`{"resolved":"qsv"}`), "sha256:ok", time.Now(), nil, nil, nil); err != nil { + t.Fatalf("UpdateCapabilities with a normalized URL: %v", err) + } +} + +// Everything a capability report and a health sample describe belongs to the +// worker the URL addressed. Repointing the row at a different machine has to +// drop it: the caller publishes the returned row to the pools immediately, and +// these are the fields placement reads — the GPU identities behind +// physical_gpu_keys and the scratch fill behind admission. +func TestRepositoryUpdateClearsWorkerStateWhenTheURLMoves(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("repoint-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://repoint-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + note := "verified hardware backends lost: qsv" + payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:old", time.Now(), ¬e, []byte(`{"backends":["qsv"]}`), nil); err != nil { + t.Fatalf("store capabilities: %v", err) + } + if err := repo.UpdateHealth(ctx, node.ID, node.URL, true, 2, 0, []byte(`{"system":{"cpu_pct":41}}`)); err != nil { + t.Fatalf("store health: %v", err) + } + + moved := node.URL + "-elsewhere" + updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &moved}) + if err != nil { + t.Fatalf("repoint node: %v", err) + } + + if len(updated.Capabilities) != 0 || updated.CapabilitiesHash != nil || updated.CapabilitiesRefreshedAt != nil { + t.Fatalf("capabilities survived a repoint: %+v", updated) + } + if len(updated.LastStats) != 0 { + t.Fatalf("last_stats survived a repoint: %s", updated.LastStats) + } + if updated.CapabilityDrift != nil || len(updated.CapabilityDriftBaseline) != 0 { + t.Fatalf("drift survived a repoint: %v / %s", updated.CapabilityDrift, updated.CapabilityDriftBaseline) + } + if len(updated.PhysicalGPUKeys) != 0 { + t.Fatalf("GPU identities survived a repoint: %v", updated.PhysicalGPUKeys) + } +} + +// An edit that leaves the row on the same worker keeps its state — including +// when only a trailing slash differs, which the pools normalize away and the +// column does not. +func TestRepositoryUpdateKeepsWorkerStateWithoutAMove(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("stay-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://stay-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + payload := []byte(`{"resolved":"qsv","render_device_details":[{"path":"/dev/dri/renderD128","pci_address":"0000:03:00.0"}],"boot_id":"boot-1"}`) + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, payload, "sha256:keep", time.Now(), nil, nil, nil); err != nil { + t.Fatalf("store capabilities: %v", err) + } + + renamed := "renamed-" + node.Name + if updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{Name: &renamed}); err != nil { + t.Fatalf("rename node: %v", err) + } else if len(updated.Capabilities) == 0 || updated.CapabilitiesHash == nil { + t.Fatalf("a rename dropped the worker's capabilities: %+v", updated) + } + + slashed := node.URL + "/" + if updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &slashed}); err != nil { + t.Fatalf("re-save url: %v", err) + } else if len(updated.Capabilities) == 0 { + t.Fatal("a trailing-slash change was treated as a different worker") + } +} + +func ptrString(value string) *string { return &value } + +// Every API replica runs its own health sweep, so two can fetch successive +// reports from one node at once. Without a fence on the report being replaced, +// a slower fetch of the older one lands last and takes the durable GPU +// identities and drift state back with it. +func TestUpdateCapabilitiesRefusesAReportThatFollowsAStaleOne(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("capability-cas-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://capability-cas-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"qsv"}`), "sha256:first", time.Now(), nil, nil, nil); err != nil { + t.Fatalf("first report: %v", err) + } + // A second replica stores its newer report. + if err := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"nvenc"}`), "sha256:second", time.Now(), nil, nil, ptrString("sha256:first")); err != nil { + t.Fatalf("second report: %v", err) + } + // The first replica's *other* in-flight fetch finally lands. It read the + // row before either write, so it must not overwrite what is there now. + overtaken := repo.UpdateCapabilities(ctx, node.ID, node.URL, + []byte(`{"resolved":"vaapi"}`), "sha256:overtaken", time.Now(), nil, nil, ptrString("sha256:first")) + // Superseded, not moved: the row still addresses this worker, and the + // caller has something to learn from it. The distinction is what stops the + // losing replica from sweeping against a stale hash forever. + if !errors.Is(overtaken, ErrCapabilitiesSuperseded) { + t.Fatalf("overtaken report error = %v, want ErrCapabilitiesSuperseded", overtaken) + } + if errors.Is(overtaken, ErrNodeMoved) { + t.Fatal("a superseded report reported as a moved node; the caller would discard rather than reconcile") + } + + stored, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if stored.CapabilitiesHash == nil || *stored.CapabilitiesHash != "sha256:second" { + t.Fatalf("stored hash = %v, want the newer report kept", stored.CapabilitiesHash) + } +} diff --git a/internal/nodepool/repository_hw_overrides_test.go b/internal/nodepool/repository_hw_overrides_test.go new file mode 100644 index 000000000..4e102494d --- /dev/null +++ b/internal/nodepool/repository_hw_overrides_test.go @@ -0,0 +1,284 @@ +package nodepool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" +) + +// The two override fields are the whole point of per-node acceleration policy, +// so setting one, reading it back through an ordinary List/Get, and clearing it +// again all have to survive the round trip through Postgres. +func TestRepositoryUpdateHWOverridesRoundTrip(t *testing.T) { + pool := newNodeTestPool(t) + ctx := context.Background() + repo := NewRepository(pool) + + node := createTestNode(t, repo, "hw-override") + + if node.HWAccelOverride != nil || node.HWDeviceOverride != nil { + t.Fatalf("new node already carries overrides: %+v", node) + } + + accel, device := "vaapi", "/dev/dri/renderD129" + updated, err := repo.Update(ctx, node.ID, UpdateNodeInput{ + HWAccelOverride: &accel, + HWDeviceOverride: &device, + }) + if err != nil { + t.Fatalf("set overrides: %v", err) + } + if updated.HWAccelOverride == nil || *updated.HWAccelOverride != accel { + t.Fatalf("hw_accel_override = %v, want %q", updated.HWAccelOverride, accel) + } + if updated.HWDeviceOverride == nil || *updated.HWDeviceOverride != device { + t.Fatalf("hw_device_override = %v, want %q", updated.HWDeviceOverride, device) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.HWAccelOverride == nil || *reloaded.HWAccelOverride != accel { + t.Fatalf("reloaded hw_accel_override = %v, want %q", reloaded.HWAccelOverride, accel) + } + + // An unrelated update must not disturb the overrides: omitted means + // unchanged, exactly like the other optional fields. + renamed := node.Name + "-renamed" + untouched, err := repo.Update(ctx, node.ID, UpdateNodeInput{Name: &renamed}) + if err != nil { + t.Fatalf("rename node: %v", err) + } + if untouched.HWAccelOverride == nil || *untouched.HWAccelOverride != accel { + t.Fatalf("unrelated update dropped hw_accel_override: %v", untouched.HWAccelOverride) + } + + // The empty-string sentinel restores inheritance of the cluster setting. + cleared, err := repo.Update(ctx, node.ID, UpdateNodeInput{ + HWAccelOverride: new(string), + HWDeviceOverride: new(string), + }) + if err != nil { + t.Fatalf("clear overrides: %v", err) + } + if cleared.HWAccelOverride != nil || cleared.HWDeviceOverride != nil { + t.Fatalf("overrides after clear = %v / %v, want nil (inherit)", cleared.HWAccelOverride, cleared.HWDeviceOverride) + } +} + +// A value outside the enum must be refused before it reaches the CHECK +// constraint, so the operator sees which values are legal. +func TestRepositoryUpdateRejectsUnknownHWAccelOverride(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + node := createTestNode(t, repo, "hw-override-invalid") + + bogus := "videotoolbox" + _, err := repo.Update(context.Background(), node.ID, UpdateNodeInput{HWAccelOverride: &bogus}) + if !errors.Is(err, ErrInvalidNodeInput) { + t.Fatalf("err = %v, want ErrInvalidNodeInput", err) + } + + reloaded, err := repo.GetByID(context.Background(), node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.HWAccelOverride != nil { + t.Fatalf("rejected update still wrote %v", reloaded.HWAccelOverride) + } +} + +// A mixed-case override is accepted like the cluster-wide setting accepts one, +// and reaches the column lowercase — the only casing its CHECK allows. +func TestRepositoryUpdateLowercasesHWAccelOverride(t *testing.T) { + repo := NewRepository(newNodeTestPool(t)) + node := createTestNode(t, repo, "hw-override-case") + + value, device := "QSV", "/dev/dri/renderD129" + updated, err := repo.Update(context.Background(), node.ID, UpdateNodeInput{ + HWAccelOverride: &value, + HWDeviceOverride: &device, + }) + if err != nil { + t.Fatalf("set overrides: %v", err) + } + if updated.HWAccelOverride == nil || *updated.HWAccelOverride != "qsv" { + t.Fatalf("hw_accel_override = %v, want %q", updated.HWAccelOverride, "qsv") + } + // The device is a path, not an enum: case survives. + if updated.HWDeviceOverride == nil || *updated.HWDeviceOverride != device { + t.Fatalf("hw_device_override = %v, want %q", updated.HWDeviceOverride, device) + } +} + +func createTestNode(t *testing.T, repo *Repository, prefix string) *Node { + t.Helper() + ctx := context.Background() + unique := time.Now().UnixNano() + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("%s-%d", prefix, unique), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://%s-%d", prefix, unique), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + return node +} + +// An admin UI clears a field by sending null. Standard decoding cannot tell +// that apart from an omitted field, so the type maps it onto the clear +// sentinel; getting this wrong makes "inherit again" a silent no-op. +func TestUpdateNodeInputDecodesExplicitNullAsClear(t *testing.T) { + tests := []struct { + name string + body string + wantAccel *string + wantDevice *string + }{ + {name: "omitted leaves both unchanged", body: `{"name":"gpu-1"}`}, + { + name: "explicit null clears the accel override", + body: `{"hw_accel_override":null}`, + wantAccel: new(string), + }, + { + name: "explicit null clears both", + body: `{"hw_accel_override":null,"hw_device_override":null}`, + wantAccel: new(string), + wantDevice: new(string), + }, + { + name: "values decode normally", + body: `{"hw_accel_override":"qsv","hw_device_override":"/dev/dri/renderD128"}`, + wantAccel: ptrTo("qsv"), + wantDevice: ptrTo("/dev/dri/renderD128"), + }, + { + name: "empty string is the same clear sentinel", + body: `{"hw_accel_override":""}`, + wantAccel: new(string), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var input UpdateNodeInput + if err := json.Unmarshal([]byte(test.body), &input); err != nil { + t.Fatalf("decode: %v", err) + } + if !equalStringPtr(input.HWAccelOverride, test.wantAccel) { + t.Fatalf("HWAccelOverride = %v, want %v", input.HWAccelOverride, test.wantAccel) + } + if !equalStringPtr(input.HWDeviceOverride, test.wantDevice) { + t.Fatalf("HWDeviceOverride = %v, want %v", input.HWDeviceOverride, test.wantDevice) + } + }) + } +} + +// Decoding must not lose the fields it always carried. +func TestUpdateNodeInputKeepsExistingFields(t *testing.T) { + var input UpdateNodeInput + body := `{"name":"gpu-1","url":"http://gpu-1","enabled":true,"group":"rack-a","max_jobs":3,"max_bandwidth_kbps":0}` + if err := json.Unmarshal([]byte(body), &input); err != nil { + t.Fatalf("decode: %v", err) + } + if input.Name == nil || *input.Name != "gpu-1" || input.URL == nil || *input.URL != "http://gpu-1" { + t.Fatalf("identity fields lost: %+v", input) + } + if input.Enabled == nil || !*input.Enabled || input.Group == nil || *input.Group != "rack-a" { + t.Fatalf("enabled/group lost: %+v", input) + } + if input.MaxJobs == nil || *input.MaxJobs != 3 || input.MaxBandwidthKbps == nil || *input.MaxBandwidthKbps != 0 { + t.Fatalf("caps lost: %+v", input) + } +} + +func TestUpdateNodeInputValidate(t *testing.T) { + for _, value := range hwAccelOverrideValues { + if err := (UpdateNodeInput{HWAccelOverride: &value}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", value, err) + } + } + if err := (UpdateNodeInput{}).Validate(); err != nil { + t.Fatalf("Validate(omitted) = %v, want nil", err) + } + if err := (UpdateNodeInput{HWAccelOverride: new(string)}).Validate(); err != nil { + t.Fatalf("Validate(clear) = %v, want nil", err) + } + bogus := "cuda" + if err := (UpdateNodeInput{HWAccelOverride: &bogus}).Validate(); !errors.Is(err, ErrInvalidNodeInput) { + t.Fatalf("Validate(%q) = %v, want ErrInvalidNodeInput", bogus, err) + } +} + +// The cluster-wide playback.hw_accel accepts any casing, and the override is +// documented as taking the same values, so a third-party admin client must not +// get a 400 here for a body /admin/settings would have accepted. +func TestUpdateNodeInputValidateIgnoresCase(t *testing.T) { + for _, value := range []string{"QSV", " Vaapi ", "NONE", "Auto"} { + if err := (UpdateNodeInput{HWAccelOverride: &value}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", value, err) + } + } +} + +// Only the acceleration enum is case-folded on the way to the column, whose +// CHECK list is lowercase. A render device is a filesystem path and keeps its +// case. +func TestNormalizeHWAccelOverride(t *testing.T) { + if got := normalizeHWAccelOverride(" QSV "); got == nil || *got != "qsv" { + t.Fatalf("normalizeHWAccelOverride(%q) = %v, want %q", " QSV ", got, "qsv") + } + if got := normalizeHWAccelOverride(" "); got != nil { + t.Fatalf("blank override = %v, want nil (inherit)", got) + } + if got := normalizeOverride(" /dev/dri/renderD129 "); got == nil || *got != "/dev/dri/renderD129" { + t.Fatalf("device override = %v, want the path unchanged", got) + } +} + +// Dispatch names the node's own override, and otherwise the cluster value +// verbatim — "auto" included, so the node resolves it against live hardware at +// session start instead of inheriting a snapshot's answer. +func TestNodeEffectiveHWAccel(t *testing.T) { + tests := []struct { + name string + node *Node + cluster string + want string + }{ + {name: "no node at all", cluster: "qsv", want: "qsv"}, + {name: "no override inherits", node: &Node{}, cluster: "qsv", want: "qsv"}, + {name: "no override inherits auto", node: &Node{}, cluster: hwAccelAuto, want: hwAccelAuto}, + {name: "override wins", node: &Node{HWAccelOverride: ptrTo("none")}, cluster: "qsv", want: "none"}, + {name: "override wins over auto", node: &Node{HWAccelOverride: ptrTo(hwAccelNVENC)}, cluster: hwAccelAuto, want: hwAccelNVENC}, + {name: "blank override is not an override", node: &Node{HWAccelOverride: ptrTo(" ")}, cluster: "qsv", want: "qsv"}, + { + name: "a stale capability report is not consulted", + node: &Node{Capabilities: json.RawMessage(`{"resolved":"none"}`)}, + cluster: "qsv", + want: "qsv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.node.EffectiveHWAccel(test.cluster); got != test.want { + t.Fatalf("EffectiveHWAccel(%q) = %q, want %q", test.cluster, got, test.want) + } + }) + } +} + +func ptrTo(v string) *string { return &v } + +func equalStringPtr(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} diff --git a/internal/nodepool/repository_last_stats_test.go b/internal/nodepool/repository_last_stats_test.go new file mode 100644 index 000000000..5edf93f88 --- /dev/null +++ b/internal/nodepool/repository_last_stats_test.go @@ -0,0 +1,156 @@ +package nodepool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" +) + +// newLastStatsTestPool follows the repository-wide convention: skip without a +// database, and skip again on a database that predates the migration under test +// rather than failing on a missing column. +func newLastStatsTestPool(t *testing.T) *Repository { + t.Helper() + pool := newNodeTestPool(t) + var column *string + if err := pool.QueryRow(context.Background(), + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'stream_nodes' AND column_name = 'last_stats'`).Scan(&column); err != nil { + t.Skip("test database has not applied the node last_stats migration") + } + return NewRepository(pool) +} + +func createLastStatsNode(t *testing.T, repo *Repository) *Node { + t.Helper() + ctx := context.Background() + unique := time.Now().UnixNano() + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("last-stats-test-%d", unique), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://last-stats-test-%d", unique), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + return node +} + +// The health sweep's existing 30s write is the only write path for resource +// stats. It has to round-trip through an ordinary List — the same read the +// pools and the admin API use. +func TestRepositoryUpdateHealthPersistsLastStats(t *testing.T) { + repo := newLastStatsTestPool(t) + ctx := context.Background() + node := createLastStatsNode(t, repo) + + if node.LastStats != nil { + t.Fatalf("new node already carries stats: %s", node.LastStats) + } + + stats := []byte(`{"system":{"cpu_pct":41,"mem_used_mb":9011},"gpu":[{"device":"/dev/dri/renderD128","source":"fdinfo"}]}`) + if err := repo.UpdateHealth(ctx, node.ID, node.URL, true, 3, 17, stats); err != nil { + t.Fatalf("update health: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.ActiveJobs != 3 || reloaded.EgressKbps != 17 || !reloaded.Healthy { + t.Fatalf("existing health fields = %+v, want them unchanged by the new column", reloaded) + } + var decoded struct { + System struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + } `json:"gpu"` + } + if err := json.Unmarshal(reloaded.LastStats, &decoded); err != nil { + t.Fatalf("stored stats are not json: %v (%s)", err, reloaded.LastStats) + } + if decoded.System.CPUPct != 41 { + t.Fatalf("stored system.cpu_pct = %d, want 41", decoded.System.CPUPct) + } + if len(decoded.GPU) != 1 || decoded.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("stored gpu = %+v", decoded.GPU) + } +} + +// A node that reports no sample writes NULL, and a node that stops reporting +// clears what it had. Keeping the previous row would leave a dead node's +// numbers on the Nodes page looking current. +func TestRepositoryUpdateHealthWritesNullForNodesWithoutStats(t *testing.T) { + repo := newLastStatsTestPool(t) + ctx := context.Background() + node := createLastStatsNode(t, repo) + + if err := repo.UpdateHealth(ctx, node.ID, node.URL, true, 1, 0, []byte(`{"system":{"cpu_pct":41}}`)); err != nil { + t.Fatalf("update health: %v", err) + } + if err := repo.UpdateHealth(ctx, node.ID, node.URL, false, 0, 0, nil); err != nil { + t.Fatalf("update health without stats: %v", err) + } + + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if reloaded.LastStats != nil { + t.Fatalf("LastStats = %s, want NULL after a check that carried none", reloaded.LastStats) + } + + // The absent column must also be absent from the admin API's JSON, so a + // client can tell "no sample" from "a sample of zeroes". + encoded, err := json.Marshal(reloaded) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if err := json.Unmarshal(encoded, &body); err != nil { + t.Fatal(err) + } + if _, ok := body["last_stats"]; ok { + t.Fatalf("last_stats emitted for a node with none: %s", encoded) + } +} + +// last_stats carries the scratch fill transcode admission reads, so one worker's +// disk reading must never land on a row an administrator has repointed at +// another during the health request. +func TestRepositoryUpdateHealthRefusesAfterAURLEdit(t *testing.T) { + ctx := context.Background() + repo := NewRepository(newNodeTestPool(t)) + node, err := repo.Create(ctx, CreateNodeInput{ + Name: fmt.Sprintf("health-moved-%d", time.Now().UnixNano()), + Type: NodeTypeTranscode, + URL: fmt.Sprintf("http://health-moved-%d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatalf("create node: %v", err) + } + t.Cleanup(func() { _ = repo.Delete(ctx, node.ID) }) + + moved := node.URL + "-elsewhere" + if _, err := repo.Update(ctx, node.ID, UpdateNodeInput{URL: &moved}); err != nil { + t.Fatalf("repoint node: %v", err) + } + + err = repo.UpdateHealth(ctx, node.ID, node.URL, true, 3, 17, []byte(`{"system":{"cpu_pct":41}}`)) + if !errors.Is(err, ErrNodeMoved) { + t.Fatalf("err = %v, want ErrNodeMoved after the row was repointed", err) + } + reloaded, err := repo.GetByID(ctx, node.ID) + if err != nil { + t.Fatalf("reload node: %v", err) + } + if len(reloaded.LastStats) != 0 { + t.Fatalf("last_stats = %s, want the stale sample discarded", reloaded.LastStats) + } +} diff --git a/internal/nodepool/repository_scan_test.go b/internal/nodepool/repository_scan_test.go new file mode 100644 index 000000000..40e34e682 --- /dev/null +++ b/internal/nodepool/repository_scan_test.go @@ -0,0 +1,58 @@ +package nodepool + +import ( + "errors" + "fmt" + "slices" + "strings" + "testing" +) + +// fakeRow exercises scanNode without a database. pgx.Row is a single Scan +// method, so a stored capability payload can be handed to the real row scanner +// by writing into the destination pointers pgx would have filled. +type fakeRow struct { + capabilities []byte +} + +func (r fakeRow) Scan(dest ...any) error { + if want := strings.Count(nodeColumns, ",") + 1; len(dest) != want { + return fmt.Errorf("scanNode passed %d destinations for %d columns in nodeColumns", len(dest), want) + } + // capabilities is the first of the two jsonb columns scanned as raw bytes; + // last_stats is the second. Everything else keeps its zero value. + for _, d := range dest { + if raw, ok := d.(*[]byte); ok { + *raw = r.capabilities + return nil + } + } + return errors.New("scanNode has no raw-bytes destination for capabilities") +} + +// The admin node list is served straight from stored rows, so scanNode is the +// only producer of the physical_gpu_keys an operator sees. Losing the +// derivation there empties every Shared GPU badge silently, because the pools +// derive their own keys and keep routing correctly. +func TestScanNodeDerivesPhysicalGPUKeys(t *testing.T) { + n, err := scanNode(fakeRow{capabilities: []byte(gpuAAACapabilities)}) + if err != nil { + t.Fatalf("scanNode: %v", err) + } + if got := n.PhysicalGPUKeys; !slices.Equal(got, []string{"GPU-aaa"}) { + t.Fatalf("scanned row derived %v, want [GPU-aaa]", got) + } + if string(n.Capabilities) != gpuAAACapabilities { + t.Fatalf("stored payload was not carried through: %s", n.Capabilities) + } + + // A node that never reported capabilities carries no identities rather + // than empty ones a client would have to special-case. + bare, err := scanNode(fakeRow{}) + if err != nil { + t.Fatalf("scanNode: %v", err) + } + if bare.Capabilities != nil || bare.PhysicalGPUKeys != nil { + t.Fatalf("row without a stored report derived %v", bare.PhysicalGPUKeys) + } +} diff --git a/internal/nodepool/scratchpressure.go b/internal/nodepool/scratchpressure.go new file mode 100644 index 000000000..cf5316f9e --- /dev/null +++ b/internal/nodepool/scratchpressure.go @@ -0,0 +1,79 @@ +package nodepool + +import "encoding/json" + +// scratchStatsView is the minimal projection this package parses out of an +// otherwise opaque last_stats payload, in the same spirit as +// capabilityDriftView and gpuIdentityView: nodepool must not depend on +// nodemetrics, and admission only needs the scratch volume's fill. +// +// The scratch entry is found by its own flag rather than by matching a path, +// because the API does not know a node's transcode directory — the node does, +// and it says which entry is the one. +type scratchStatsView struct { + System struct { + Disks []struct { + Scratch bool `json:"scratch"` + UsedGB float64 `json:"used_gb"` + TotalGB float64 `json:"total_gb"` + Stale bool `json:"stale"` + Unavailable bool `json:"unavailable"` + } `json:"disks"` + } `json:"system"` +} + +// scratchPressureFillPercent is the scratch-volume fill at which a transcode +// node stops being offered new sessions. +// +// A transcode writes HLS segments to that volume for the life of the session, so +// a node that is nearly full does not fail fast: it accepts the session, streams +// for a while, and then dies mid-playback with a write error — the worst failure +// shape available, because the client has already committed to it. Five percent +// headroom is a few minutes of segments at any realistic bitrate, which is what +// makes it enough to notice and act on rather than a hard stop. +// +// It is deliberately high. A scratch volume sitting at 80% is a normal steady +// state for a node with a large segment retention, and excluding those would +// shrink a healthy cluster for no reason. +const scratchPressureFillPercent = 95 + +// scratchDiskFillPercent reports the fill percentage of a node's transcode +// scratch volume from its last health sample. +// +// ok is false whenever the answer would be a guess rather than a measurement: +// no sample, an unparseable one, no scratch entry (a node predating the flag, or +// a proxy with no scratch dir), a path the node could not measure, a capacity of +// zero, or numbers the node itself marked stale. Every one of those means the +// admission guard must not fire — excluding a node on a fill we cannot read +// would take capacity away on no evidence, and a full disk that is still being +// written to shows up as a failing transcode, which is recoverable, while an +// empty pool is not. +func scratchDiskFillPercent(n *Node) (pct int, ok bool) { + if n == nil || len(n.LastStats) == 0 { + return 0, false + } + var view scratchStatsView + if err := json.Unmarshal(n.LastStats, &view); err != nil { + return 0, false + } + for _, disk := range view.System.Disks { + if !disk.Scratch { + continue + } + if disk.Unavailable || disk.Stale || disk.TotalGB <= 0 || disk.UsedGB < 0 { + return 0, false + } + // Floored, so the threshold reads as "95% or more of the volume is + // used" rather than rounding a 94.6% volume into exclusion. + return int(disk.UsedGB / disk.TotalGB * 100), true + } + return 0, false +} + +// scratchPressured reports whether a node's scratch volume is too full to admit +// new work. A node whose fill cannot be read is never pressured; see +// scratchDiskFillPercent. +func scratchPressured(n *Node) bool { + pct, ok := scratchDiskFillPercent(n) + return ok && pct >= scratchPressureFillPercent +} diff --git a/internal/nodepool/scratchpressure_test.go b/internal/nodepool/scratchpressure_test.go new file mode 100644 index 000000000..50a80fd21 --- /dev/null +++ b/internal/nodepool/scratchpressure_test.go @@ -0,0 +1,81 @@ +package nodepool + +import ( + "encoding/json" + "fmt" + "testing" +) + +// scratchStats builds the last_stats blob a node's health response produces for +// a scratch volume at the given fill, in the shape nodemetrics publishes. +func scratchStats(usedGB, totalGB float64, flags ...string) json.RawMessage { + extra := "" + for _, flag := range flags { + extra += fmt.Sprintf(`,"%s":true`, flag) + } + return json.RawMessage(fmt.Sprintf( + `{"system":{"disks":[{"path":"/transcode","used_gb":%g,"total_gb":%g,"scratch":true%s},`+ + `{"path":"/media","used_gb":10,"total_gb":100}]}}`, usedGB, totalGB, extra)) +} + +func TestScratchDiskFillPercent(t *testing.T) { + tests := []struct { + name string + stats json.RawMessage + wantPct int + wantOK bool + }{ + {name: "measured", stats: scratchStats(50, 100), wantPct: 50, wantOK: true}, + {name: "at threshold", stats: scratchStats(95, 100), wantPct: 95, wantOK: true}, + // Floored rather than rounded, so "95%" means at least 95% used. + {name: "just under threshold", stats: scratchStats(94.99, 100), wantPct: 94, wantOK: true}, + {name: "full", stats: scratchStats(100, 100), wantPct: 100, wantOK: true}, + {name: "no stats at all", stats: nil}, + {name: "unparseable", stats: json.RawMessage(`not json`)}, + {name: "stale numbers", stats: scratchStats(99, 100, "stale")}, + {name: "unmeasurable path", stats: scratchStats(0, 0, "unavailable")}, + {name: "zero capacity", stats: scratchStats(0, 0)}, + { + name: "no scratch entry", + stats: json.RawMessage(`{"system":{"disks":[{"path":"/media","used_gb":99,"total_gb":100}]}}`), + }, + {name: "no system section", stats: json.RawMessage(`{"gpu":[]}`)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pct, ok := scratchDiskFillPercent(&Node{LastStats: test.stats}) + if ok != test.wantOK { + t.Fatalf("ok = %v, want %v (pct = %d)", ok, test.wantOK, pct) + } + if ok && pct != test.wantPct { + t.Fatalf("pct = %d, want %d", pct, test.wantPct) + } + }) + } + if _, ok := scratchDiskFillPercent(nil); ok { + t.Fatal("a nil node reported a readable scratch fill") + } +} + +// The threshold is the whole contract of the guard, so the boundary is asserted +// on the predicate the planner actually calls. +func TestScratchPressuredThresholdBoundary(t *testing.T) { + for _, test := range []struct { + usedGB float64 + want bool + }{ + {usedGB: 94, want: false}, + {usedGB: 94.99, want: false}, + {usedGB: 95, want: true}, + {usedGB: 99.5, want: true}, + } { + if got := scratchPressured(&Node{LastStats: scratchStats(test.usedGB, 100)}); got != test.want { + t.Fatalf("scratchPressured(%g%% used) = %v, want %v", test.usedGB, got, test.want) + } + } + // Missing evidence is never pressure: excluding a node on a fill we cannot + // read would take capacity away for nothing. + if scratchPressured(&Node{}) { + t.Fatal("a node with no resource sample was treated as under scratch pressure") + } +} diff --git a/internal/nodepool/transcode_pool.go b/internal/nodepool/transcode_pool.go index 2367fa334..6fde483b8 100644 --- a/internal/nodepool/transcode_pool.go +++ b/internal/nodepool/transcode_pool.go @@ -1,6 +1,7 @@ package nodepool import ( + "encoding/json" "strings" "sync" "time" @@ -21,12 +22,16 @@ func NewTranscodePool() *TranscodePool { // SetNodes replaces the node list. Node URLs are normalized (trailing slashes // trimmed) at the storage boundary so every consumer compares them // consistently, including TranscodeNodeHealthy and remote-start adoption. +// Physical GPU identities are derived here too, so the planner's shared-GPU +// accounting works from the stored capability report immediately after a +// restart, before any node has advertised a changed hash. func (p *TranscodePool) SetNodes(nodes []*Node) { p.mu.Lock() defer p.mu.Unlock() for _, n := range nodes { if n != nil { n.URL = normalizeNodeURL(n.URL) + applyPhysicalGPUKeys(n) } } p.nodes = nodes @@ -81,16 +86,59 @@ func (p *TranscodePool) Nodes() []*Node { // ApplyHealth records a health check result by swapping the node for an // updated copy, keeping published *Node values immutable. -func (p *TranscodePool) ApplyHealth(id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +func (p *TranscodePool) ApplyHealth(id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { p.mu.Lock() defer p.mu.Unlock() - applyNodeHealth(p.nodes, id, healthy, activeJobs, egressKbps, checkedAt) + applyNodeHealth(p.nodes, id, checkedURL, healthy, activeJobs, egressKbps, advertisedHash, lastStats, checkedAt) +} + +// ApplyCapabilities records a freshly fetched capability report by swapping the +// node for an updated copy, keeping published *Node values immutable. +func (p *TranscodePool) ApplyCapabilities(id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { + p.mu.Lock() + defer p.mu.Unlock() + applyNodeCapabilities(p.nodes, id, fetchedFrom, capabilities, hash, refreshedAt, drift, driftBaseline) +} + +// NormalizeNodeURL is how a node's address is written wherever it is used as an +// identity: a map key, a comparison, or the base of a request. +// +// A stored URL may carry a trailing slash and mean the same worker, so anything +// that keys by the raw value ends up with two entries for one node — and the +// one an invalidation deletes is not the one a lookup finds. +func NormalizeNodeURL(nodeURL string) string { + return normalizeNodeURL(nodeURL) +} + +// NodeEndpoint joins a stored node URL with one of the node's own routes. +// +// A stored URL may carry a trailing slash — an operator pasting a base URL is +// the usual way, and everything here already treats the two forms as the same +// worker. Concatenating a route onto it produces "//admin/…", which the node's +// router does not have: the request 404s, and the operator's action fails +// against a node that is running and reachable. The normalization that makes +// the two forms equal for comparison has to make them equal for addressing too. +func NodeEndpoint(nodeURL, path string) string { + return normalizeNodeURL(nodeURL) + path +} + +// sameNodeURL compares two node addresses the way the pools store them, so a +// trailing slash on one side is not a different worker. +func sameNodeURL(a, b string) bool { + return normalizeNodeURL(a) == normalizeNodeURL(b) } // applyNodeHealth replaces the slice entry for id with an updated copy. -func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps int, checkedAt time.Time) { +// +// checkedURL fences the write the same way the database update does, and for +// the same reason: a health request is bounded at five seconds, which is ample +// time for an administrator to repoint the row and reload the pools. Publishing +// by id alone would then write one worker's health — and the scratch fill +// transcode admission reads — onto the replacement, and the database fence +// downstream cannot undo that. The pool would stay wrong until a later sweep. +func applyNodeHealth(nodes []*Node, id int, checkedURL string, healthy bool, activeJobs, egressKbps int, advertisedHash string, lastStats []byte, checkedAt time.Time) { for i, n := range nodes { - if n.ID != id { + if n.ID != id || !sameNodeURL(n.URL, checkedURL) { continue } clone := *n @@ -98,6 +146,49 @@ func applyNodeHealth(nodes []*Node, id int, healthy bool, activeJobs, egressKbps clone.ActiveJobs = activeJobs clone.EgressKbps = egressKbps clone.LastHealthCheck = &checkedAt + // Always a pointer once a check has happened, empty string included: the + // distinction between "this node reports no hash" and "no one has asked" + // is the whole reason the field is one. + clone.AdvertisedCapabilitiesHash = &advertisedHash + // A check that carried no stats clears them rather than keeping the + // previous sample: an unreachable node's five-minute-old CPU number + // looks live on a dashboard, which is worse than no number at all. The + // payload is cloned because the caller's buffer is a decoded HTTP body. + if len(lastStats) > 0 { + clone.LastStats = append(json.RawMessage(nil), lastStats...) + } else { + clone.LastStats = nil + } + nodes[i] = &clone + return + } +} + +// applyNodeCapabilities replaces the slice entry for id with a copy carrying +// the new capability report. The payload is cloned because the caller's buffer +// (a decoded HTTP response) is not ours to publish. +// +// drift is set verbatim, nil included: the note describes the comparison that +// produced this payload, so a node whose hardware recovered must lose the note +// at the same moment it gains the clean report. +func applyNodeCapabilities(nodes []*Node, id int, fetchedFrom string, capabilities []byte, hash string, refreshedAt time.Time, drift *string, driftBaseline []byte) { + for i, n := range nodes { + if n.ID != id || !sameNodeURL(n.URL, fetchedFrom) { + continue + } + clone := *n + clone.Capabilities = append(json.RawMessage(nil), capabilities...) + clone.CapabilitiesHash = &hash + clone.CapabilitiesRefreshedAt = &refreshedAt + clone.CapabilityDrift = drift + if len(driftBaseline) > 0 { + clone.CapabilityDriftBaseline = append(json.RawMessage(nil), driftBaseline...) + } else { + clone.CapabilityDriftBaseline = nil + } + // The GPU identities belong to the payload being replaced, so they are + // re-derived rather than carried over from the previous report. + applyPhysicalGPUKeys(&clone) nodes[i] = &clone return } diff --git a/internal/opslog/repo.go b/internal/opslog/repo.go index fb70adada..b282cf551 100644 --- a/internal/opslog/repo.go +++ b/internal/opslog/repo.go @@ -27,9 +27,13 @@ type EntryRow struct { } type ListOptions struct { - From *time.Time - To *time.Time + From *time.Time + To *time.Time + // Level filters on a single level. Levels filters on any of several and + // wins when both are set; Level stays for callers that only ever ask for + // one. Level string + Levels []string Component string NodeID string RequestID string @@ -55,6 +59,87 @@ func NewRepo(pool *pgxpool.Pool) *Repo { } func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { + query, args, limit, err := buildListQuery(opts) + if err != nil { + return ListResult{}, err + } + + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return ListResult{}, fmt.Errorf("list operational logs: %w", err) + } + defer rows.Close() + + entries := make([]EntryRow, 0, limit+1) + for rows.Next() { + var entry EntryRow + var attrsJSON []byte + if err := rows.Scan( + &entry.ID, + &entry.Timestamp, + &entry.Level, + &entry.Component, + &entry.Message, + &entry.RequestID, + &entry.UserID, + &entry.SessionID, + &entry.PlaybackSessionID, + &entry.ClientIP, + &entry.NodeID, + &attrsJSON, + ); err != nil { + return ListResult{}, fmt.Errorf("scan operational log row: %w", err) + } + if len(attrsJSON) > 0 { + if err := json.Unmarshal(attrsJSON, &entry.Attrs); err != nil { + return ListResult{}, fmt.Errorf("decode operational log attrs: %w", err) + } + } + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return ListResult{}, fmt.Errorf("iterate operational logs: %w", err) + } + + result := ListResult{} + if len(entries) > limit { + last := entries[limit-1] + result.NextCursor = encodeCursor(last.Timestamp, last.ID) + entries = entries[:limit] + } + result.Entries = entries + return result, nil +} + +// NormalizeLevels lowercases, trims and de-duplicates a level filter, dropping +// empty entries. It returns nil when nothing usable is left so callers can test +// the result for "no level filter". +func NormalizeLevels(levels []string) []string { + if len(levels) == 0 { + return nil + } + normalized := make([]string, 0, len(levels)) + seen := make(map[string]struct{}, len(levels)) + for _, level := range levels { + level = strings.ToLower(strings.TrimSpace(level)) + if level == "" { + continue + } + if _, ok := seen[level]; ok { + continue + } + seen[level] = struct{}{} + normalized = append(normalized, level) + } + if len(normalized) == 0 { + return nil + } + return normalized +} + +// buildListQuery renders the filtered query and its arguments. It is separate +// from List so the predicate assembly can be tested without a database. +func buildListQuery(opts ListOptions) (string, []any, int, error) { limit := opts.Limit if limit <= 0 || limit > 200 { limit = 100 @@ -74,7 +159,11 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { args = append(args, *opts.To) argIdx++ } - if opts.Level != "" { + if levels := NormalizeLevels(opts.Levels); len(levels) > 0 { + conditions = append(conditions, fmt.Sprintf("level = ANY($%d)", argIdx)) + args = append(args, levels) + argIdx++ + } else if opts.Level != "" { conditions = append(conditions, fmt.Sprintf("level = $%d", argIdx)) args = append(args, strings.ToLower(opts.Level)) argIdx++ @@ -117,7 +206,7 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { if opts.Cursor != "" { cursorTs, cursorID, err := decodeCursor(opts.Cursor) if err != nil { - return ListResult{}, err + return "", nil, 0, err } conditions = append(conditions, fmt.Sprintf("(timestamp, id) < ($%d, $%d)", argIdx, argIdx+1)) args = append(args, cursorTs, cursorID) @@ -134,51 +223,7 @@ func (r *Repo) List(ctx context.Context, opts ListOptions) (ListResult, error) { `, strings.Join(conditions, " AND "), argIdx) args = append(args, limit+1) - rows, err := r.pool.Query(ctx, query, args...) - if err != nil { - return ListResult{}, fmt.Errorf("list operational logs: %w", err) - } - defer rows.Close() - - entries := make([]EntryRow, 0, limit+1) - for rows.Next() { - var entry EntryRow - var attrsJSON []byte - if err := rows.Scan( - &entry.ID, - &entry.Timestamp, - &entry.Level, - &entry.Component, - &entry.Message, - &entry.RequestID, - &entry.UserID, - &entry.SessionID, - &entry.PlaybackSessionID, - &entry.ClientIP, - &entry.NodeID, - &attrsJSON, - ); err != nil { - return ListResult{}, fmt.Errorf("scan operational log row: %w", err) - } - if len(attrsJSON) > 0 { - if err := json.Unmarshal(attrsJSON, &entry.Attrs); err != nil { - return ListResult{}, fmt.Errorf("decode operational log attrs: %w", err) - } - } - entries = append(entries, entry) - } - if err := rows.Err(); err != nil { - return ListResult{}, fmt.Errorf("iterate operational logs: %w", err) - } - - result := ListResult{} - if len(entries) > limit { - last := entries[limit-1] - result.NextCursor = encodeCursor(last.Timestamp, last.ID) - entries = entries[:limit] - } - result.Entries = entries - return result, nil + return query, args, limit, nil } func encodeCursor(ts time.Time, id int64) string { diff --git a/internal/opslog/repo_test.go b/internal/opslog/repo_test.go new file mode 100644 index 000000000..d1a861b7a --- /dev/null +++ b/internal/opslog/repo_test.go @@ -0,0 +1,138 @@ +package opslog + +import ( + "reflect" + "strconv" + "strings" + "testing" +) + +func TestNormalizeLevels(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input []string + want []string + }{ + {name: "nil stays nil", input: nil, want: nil}, + {name: "blank entries are dropped", input: []string{"", " "}, want: nil}, + {name: "trimmed and lowercased", input: []string{" Error ", "WARN"}, want: []string{"error", "warn"}}, + {name: "duplicates collapse", input: []string{"error", "Error", "error"}, want: []string{"error"}}, + {name: "order is preserved", input: []string{"warn", "error", "info"}, want: []string{"warn", "error", "info"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := NormalizeLevels(tt.input); !reflect.DeepEqual(got, tt.want) { + t.Fatalf("NormalizeLevels(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestBuildListQueryLevelFilters(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + opts ListOptions + wantPredicate string + wantArg any + }{ + { + name: "no level filter", + opts: ListOptions{}, + wantPredicate: "", + }, + { + name: "single level keeps the equality predicate", + opts: ListOptions{Level: "ERROR"}, + wantPredicate: "level = $1", + wantArg: "error", + }, + { + name: "several levels use ANY", + opts: ListOptions{Levels: []string{"error", "warn"}}, + wantPredicate: "level = ANY($1)", + wantArg: []string{"error", "warn"}, + }, + { + name: "levels win over level", + opts: ListOptions{Level: "info", Levels: []string{"error"}}, + wantPredicate: "level = ANY($1)", + wantArg: []string{"error"}, + }, + { + name: "an all-blank levels list falls back to level", + opts: ListOptions{Level: "info", Levels: []string{"", " "}}, + wantPredicate: "level = $1", + wantArg: "info", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + query, args, _, err := buildListQuery(tt.opts) + if err != nil { + t.Fatalf("buildListQuery: %v", err) + } + if tt.wantPredicate == "" { + if strings.Contains(query, "level =") { + t.Fatalf("query filters on level without a filter being set:\n%s", query) + } + return + } + if !strings.Contains(query, tt.wantPredicate) { + t.Fatalf("query missing %q:\n%s", tt.wantPredicate, query) + } + if len(args) == 0 { + t.Fatal("no arguments bound") + } + if !reflect.DeepEqual(args[0], tt.wantArg) { + t.Fatalf("arg[0] = %#v, want %#v", args[0], tt.wantArg) + } + }) + } +} + +// The placeholder numbering has to stay in step with the argument slice +// whichever level filter is used, or a multi-filter query binds the wrong +// values. +func TestBuildListQueryPlaceholdersMatchArguments(t *testing.T) { + t.Parallel() + + userID := 42 + opts := ListOptions{ + Levels: []string{"error", "warn"}, + Component: "playback", + UserID: &userID, + Query: "timeout", + Limit: 25, + } + + query, args, limit, err := buildListQuery(opts) + if err != nil { + t.Fatalf("buildListQuery: %v", err) + } + if limit != 25 { + t.Fatalf("limit = %d, want 25", limit) + } + for i := range args { + placeholder := "$" + strconv.Itoa(i+1) + if !strings.Contains(query, placeholder) { + t.Fatalf("query does not bind %s:\n%s", placeholder, query) + } + } + // levels, component, user_id, message, limit + if len(args) != 5 { + t.Fatalf("args = %#v, want 5 entries", args) + } + if args[len(args)-1] != opts.Limit+1 { + t.Fatalf("limit arg = %v, want %d (limit + 1 for the cursor probe)", args[len(args)-1], opts.Limit+1) + } +} diff --git a/internal/playback/capabilityhash.go b/internal/playback/capabilityhash.go new file mode 100644 index 000000000..4b0a00ea3 --- /dev/null +++ b/internal/playback/capabilityhash.go @@ -0,0 +1,193 @@ +package playback + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "slices" + "strings" + + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// ComputeCapabilityHash summarizes what a host has and can do into one +// comparable token, so a reader (the node health sweep, an operator) detects +// change without diffing a whole report or re-probing. +// +// Only hardware identity and capability are hashed. Fields that vary with who +// asked rather than with the host — Source, NodeURL, and the hash itself — are +// excluded, because a report of unchanged hardware must keep its hash no matter +// who asked for it. IntelDetected is excluded as well: it is derived from the +// render devices already covered. +// +// ProbeRequestTimeoutMillis is included, though it describes the report rather +// than the hardware. It is the node's own statement of how long its answer may +// take, and the control plane sizes real deadlines from the stored copy. A node +// upgraded to a build that needs longer changes nothing else about itself, so +// leaving it out of the change signal meant the sweep never refetched and the +// API kept canceling that node's re-probes against a budget it had outgrown. +// It is stable for a given build and configuration, so including it costs one +// refetch at upgrade and nothing after. +// +// Every slice is ordered here rather than trusted from the input: probe and +// filesystem enumeration order is incidental, so two reports of the same host +// hash identically regardless of it. +func ComputeCapabilityHash(info HWAccelInfo) string { + payload, err := json.Marshal(canonicalCapabilities(info)) + if err != nil { + // Unreachable: every field below is a string, bool, or slice of them. + return "" + } + sum := sha256.Sum256(payload) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// canonicalCapability is the hashed projection of an HWAccelInfo. It is a +// distinct type rather than a reordered copy of the input so that adding a +// field to HWAccelInfo never silently changes every node's hash. +type canonicalCapability struct { + Resolved string `json:"resolved"` + BootID string `json:"boot_id"` + RenderDevices []string `json:"render_devices"` + RenderDeviceDetails []canonicalRenderDevice `json:"render_device_details"` + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids"` + ProbeRequestTimeout int64 `json:"probe_request_timeout_ms"` + DetectedBackends []canonicalDetectedBackend `json:"detected_backends"` + Transformations []canonicalTransformation `json:"transformations"` + ToneMapCapabilities []canonicalToneMap `json:"tone_map_capabilities"` +} + +// canonicalRenderDevice serializes a device strongest identity first: the GPU +// uuid identifies the card anywhere, the PCI address identifies a slot, and the +// path only identifies an enumeration position. +type canonicalRenderDevice struct { + GPUUUID string `json:"gpu_uuid"` + PCIAddress string `json:"pci_address"` + Path string `json:"path"` + Description string `json:"description"` +} + +type canonicalDetectedBackend struct { + Backend string `json:"backend"` + Verified bool `json:"verified"` + Devices []string `json:"devices"` + Device string `json:"device"` + Reason string `json:"reason"` + Skipped bool `json:"skipped"` +} + +type canonicalTransformation struct { + Name string `json:"name"` + Executor string `json:"executor"` + RecipeVersion string `json:"recipe_version"` + ValidatedClaims []string `json:"validated_claims"` +} + +type canonicalToneMap struct { + Mode string `json:"mode"` + Backend string `json:"backend"` + Filter string `json:"filter"` + SourceKinds []string `json:"source_kinds"` +} + +func canonicalCapabilities(info HWAccelInfo) canonicalCapability { + return canonicalCapability{ + Resolved: info.Resolved, + BootID: info.BootID, + RenderDevices: sortedStrings(info.RenderDevices), + RenderDeviceDetails: canonicalRenderDevices(info.RenderDeviceDetails), + NVIDIAGPUUUIDs: sortedStrings(info.NVIDIAGPUUUIDs), + ProbeRequestTimeout: info.ProbeRequestTimeoutMillis, + DetectedBackends: canonicalDetectedBackends(info.DetectedBackends), + Transformations: canonicalTransformations(info.Transformations), + ToneMapCapabilities: canonicalToneMaps(info.ToneMapCapabilities), + } +} + +func canonicalRenderDevices(details []RenderDeviceInfo) []canonicalRenderDevice { + out := make([]canonicalRenderDevice, 0, len(details)) + for _, detail := range details { + out = append(out, canonicalRenderDevice{ + GPUUUID: detail.GPUUUID, + PCIAddress: detail.PCIAddress, + Path: detail.Path, + Description: detail.Description, + }) + } + slices.SortFunc(out, func(a, b canonicalRenderDevice) int { + return strings.Compare(a.Path, b.Path) + }) + return out +} + +func canonicalDetectedBackends(backends []DetectedBackend) []canonicalDetectedBackend { + out := make([]canonicalDetectedBackend, 0, len(backends)) + for _, backend := range backends { + out = append(out, canonicalDetectedBackend{ + Backend: backend.Backend, + Verified: backend.Verified, + Devices: sortedStrings(backend.Devices), + Device: backend.Device, + Reason: backend.Reason, + Skipped: backend.Skipped, + }) + } + slices.SortFunc(out, func(a, b canonicalDetectedBackend) int { + return strings.Compare(a.Backend, b.Backend) + }) + return out +} + +func canonicalTransformations(transformations []TransformationV3) []canonicalTransformation { + out := make([]canonicalTransformation, 0, len(transformations)) + for _, transformation := range transformations { + out = append(out, canonicalTransformation{ + Name: transformation.Name, + Executor: transformation.Executor, + RecipeVersion: transformation.RecipeVersion, + ValidatedClaims: sortedStrings(transformation.ValidatedClaims), + }) + } + slices.SortFunc(out, func(a, b canonicalTransformation) int { + if byName := strings.Compare(a.Name, b.Name); byName != 0 { + return byName + } + return strings.Compare(a.RecipeVersion, b.RecipeVersion) + }) + return out +} + +func canonicalToneMaps(capabilities tonemap.Capabilities) []canonicalToneMap { + out := make([]canonicalToneMap, 0, len(capabilities)) + for _, capability := range capabilities { + kinds := make([]string, 0, len(capability.SourceKinds)) + for _, kind := range capability.SourceKinds { + kinds = append(kinds, string(kind)) + } + out = append(out, canonicalToneMap{ + Mode: string(capability.Mode), + Backend: capability.Backend, + Filter: capability.Filter, + SourceKinds: sortedStrings(kinds), + }) + } + slices.SortFunc(out, func(a, b canonicalToneMap) int { + if byMode := strings.Compare(a.Mode, b.Mode); byMode != 0 { + return byMode + } + if byBackend := strings.Compare(a.Backend, b.Backend); byBackend != 0 { + return byBackend + } + return strings.Compare(a.Filter, b.Filter) + }) + return out +} + +// sortedStrings returns an ordered copy, leaving the caller's slice alone: the +// input is a live report another goroutine may still be serving. +func sortedStrings(values []string) []string { + out := make([]string, 0, len(values)) + out = append(out, values...) + slices.Sort(out) + return out +} diff --git a/internal/playback/capabilityhash_test.go b/internal/playback/capabilityhash_test.go new file mode 100644 index 000000000..5a9c1a7fa --- /dev/null +++ b/internal/playback/capabilityhash_test.go @@ -0,0 +1,167 @@ +package playback + +import ( + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +func sampleCapabilityInfo() HWAccelInfo { + return HWAccelInfo{ + Resolved: "nvenc", + BootID: "5b2c1f0e-1111-4a2b-9c3d-2f6e7a8b9c0d", + RenderDevices: []string{"/dev/dri/renderD128", "/dev/dri/renderD129"}, + RenderDeviceDetails: []RenderDeviceInfo{ + {Path: "/dev/dri/renderD128", PCIAddress: "0000:03:00.0", GPUUUID: "GPU-aaa", Description: "NVIDIA GPU (0x2204)"}, + {Path: "/dev/dri/renderD129", PCIAddress: "0000:04:00.0", Description: "Intel GPU (0x9a49)"}, + }, + DetectedBackends: []DetectedBackend{ + {Backend: "nvenc", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "vaapi", Verified: false, Devices: []string{"/dev/dri/renderD129", "/dev/dri/renderD128"}, Reason: "h264_vaapi encoder unavailable"}, + }, + Transformations: []TransformationV3{ + {Name: "tone_map", Executor: "node", RecipeVersion: "v2", ValidatedClaims: []string{"hdr10", "hlg"}}, + {Name: "audio_to_aac", Executor: "node", RecipeVersion: "v1"}, + }, + ToneMapCapabilities: tonemap.Capabilities{ + {Mode: tonemap.ModeHardware, Backend: "nvenc", Filter: "tonemap_cuda", SourceKinds: []tonemap.SourceKind{"hdr10", "hlg"}}, + {Mode: tonemap.ModeSoftware, Backend: "software", Filter: "tonemap", SourceKinds: []tonemap.SourceKind{"hdr10"}}, + }, + } +} + +// A node reports the same hardware twice with its slices enumerated in a +// different order — probe order and directory listing order are incidental. If +// that moved the hash, every sweep would look like a hardware change and refetch +// the whole inventory forever. +func TestComputeCapabilityHashIgnoresSliceOrder(t *testing.T) { + info := sampleCapabilityInfo() + shuffled := sampleCapabilityInfo() + shuffled.RenderDevices = []string{"/dev/dri/renderD129", "/dev/dri/renderD128"} + shuffled.RenderDeviceDetails = []RenderDeviceInfo{shuffled.RenderDeviceDetails[1], shuffled.RenderDeviceDetails[0]} + shuffled.DetectedBackends = []DetectedBackend{shuffled.DetectedBackends[1], shuffled.DetectedBackends[0]} + shuffled.DetectedBackends[0].Devices = []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} + shuffled.Transformations = []TransformationV3{shuffled.Transformations[1], shuffled.Transformations[0]} + shuffled.Transformations[1].ValidatedClaims = []string{"hlg", "hdr10"} + shuffled.ToneMapCapabilities = tonemap.Capabilities{shuffled.ToneMapCapabilities[1], shuffled.ToneMapCapabilities[0]} + shuffled.ToneMapCapabilities[1].SourceKinds = []tonemap.SourceKind{"hlg", "hdr10"} + + if got, want := ComputeCapabilityHash(shuffled), ComputeCapabilityHash(info); got != want { + t.Fatalf("reordered report hashed differently:\n got %s\nwant %s", got, want) + } +} + +// Fields that vary with who asked must not move the hash: otherwise the same +// node hashes differently depending on the caller. +func TestComputeCapabilityHashIgnoresPerCallerMetadata(t *testing.T) { + info := sampleCapabilityInfo() + want := ComputeCapabilityHash(info) + + info.Source = "remote" + info.NodeURL = "http://node-7:8080" + info.CapabilityHash = "sha256:stale" + + if got := ComputeCapabilityHash(info); got != want { + t.Fatalf("report metadata changed the hash:\n got %s\nwant %s", got, want) + } +} + +// The advertised probe budget does move it, though it describes the report +// rather than the hardware. The control plane sizes real deadlines from the +// stored copy, so a node upgraded to a build that needs longer — changing +// nothing else about itself — has to reach the sweep, or the API keeps +// canceling that node's re-probes against a budget it has outgrown. +func TestComputeCapabilityHashTracksTheAdvertisedProbeBudget(t *testing.T) { + info := sampleCapabilityInfo() + info.ProbeRequestTimeoutMillis = 111_000 + before := ComputeCapabilityHash(info) + + info.ProbeRequestTimeoutMillis = 136_000 + if got := ComputeCapabilityHash(info); got == before { + t.Fatal("a node that raised its advertised probe budget hashed identically; the sweep would never refetch it") + } +} + +func TestComputeCapabilityHashIsPrefixedSHA256(t *testing.T) { + hash := ComputeCapabilityHash(sampleCapabilityInfo()) + if !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("hash = %q, want a sha256: prefix", hash) + } + if len(hash) != len("sha256:")+64 { + t.Fatalf("hash = %q, want 64 hex digits after the prefix", hash) + } +} + +// Every hardware or capability change must move the hash, since the hash is the +// only thing the health sweep looks at before deciding nothing changed. +func TestComputeCapabilityHashDetectsRealChanges(t *testing.T) { + base := ComputeCapabilityHash(sampleCapabilityInfo()) + tests := []struct { + name string + mutate func(*HWAccelInfo) + }{ + {"resolved backend", func(i *HWAccelInfo) { i.Resolved = "vaapi" }}, + {"boot id", func(i *HWAccelInfo) { i.BootID = "0000ffff-2222-4a2b-9c3d-2f6e7a8b9c0d" }}, + {"render device removed", func(i *HWAccelInfo) { + i.RenderDevices = i.RenderDevices[:1] + i.RenderDeviceDetails = i.RenderDeviceDetails[:1] + }}, + {"pci address", func(i *HWAccelInfo) { i.RenderDeviceDetails[0].PCIAddress = "0000:07:00.0" }}, + {"gpu uuid", func(i *HWAccelInfo) { i.RenderDeviceDetails[0].GPUUUID = "GPU-bbb" }}, + {"device description", func(i *HWAccelInfo) { i.RenderDeviceDetails[1].Description = "AMD GPU" }}, + {"backend verification lost", func(i *HWAccelInfo) { i.DetectedBackends[0].Verified = false }}, + {"backend failure reason", func(i *HWAccelInfo) { i.DetectedBackends[1].Reason = "no driver" }}, + {"verified device", func(i *HWAccelInfo) { i.DetectedBackends[0].Device = "/dev/dri/renderD129" }}, + {"transformation recipe version", func(i *HWAccelInfo) { i.Transformations[0].RecipeVersion = "v3" }}, + {"validated claims", func(i *HWAccelInfo) { i.Transformations[0].ValidatedClaims = []string{"hdr10"} }}, + {"tone map filter", func(i *HWAccelInfo) { i.ToneMapCapabilities[0].Filter = "tonemap_opencl" }}, + {"tone map source kinds", func(i *HWAccelInfo) { + i.ToneMapCapabilities[0].SourceKinds = []tonemap.SourceKind{"hdr10"} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := sampleCapabilityInfo() + tt.mutate(&info) + if got := ComputeCapabilityHash(info); got == base { + t.Fatalf("%s did not change the hash (%s)", tt.name, got) + } + }) + } +} + +// An empty report still hashes, so a node with no hardware advertises a stable +// identity rather than an empty one the sweep would read as "cannot say". +func TestComputeCapabilityHashOfEmptyReport(t *testing.T) { + hash := ComputeCapabilityHash(HWAccelInfo{}) + if hash == "" { + t.Fatal("empty report produced no hash") + } + if hash == ComputeCapabilityHash(sampleCapabilityInfo()) { + t.Fatal("empty report hashed the same as a populated one") + } +} + +// The ceiling on a node-advertised budget has to sit above what a real +// configuration asks for. Picked as a round five minutes it was already below a +// nine-device node's legitimate 311 seconds, so the API canceled that node's +// re-probe before its own deadline every time and its inventory never landed. +func TestNormalizeProbeRequestTimeoutAdmitsALargeButRealBudget(t *testing.T) { + nineDevices := CapabilityRequestTimeout(tonemap.BackendQSV, + "/dev/dri/renderD128,/dev/dri/renderD129,/dev/dri/renderD130,/dev/dri/renderD131,"+ + "/dev/dri/renderD132,/dev/dri/renderD133,/dev/dri/renderD134,/dev/dri/renderD135,/dev/dri/renderD136") + + got := NormalizeProbeRequestTimeout(nineDevices.Milliseconds(), time.Minute) + if got != nineDevices { + t.Fatalf("normalized = %v, want the %v a nine-device node legitimately asks for", got, nineDevices) + } + + // It is still a ceiling: the value comes off the wire from a worker, and a + // caller holds a connection open for it. + absurd := 24 * time.Hour + if got := NormalizeProbeRequestTimeout(absurd.Milliseconds(), time.Minute); got != MaxCapabilityRequestTimeout() { + t.Fatalf("normalized = %v for an absurd advertisement, want the %v ceiling", got, MaxCapabilityRequestTimeout()) + } +} diff --git a/internal/playback/directplay_test.go b/internal/playback/directplay_test.go index 29797fba4..bc15cd6cc 100644 --- a/internal/playback/directplay_test.go +++ b/internal/playback/directplay_test.go @@ -20,12 +20,6 @@ import ( dto "github.com/prometheus/client_model/go" ) -const ( - directPlayDarwinGOOS = "darwin" - directPlayLinuxGOOS = "linux" - directPlayWindowsGOOS = "windows" -) - func TestServeDirectPlayHTTPContract(t *testing.T) { const content = "0123456789abcdefghijklmnopqrstuvwxyz" filePath := filepath.Join(t.TempDir(), "fixture.mp4") @@ -528,7 +522,7 @@ func TestServeDirectPlayChangedEntityRejectsOldValidators(t *testing.T) { } func TestServeDirectPlayPermissionChangePreservesEntityTag(t *testing.T) { - if runtime.GOOS != directPlayLinuxGOOS { + if runtime.GOOS != linuxGOOS { t.Skip("Linux direct-play validators ignore permission-only ctime changes") } if !platformRequiresDirectPlayValidator() { @@ -640,7 +634,7 @@ func (fileInfoWithoutSystem) Sys() any { func platformRequiresDirectPlayValidator() bool { switch runtime.GOOS { - case directPlayDarwinGOOS, directPlayLinuxGOOS, directPlayWindowsGOOS: + case darwinGOOS, linuxGOOS, windowsGOOS: return true default: return false diff --git a/internal/playback/encoder_warmup.go b/internal/playback/encoder_warmup.go index 6979f2634..47c581795 100644 --- a/internal/playback/encoder_warmup.go +++ b/internal/playback/encoder_warmup.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/Silo-Server/silo-server/internal/tonemap" "golang.org/x/sync/singleflight" ) @@ -52,7 +53,7 @@ func warmHardwareEncoderCached( ctx context.Context, ffmpegPath, configuredHWAccel, configuredHWDevice string, state *hardwareEncoderWarmupState, - resolve func(context.Context, string, string) string, + resolve func(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string, run hardwareEncoderWarmupRunner, ) error { if ctx == nil { @@ -62,12 +63,12 @@ func warmHardwareEncoderCached( return nil } ffmpegPath = ResolveFFmpegPath(ffmpegPath) - backend := resolve(ctx, configuredHWAccel, ffmpegPath) + backend := resolve(ctx, configuredHWAccel, ffmpegPath, configuredHWDevice) if backend == "" || backend == HWAccelNone { return nil } devices := hardwareEncoderWarmupDevices(backend, configuredHWDevice) - cacheKey := nvencProbeCacheKey(ffmpegPath) + "\x00" + backend + "\x00" + strings.Join(devices, ",") + cacheKey := ffmpegIdentityKey(ffmpegPath) + "\x00" + backend + "\x00" + strings.Join(devices, ",") now := time.Now() state.Lock() entry, ok := state.entries[cacheKey] @@ -109,7 +110,7 @@ func warmHardwareEncoderWithRunner(ctx context.Context, timeout time.Duration, f } var result error for _, device := range devices { - args := hardwareEncoderWarmupArgs(backend, device) + args := hardwareSmokeEncodeArgs(backend, device) if len(args) == 0 { continue } @@ -155,23 +156,25 @@ func hardwareEncoderWarmupDevices(backend, configured string) []string { return []string{defaultRenderDevicePath} } -func hardwareEncoderWarmupArgs(backend, device string) []string { +// hardwareSmokeEncodeArgs builds the one-frame synthetic encode shared by +// encoder warmup and the auto-detection capability probes: the same +// initialization chain a real transcode uses, driven by testsrc2 so no media +// file is required. +func hardwareSmokeEncodeArgs(backend, device string) []string { base := []string{ffmpegHideBannerArg, ffmpegLogLevelArg, ffmpegErrorLogLevel} switch backend { case transcodeHWQSV: if strings.TrimSpace(device) == "" { device = defaultRenderDevicePath } - base = append(base, - "-init_hw_device", qsvVAAPIInitDevice(device), - "-init_hw_device", "qsv=qs@va", - "-filter_hw_device", "qs", - ) + base = append(base, tonemap.QSVInitDeviceArgs(device)...) + base = append(base, "-filter_hw_device", "qs") case transcodeHWVAAPI: if strings.TrimSpace(device) == "" { device = defaultRenderDevicePath } - base = append(base, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va") + base = append(base, tonemap.VAAPIInitDeviceArgs(vaapiHWDeviceAlias, device)...) + base = append(base, "-filter_hw_device", vaapiHWDeviceAlias) case transcodeHWNVENC: if strings.TrimSpace(device) != "" { base = append(base, "-init_hw_device", "cuda=cu:"+device, "-filter_hw_device", "cu") @@ -179,7 +182,7 @@ func hardwareEncoderWarmupArgs(backend, device string) []string { default: return nil } - base = append(base, "-f", "lavfi", "-i", "testsrc2=size=640x360:rate=1") + base = append(base, "-f", "lavfi", "-i", smokeEncodeSource) switch backend { case transcodeHWQSV: base = append(base, "-vf", "format=nv12,hwupload=extra_hw_frames=64", "-frames:v", "1", "-an", "-c:v", "h264_qsv") diff --git a/internal/playback/encoder_warmup_test.go b/internal/playback/encoder_warmup_test.go index 66913a27e..d5390ff80 100644 --- a/internal/playback/encoder_warmup_test.go +++ b/internal/playback/encoder_warmup_test.go @@ -10,7 +10,7 @@ import ( "time" ) -func TestHardwareEncoderWarmupArgs(t *testing.T) { +func TestHardwareSmokeEncodeArgs(t *testing.T) { tests := []struct { name string backend string @@ -31,7 +31,7 @@ func TestHardwareEncoderWarmupArgs(t *testing.T) { backend: transcodeHWVAAPI, device: "/dev/dri/renderD130", contains: []string{ - "vaapi=va:/dev/dri/renderD130", "format=nv12,hwupload", "h264_vaapi", + "vaapi=hw:/dev/dri/renderD130", "format=nv12,hwupload", "h264_vaapi", }, }, { @@ -43,18 +43,18 @@ func TestHardwareEncoderWarmupArgs(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - args := hardwareEncoderWarmupArgs(test.backend, test.device) + args := hardwareSmokeEncodeArgs(test.backend, test.device) for _, want := range test.contains { if !slices.Contains(args, want) { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) = %v, missing %q", test.backend, test.device, args, want) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) = %v, missing %q", test.backend, test.device, args, want) } } framesIndex := slices.Index(args, "-frames:v") if framesIndex < 0 || framesIndex+1 >= len(args) || args[framesIndex+1] != "1" { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) = %v, want one output frame", test.backend, test.device, args) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) = %v, want one output frame", test.backend, test.device, args) } if got := args[len(args)-3:]; !slices.Equal(got, []string{"-f", "null", "-"}) { - t.Fatalf("hardwareEncoderWarmupArgs(%q, %q) tail = %v, want null sink", test.backend, test.device, got) + t.Fatalf("hardwareSmokeEncodeArgs(%q, %q) tail = %v, want null sink", test.backend, test.device, got) } }) } @@ -134,7 +134,7 @@ func TestWarmHardwareEncoderCachedSharesSuccessfulWarmupPastCallerCancellation(t return nil, ctx.Err() } } - resolve := func(context.Context, string, string) string { return transcodeHWNVENC } + resolve := func(context.Context, string, string, string) string { return transcodeHWNVENC } callerCtx, cancel := context.WithCancel(context.Background()) result := make(chan error, 1) diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 27e23f2b4..43baf944f 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -2,27 +2,42 @@ package playback import ( "context" + "encoding/json" + "errors" "fmt" "log/slog" "os" "os/exec" "path/filepath" "runtime" + "slices" "sort" + "strconv" "strings" "sync" + "sync/atomic" "time" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/tonemap" "golang.org/x/sync/singleflight" ) -const darwinGOOS = "darwin" +// GOOS names this package and its tests compare runtime.GOOS against. +const ( + darwinGOOS = "darwin" + linuxGOOS = "linux" + windowsGOOS = "windows" +) const ( ffmpegFlagHideBanner = "-hide_banner" ffmpegFlagLogLevel = "-loglevel" ffmpegLogLevelError = "error" + // smokeEncodeSource is the synthetic one-frame input every hardware smoke + // encode reads. Small and deterministic, so a probe costs a frame rather + // than a file the host may not have. + smokeEncodeSource = "testsrc2=size=640x360:rate=1" ) var ( @@ -30,11 +45,21 @@ var ( defaultNVIDIAControlDevice = "/dev/nvidiactl" defaultNVIDIADeviceGlob = "/dev/nvidia[0-9]*" sysClassDRMDir = "/sys/class/drm" + procBootIDPath = "/proc/sys/kernel/random/boot_id" currentGOOS = runtime.GOOS - nvencProbeCommandTimeout = 3 * time.Second - nvencProbeNegativeTTL = 15 * time.Second + hwProbeCommandTimeout = 3 * time.Second + hwProbeNegativeTTL = 15 * time.Second + // hwProbeNow is the cache clock; tests advance it instead of sleeping. + hwProbeNow = time.Now + // hwProbeFlightStarted, when set, is called on the shared probe goroutine + // once it has committed to running a probe. It is the seam a test uses to + // order an invalidation against a flight that is genuinely in progress, + // rather than sleeping and hoping. Production leaves it nil. + hwProbeFlightStarted func() ) +// hardwareProbeResult records a VideoToolbox probe, which reports per-codec +// availability because macOS can offer H.264 without HEVC on older hardware. type hardwareProbeResult struct { available bool reason string @@ -42,22 +67,61 @@ type hardwareProbeResult struct { hevcAvailable bool } -type nvencProbeResult struct { +// hwProbeResult records whether one backend was verified end to end on this +// host. reason is populated only for a failure and is operator-facing. +// +// It covers every backend the Linux walk probes, not just NVENC, which is why +// it is not named for one of them. +type hwProbeResult struct { available bool reason string } -type nvencProbeCacheEntry struct { - result nvencProbeResult +type hwProbeCacheEntry struct { + result hwProbeResult expiresAt time.Time } -var nvencProbeCache = struct { +var hwProbeCache = struct { sync.Mutex - byPath map[string]nvencProbeCacheEntry - group singleflight.Group + entries map[string]hwProbeCacheEntry + group singleflight.Group + // generation counts invalidations. It is part of every cache and + // singleflight key, which is what makes InvalidateHWProbeCache supersede a + // probe already in flight rather than merely clearing the map in front of + // it: the flight stores its result under the generation it started in, and + // a caller arriving afterwards asks a different key and therefore starts a + // fresh probe instead of joining the stale one. + generation uint64 + // verifiedDevices records, per generation and backend, every candidate + // device whose smoke encode passed, in probe order. Execution reads it so a + // backend verified on one render node is not then run on another, and so + // balancing across a configured multi-device list cannot land a workload on + // a card no probe ever passed; see VerifiedHWDevice and VerifiedHWDevices. + verifiedDevices map[string][]string }{ - byPath: make(map[string]nvencProbeCacheEntry), + entries: make(map[string]hwProbeCacheEntry), + verifiedDevices: make(map[string][]string), +} + +// DetectedBackend reports one hardware backend that has candidate devices on +// this host, together with the outcome of its FFmpeg verification probe. +type DetectedBackend struct { + Backend string `json:"backend"` + // Verified reports whether at least one candidate device passed its probe. + Verified bool `json:"verified"` + // Devices lists every candidate considered for this backend, in probe order. + Devices []string `json:"devices,omitempty"` + // Device is the candidate whose probe passed. NVENC addresses its GPU + // through the CUDA runtime, so it stays empty there even when verified. + Device string `json:"device,omitempty"` + // Reason explains a failure, attributed per device when several were tried. + Reason string `json:"reason,omitempty"` + // Skipped reports that no probe was attempted because none of the + // backend's candidate devices is accessible to this process — a proxy + // node reading a cluster-wide hw_device meant for the transcode nodes, + // not a driver failure. Reason still says which devices were skipped. + Skipped bool `json:"skipped,omitempty"` } // videoToolboxProbeRetryDelay bounds how long a negative probe result is @@ -66,6 +130,11 @@ var nvencProbeCache = struct { // the process lifetime; successful fully-capable probes are cached forever. var videoToolboxProbeRetryDelay = time.Minute +// videoToolboxProbeStarted is a test seam: it fires inside a VideoToolbox probe +// flight so a test can order an invalidation against it by channel receipt +// rather than by sleeping. Production leaves it nil. +var videoToolboxProbeStarted func() + type videoToolboxProbeEntry struct { result hardwareProbeResult expiresAt time.Time // zero: cached for the process lifetime @@ -91,22 +160,45 @@ type HWAccelInfo struct { RenderDevices []string `json:"render_devices"` RenderDeviceDetails []RenderDeviceInfo `json:"render_device_details"` IntelDetected bool `json:"intel_detected"` + DetectedBackends []DetectedBackend `json:"detected_backends,omitempty"` Source string `json:"source"` NodeURL string `json:"node_url,omitempty"` Transformations []TransformationV3 `json:"transformations,omitempty"` ToneMapCapabilities tonemap.Capabilities `json:"tone_map_capabilities,omitempty"` + // BootID is this host's kernel boot identity (Linux only). Paired with a + // render device's PCI address it distinguishes "same GPU, same boot" from + // "same device path on a host that rebooted or was replaced". + BootID string `json:"boot_id,omitempty"` + // NVIDIAGPUUUIDs lists every GPU nvidia-smi reports on this host, sorted. + // + // It exists because a card is not always reachable through a DRM render + // node. An NVIDIA container is routinely given /dev/nvidia* and the toolkit + // with no /dev/dri at all: NVENC works, RenderDeviceDetails is empty, and + // the whole host would otherwise contribute no hardware identity. Two such + // containers sharing one card would then look like two independent GPUs to + // the planner — which is precisely the deployment where GPU sharing is most + // common, and the placement mistake most expensive. + NVIDIAGPUUUIDs []string `json:"nvidia_gpu_uuids,omitempty"` + // CapabilityHash summarizes every hardware-identity and capability field + // below, so a reader can detect change without diffing the whole report. + // Set by the node that serves the report; see ComputeCapabilityHash. + CapabilityHash string `json:"capability_hash,omitempty"` // ProbeRequestTimeoutMillis is the caller-side budget for this node's // effective tone-map probe matrix, including endpoint and transport slack. ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms,omitempty"` } -const ( - probeRequestMinTimeout = 5 * time.Second - probeRequestMaxTimeout = 5 * time.Minute -) +const probeRequestMinTimeout = 5 * time.Second // NormalizeProbeRequestTimeout bounds a node-advertised probe budget while // preserving the caller's established fallback for a missing advertisement. +// +// The ceiling exists because the value comes off the wire from a worker, and a +// caller holds a connection open for it. It is derived from the probe formula +// rather than picked, because a round number picked once was already below what +// a nine-device node legitimately asks for: the API then canceled that node's +// re-probe before its own deadline, every time, and its inventory never landed. +// A ceiling that binds a real configuration is indistinguishable from a bug. func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Duration { if millis <= 0 { return fallback @@ -114,39 +206,98 @@ func NormalizeProbeRequestTimeout(millis int64, fallback time.Duration) time.Dur if millis < probeRequestMinTimeout.Milliseconds() { return probeRequestMinTimeout } - if millis > probeRequestMaxTimeout.Milliseconds() { - return probeRequestMaxTimeout + if ceiling := MaxCapabilityRequestTimeout(); millis > ceiling.Milliseconds() { + return ceiling } return time.Duration(millis) * time.Millisecond } // DetectHWAccel probes this host's GPU hardware and returns structured info. func DetectHWAccel() HWAccelInfo { - return DetectHWAccelWithFFmpeg("") + return DetectHWAccelWithFFmpeg(hwAccelAuto, "", "") } // DetectHWAccelWithFFmpeg probes this host's GPU hardware and configured FFmpeg. -func DetectHWAccelWithFFmpeg(ffmpegPath string) HWAccelInfo { - return DetectHWAccelWithFFmpegContext(context.Background(), ffmpegPath) +func DetectHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { + return DetectHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath, hwDevice) } -// DetectHWAccelWithFFmpegContext probes this host without outliving ctx. -func DetectHWAccelWithFFmpegContext(ctx context.Context, ffmpegPath string) HWAccelInfo { - devices := listRenderDevices(defaultDRIDir) - intel := false - for _, d := range devices { - if isIntelDevice(d) { - intel = true - break +// DetectHWAccelWithFFmpegContext probes this host without outliving ctx. Unlike +// resolution it verifies every backend with candidate hardware, so an operator +// sees why a present GPU was not selected. Resolved still honors the +// pass-through contract: an explicitly configured backend wins even when its +// probe failed, and the report carries the failure reason. +func DetectHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) HWAccelInfo { + info, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccel, ffmpegPath, hwDevice) + return info +} + +// ErrHardwareDetectionIncomplete reports that a detection walk ended before it +// had probed every candidate backend, because its own budget or the caller's +// context ran out. +// +// The report it accompanies is still returned — an operator-facing surface can +// show what was learned — but it must never be hashed and published as this +// host's capabilities. A cut-short walk marks unprobed backends Verified=false +// and resolves to software, which is byte-for-byte what a real hardware failure +// looks like: the API's health sweep would then persist a capability_drift note +// for hardware that is fine, latch it until a clean report arrives, and route +// the node to software encoding in the meantime. +var ErrHardwareDetectionIncomplete = errors.New("hardware detection did not complete within its budget") + +// DetectHWAccelWithFFmpegContextResult is DetectHWAccelWithFFmpegContext with +// the walk's completeness reported. Callers that publish or hash the report — +// the node capability endpoints and their background snapshots — must use this +// form and refuse to publish on ErrHardwareDetectionIncomplete. +func DetectHWAccelWithFFmpegContextResult(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) (HWAccelInfo, error) { + // One listing per walk: the identities below are re-read here, while the + // probe verdicts they accompany stay cached. That asymmetry is deliberate — + // a probe is several ffmpeg execs and a listing is one cheap query, and it + // is the listing that answers "is this card still here". + resetNVIDIAGPUUUIDs() + candidates := collectHWCandidates(hwDevice) + resolved := HWAccelNone + var detected []DetectedBackend + complete := true + switch currentGOOS { + case linuxGOOS: + resolved, detected, complete = walkHWAccelBackends(ctx, ffmpegPath, candidates, false) + case darwinGOOS: + // macOS has no render devices to walk, but it does have hardware: the + // same VideoToolbox probe resolution uses. Without this a capable Mac + // publishes resolved:"none" with no detected backends, and the API + // stores that as its durable inventory — a software-only node, planned + // for software tone mapping, with an operator re-probe that cannot + // verify otherwise because there is nothing here to verify. + entry := DetectedBackend{Backend: transcodeHWVideoToolbox} + if ok, reason := ffmpegSupportsVideoToolboxContext(ctx, ffmpegPath); ok { + entry.Verified = true + resolved = transcodeHWVideoToolbox + } else { + entry.Reason = reason + // A probe cut short by the caller's deadline is not a verdict about + // the hardware, and must not be hashed as one. + complete = ctx.Err() == nil } + detected = append(detected, entry) } - return HWAccelInfo{ - Resolved: ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpegPath), - RenderDevices: devices, - RenderDeviceDetails: renderDeviceDetails(devices), - IntelDetected: intel, + if configured := strings.TrimSpace(hwAccel); configured != "" && configured != hwAccelAuto { + resolved = configured + } + info := HWAccelInfo{ + Resolved: resolved, + RenderDevices: candidates.renderDevices, + RenderDeviceDetails: renderDeviceDetails(candidates.renderDevices), + IntelDetected: candidates.intelPresent, + DetectedBackends: detected, + BootID: detectBootID(), + NVIDIAGPUUUIDs: nvidiaGPUUUIDList(), Source: "local", } + if !complete { + return info, ErrHardwareDetectionIncomplete + } + return info, nil } // PickRenderDevice returns the GPU render device path to use. @@ -166,27 +317,24 @@ func PickRenderDevice(explicit string) string { return dev } -// ResolveHWAccel resolves "auto" using the default FFmpeg binary. -func ResolveHWAccel(hwAccel string) string { - return ResolveHWAccelWithFFmpeg(hwAccel, "") -} - // ResolveHWAccelWithFFmpeg resolves "auto" into a concrete acceleration method // by probing the system and the configured FFmpeg binary. // Preference order: nvenc > qsv > vaapi > none. // Non-"auto" values are returned unchanged. -func ResolveHWAccelWithFFmpeg(hwAccel string, ffmpegPath string) string { - return ResolveHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath) +// hwDevice is the configured playback.hw_device value; probes run against it so +// verification covers the device a transcode will actually open. +func ResolveHWAccelWithFFmpeg(hwAccel, ffmpegPath, hwDevice string) string { + return ResolveHWAccelWithFFmpegContext(context.Background(), hwAccel, ffmpegPath, hwDevice) } -// ResolveHWAccelWithFFmpegContext resolves auto hardware without blocking the -// caller past ctx. A coalesced probe may continue for other callers and cache -// its bounded result after this caller leaves. -func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel string, ffmpegPath string) string { +// ResolveHWAccelWithFFmpegContext resolves auto hardware without allowing any +// FFmpeg capability probe to outlive ctx. A coalesced probe may continue for +// other callers and cache its bounded result after this caller leaves. +func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel, ffmpegPath, hwDevice string) string { if ctx == nil { ctx = context.Background() } - if hwAccel != "auto" { + if hwAccel != hwAccelAuto { return hwAccel } if currentGOOS == darwinGOOS { @@ -199,119 +347,834 @@ func ResolveHWAccelWithFFmpegContext(ctx context.Context, hwAccel string, ffmpeg } return transcodeHWNone } - if currentGOOS != "linux" { - return "none" + if currentGOOS != linuxGOOS { + return HWAccelNone } + resolved, _, _ := walkHWAccelBackends(ctx, ffmpegPath, collectHWCandidates(hwDevice), true) + return resolved +} - devices := listRenderDevices(defaultDRIDir) - var intelDevice string - var nvidiaDevice string - var vaapiDevice string - for _, dev := range devices { +// hwAccelAuto is the configured hw_accel value that asks this package to pick +// a backend by probing the host, rather than the operator naming one outright. +const hwAccelAuto = "auto" + +// hwAccelPreferenceOrder is the auto-resolution order; the first backend whose +// probe passes wins. +var hwAccelPreferenceOrder = []string{transcodeHWNVENC, transcodeHWQSV, transcodeHWVAAPI} + +// hwAccelWalkSlack is the margin the walk deadline carries above the commands +// it may actually run, covering process spawn and the bookkeeping between them. +const hwAccelWalkSlack = 5 * time.Second + +// hwAccelWalkTimeout bounds one full backend walk, so a wedged driver cannot +// stretch detection without limit. tonemap.probeEndpointSlack budgets a +// capability request for it. +// +// It is derived from the matrix the walk will actually run rather than fixed, +// because that matrix grows with the device set: every configured render device +// is probed for both QSV and VAAPI, so three Intel devices legitimately need +// more than the thirty seconds a fixed bound allowed. The walk then marked +// itself incomplete while every individual command was still inside its own +// budget, and /hw-capabilities answered 503 for a node that was working — the +// same failure shape as a bound that is a guess rather than a derivation. +func hwAccelWalkTimeout(candidates hwCandidates) time.Duration { + commands := 0 + for _, backend := range hwAccelPreferenceOrder { + if !candidates.presentFor(backend) { + continue + } + probe, ok := hwBackendProbeFor(backend) + if !ok { + continue + } + commands += probe.commandCount * len(candidates.probeDevicesFor(backend)) + } + if commands == 0 { + commands = 1 + } + return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack +} + +// CapabilityEndpointTimeout is how long one capability endpoint may take to +// answer: the hardware detection walk plus the tone-map matrix and the overhead +// around it. +// +// Both halves scale with the configured device set, and they live in different +// packages — tonemap cannot see the walk, because playback imports tonemap and +// not the reverse. Composing them here is what stops the two from drifting: a +// constant standing in for one of them inside the other has to be raised by +// hand whenever it grows, and the first time the walk grew, it was not. +func CapabilityEndpointTimeout(hwAccel, hwDevice string) time.Duration { + return hwAccelWalkTimeout(collectHWCandidates(hwDevice)) + + tonemap.ProbeEndpointTimeout(hwAccel, hwDevice) +} + +// RegistryCapabilityEndpointTimeout is how long a capability endpoint that +// probes only the transformation registry may take to answer — a proxy's +// snapshot, which reports what its ffmpeg can do and deliberately walks no +// hardware. +// +// It is composed from the same two halves as CapabilityEndpointTimeout, so the +// two cannot drift apart, but with an empty candidate set rather than the host's +// own: no backend has candidates, so the walk falls to its one-command floor, +// and the tone-map half is asked for the software backend, which budgets no +// per-device matrix. +// +// Host-independence is the point, not an incidental saving. This value is +// advertised on the report and covered by its capability hash, so deriving it +// from /dev/dri would put the host's device count inside a proxy's identity: a +// GPU appearing on a machine that also runs a proxy would move that proxy's +// hash, cost the API a refetch and a planning-cache drop, and announce a change +// to capabilities that are by construction the same. +func RegistryCapabilityEndpointTimeout() time.Duration { + return hwAccelWalkTimeout(hwCandidates{}) + + tonemap.ProbeEndpointTimeout(HWAccelNone, "") +} + +// RegistryCapabilityRequestTimeout is RegistryCapabilityEndpointTimeout plus the +// transport margin a remote caller needs, mirroring CapabilityRequestTimeout. It +// is what a registry-only node advertises and what a caller of its capability +// endpoint must allow. +func RegistryCapabilityRequestTimeout() time.Duration { + return RegistryCapabilityEndpointTimeout() + tonemap.ProbeRequestSlack +} + +// HWAccelWalkTimeout is how long a hardware detection walk of this host takes at +// worst, for the configured device set. +// +// Exported for a caller that runs one synchronously while holding an HTTP +// connection open and must size its write deadline to cover it: the walk grows +// with the device count and passes the API listener's own write timeout at eight +// Intel render devices, so a response can be lost while every probe is still +// inside its bound. Unlike the ceiling used to clamp what a *remote* node +// advertises, this classifies devices on the host that is about to walk them, +// which is the host asking. +func HWAccelWalkTimeout(hwDevice string) time.Duration { + return hwAccelWalkTimeout(collectHWCandidates(hwDevice)) +} + +// CapabilityRequestTimeout is CapabilityEndpointTimeout plus the transport +// margin a remote caller needs. It is what a node advertises and what a caller +// of that node's capability endpoint must allow. +func CapabilityRequestTimeout(hwAccel, hwDevice string) time.Duration { + return CapabilityEndpointTimeout(hwAccel, hwDevice) + tonemap.ProbeRequestSlack +} + +// ColdCapabilityRequestTimeout is how long to allow one capability read of a +// node this process has not read successfully yet. +// +// Cold is when the read is slowest — every probe cache on the node is empty and +// the whole matrix runs — so it is exactly the wrong moment to guess low. +// Getting it low does not slow the read down, it cancels it: the node drops out +// of the capability map mid-matrix and playback plans without it. +// +// Every source is a lower bound on what the read may need, so the answer is the +// largest of them — the caller's fallback included, which is why it is a floor +// and not only a last resort. Overshooting holds a dead node's fetch open a +// little longer; undershooting loses a live one. +// +// Two of those sources describe the node, and neither dominates: +// +// - What the node advertised in the report stored for it. That is the node's +// own measurement of its own matrix, it survives an API restart because it +// is persisted with the report, and it is right even when this replica has +// never spoken to the node. It is also as old as the report: an operator who +// has just widened the node's device set has invalidated it. +// - What the node's effective acceleration policy prices — its own override +// where it has one, the cluster setting otherwise. This moves the moment an +// operator edits the node, before any refetch can land, which is exactly +// when the stored figure is wrong. But it is priced *here*, on an API +// replica that does not have the node's cards, so device classification +// falls back to the cheapest backend and it reads as a floor rather than as +// the truth. +// +// The fallback is what each caller is willing to spend on a node it knows +// nothing about, and a node that has never been inventoried is the one most +// likely to be slow — so a policy that happens to price lower does not lower it. +func ColdCapabilityRequestTimeout(storedReport json.RawMessage, hwAccel, hwDevice string, fallback time.Duration) time.Duration { + budget := fallback + if millis := AdvertisedProbeBudgetMillis(storedReport); millis > 0 { + if advertised := NormalizeProbeRequestTimeout(millis, fallback); advertised > budget { + budget = advertised + } + } + if priced := CapabilityRequestTimeout(hwAccel, hwDevice); priced > budget { + budget = priced + } + return budget +} + +// AdvertisedProbeBudgetMillis reads the probe budget out of a stored capability +// report, or 0 when it names none. +// +// The report is parsed for this one field rather than decoded whole: callers +// that want a budget have no business depending on the shape of an inventory, +// and a report they cannot parse is not a reason to fail — it reads as "no +// budget advertised" and the caller falls back. +func AdvertisedProbeBudgetMillis(storedReport json.RawMessage) int64 { + if len(storedReport) == 0 { + return 0 + } + var advertised struct { + ProbeRequestTimeoutMillis int64 `json:"probe_request_timeout_ms"` + } + if err := json.Unmarshal(storedReport, &advertised); err != nil { + return 0 + } + return advertised.ProbeRequestTimeoutMillis +} + +// MaxCapabilityRequestTimeout is the largest budget a node may advertise and be +// believed. +// +// It is computed from the command counts rather than by pricing a synthetic +// device list, because classifying devices reads *this* host's sysfs — and the +// host doing the clamping is an API replica that does not have the remote +// node's cards. Fabricated render paths there resolve to no vendor and count as +// VAAPI alone, so the ceiling came out below what a node with a dozen Intel +// devices legitimately advertises, and the clamp then canceled that node +// before its own matrix could finish. A ceiling that depends on where it is +// evaluated is not a ceiling. +func MaxCapabilityRequestTimeout() time.Duration { + return maxHWAccelWalkTimeout() + tonemap.MaxProbeRequestTimeout() +} + +// maxHWAccelWalkTimeout prices the largest walk the cap allows: every device +// classified as Intel, which is the only vendor that draws two backends, plus +// the single NVENC probe that runs regardless of the device list. +func maxHWAccelWalkTimeout() time.Duration { + perDevice := 0 + for _, backend := range []string{transcodeHWQSV, transcodeHWVAAPI} { + if probe, ok := hwBackendProbeFor(backend); ok { + perDevice += probe.commandCount + } + } + commands := perDevice * tonemap.MaxProbedDevices + if probe, ok := hwBackendProbeFor(transcodeHWNVENC); ok { + commands += probe.commandCount + } + return time.Duration(commands)*hwProbeCommandTimeout + hwAccelWalkSlack +} + +// UsableHWDevices truncates a configured device list to the ceiling this host +// will actually use, which is the same ceiling the probe matrix is capped at. +// +// Past it the walk costs more than the budget every caller allows, so probing +// further guarantees the capability request is canceled rather than finished. +// The cap therefore has to bind selection too, not only probing: a device the +// matrix never reached has no verdict behind it, and dispatching a transcode +// there means finding out whether it works after the session has started, on a +// node whose published capabilities say nothing about it. Truncating in one +// place and balancing over the full list in another is only accidentally safe — +// it holds while a walk has recorded verified devices to narrow against, and +// stops holding in a process that has not walked yet. +// +// The devices past the ceiling are still reported in the inventory, so an +// operator can see what was configured, and the omission is logged rather than +// folded silently into a shorter answer. +func UsableHWDevices(devices []string) []string { + if len(devices) <= tonemap.MaxProbedDevices { + return devices + } + noteHWProbeDevicesTruncated(len(devices)) + return devices[:tonemap.MaxProbedDevices] +} + +// hwProbeDevicesTruncatedLogged latches the truncation warning to one line per +// process: a device list is standing configuration, not an event. +var hwProbeDevicesTruncatedLogged sync.Once + +func noteHWProbeDevicesTruncated(configured int) { + hwProbeDevicesTruncatedLogged.Do(func() { + slog.Warn("hardware detection covers only the first configured devices; the rest are not verified", + "component", "playback", "configured", configured, "probed", tonemap.MaxProbedDevices) + }) +} + +// hwCandidates groups the candidate render devices by the backend each one can +// plausibly drive, before any FFmpeg verification. +type hwCandidates struct { + // renderDevices is this host's full inventory, reported to operators even + // when probes are pinned to a configured subset. + renderDevices []string + nvidia []string + intel []string + vaapi []string + // accessible records, for a configured probe set only, which devices this + // process can actually open. nil means the set came from discovery, which + // already filtered on openability. NVENC never consults it — CUDA names its + // GPU by index or uuid, neither of which is a file. + accessible map[string]bool + // nvencDevice is the CUDA identity a NVENC transcode will actually be given: + // the first configured hw_device entry, exactly what acquireHWDevice hands + // execution, or empty for the CUDA default. The probe uses it so a working + // GPU 0 cannot verify NVENC on behalf of a configured GPU 1 that is absent. + nvencDevice string + nvidiaPresent bool + // intelPresent describes the inventory rather than the probe set, so a + // pinned non-Intel device does not hide an Intel GPU from operators. + intelPresent bool +} + +// collectHWCandidates enumerates render devices once and classifies them by +// sysfs vendor id. Probes run against the configured playback.hw_device set +// when there is one, because that — not whatever sorts first under /dev/dri — +// is what a transcode opens. NVIDIA hardware also counts when only the control +// device is exposed, which is how NVENC-only containers appear. +func collectHWCandidates(configuredDevice string) hwCandidates { + configured := ParseHWDeviceSet(configuredDevice) + candidates := hwCandidates{ + renderDevices: listRenderDevices(defaultDRIDir), + // NVENC is never balanced across a list, so the first entry is the one + // execution uses and therefore the one worth probing. + nvencDevice: configured.First(), + } + probeDevices := configured.List() + if len(probeDevices) == 0 { + probeDevices = candidates.renderDevices + } + probeDevices = UsableHWDevices(probeDevices) + if len(configured.List()) > 0 { + // A configured device this process cannot open can never pass a smoke + // encode, so it is classified for reporting but never probed. This is + // the normal state of a proxy node reading the cluster-wide hw_device + // meant for the transcode nodes. + candidates.accessible = make(map[string]bool, len(probeDevices)) + for _, device := range probeDevices { + candidates.accessible[device] = deviceOpenable(device) + } + } + for _, device := range probeDevices { switch { - case isNVIDIADevice(dev): - if nvidiaDevice == "" { - nvidiaDevice = dev - } - case isIntelDevice(dev): - if intelDevice == "" { - intelDevice = dev - } + case isNVIDIADevice(device): + // NVIDIA render nodes carry no libva driver, so listing one as a + // VAAPI candidate would only fail a probe another GPU can pass. + candidates.nvidia = append(candidates.nvidia, device) + case isIntelDevice(device): + candidates.intel = append(candidates.intel, device) + candidates.vaapi = append(candidates.vaapi, device) default: - if vaapiDevice == "" { - vaapiDevice = dev - } + candidates.vaapi = append(candidates.vaapi, device) + } + } + candidates.nvidiaPresent = len(candidates.nvidia) > 0 || hasNVIDIADevice() + candidates.intelPresent = len(candidates.intel) > 0 + for _, device := range candidates.renderDevices { + if candidates.intelPresent { + break + } + candidates.intelPresent = isIntelDevice(device) + } + return candidates +} + +// devicesFor returns the devices a backend may drive. VAAPI is the generic +// fallback, so every non-NVIDIA candidate belongs to it. +func (c hwCandidates) devicesFor(backend string) []string { + switch backend { + case transcodeHWNVENC: + return c.nvidia + case transcodeHWQSV: + return c.intel + case transcodeHWVAAPI: + return c.vaapi + default: + return nil + } +} + +// presentFor reports whether a backend has hardware worth probing. +func (c hwCandidates) presentFor(backend string) bool { + if backend == transcodeHWNVENC { + return c.nvidiaPresent + } + return len(c.devicesFor(backend)) > 0 +} + +// probeDevicesFor returns the devices a backend's smoke encode is tried +// against, in order. NVENC addresses its GPU through the CUDA runtime rather +// than a render node, so it probes once with no device path. +func (c hwCandidates) probeDevicesFor(backend string) []string { + if backend == transcodeHWNVENC { + // The configured CUDA identity when there is one, so the smoke encode + // opens the same GPU -hwaccel_device will name; empty otherwise, which + // is the CUDA default execution also falls back to. + return []string{c.nvencDevice} + } + return c.devicesFor(backend) +} + +// walkHWAccelBackends verifies each backend with candidate hardware in +// preference order and reports the first one whose probe passes. Resolution +// stops there; detection continues so every candidate backend is reported. +// +// complete reports whether every candidate backend was actually probed. It is +// false when the walk budget or the caller's context ran out partway through, +// which leaves unprobed backends indistinguishable from failed ones — see +// ErrHardwareDetectionIncomplete for why a publisher must not hash such a +// report. +func walkHWAccelBackends(ctx context.Context, ffmpegPath string, candidates hwCandidates, stopAtFirstVerified bool) (resolved string, detected []DetectedBackend, complete bool) { + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithTimeout(ctx, hwAccelWalkTimeout(candidates)) + defer cancel() + complete = true + for _, backend := range hwAccelPreferenceOrder { + if !candidates.presentFor(backend) { + continue + } + // A caller that has already given up must not start further probes. + if ctx.Err() != nil { + complete = false + break } + entry, probedFully := verifyHWAccelBackend(ctx, backend, ffmpegPath, candidates) + if !probedFully { + complete = false + } + if !entry.Verified { + slog.WarnContext(ctx, "hw_accel=auto: candidate hardware failed its FFmpeg probe", + "backend", backend, "devices", entry.Devices, + "ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", entry.Reason) + } else if resolved == "" { + resolved = backend + slog.InfoContext(ctx, "hw_accel=auto: verified hardware backend", "backend", backend, "device", entry.Device) + } + detected = append(detected, entry) + if resolved != "" && stopAtFirstVerified { + // Stopping early is the caller's instruction, not a budget failure: + // the remaining backends are lower preference and would not have + // been selected either way. + return resolved, detected, complete + } + } + if resolved == "" { + slog.InfoContext(ctx, "hw_accel=auto: no verified hardware backend, using software encoding") + return HWAccelNone, detected, complete } + return resolved, detected, complete +} - if nvidiaDevice != "" || hasNVIDIADevice() { - if ok, reason := ffmpegSupportsNVENCContext(ctx, ffmpegPath); ok { - if nvidiaDevice != "" { - slog.Info("hw_accel=auto: NVIDIA GPU detected, using NVENC", "device", nvidiaDevice) - } else { - slog.Info("hw_accel=auto: NVIDIA device detected, using NVENC") +// verifyHWAccelBackend probes a backend's candidate devices in order. A broken +// GPU sorting ahead of a working one does not disable the backend for the whole +// host: the first device that passes decides the backend's verdict. +// +// Whether the walk continues past that device depends on what the rest of the +// list is for. Discovered candidates are alternatives — execution adopts the one +// device detection verified — so probing the losers costs FFmpeg launches and +// buys nothing. A configured multi-device playback.hw_device is different: it is +// a set the device balancer allocates *across*, so every entry is a device a +// transcode can be handed, and stopping early would leave the inventory +// vouching for cards nothing ever tested while acquireHWDevice happily balanced +// onto them. +// +// complete reports whether every candidate was reached. A device left unprobed +// because the budget ran out is not a device that failed, and the difference is +// invisible in the returned entry. +func verifyHWAccelBackend(ctx context.Context, backend, ffmpegPath string, candidates hwCandidates) (entry DetectedBackend, complete bool) { + devices := candidates.probeDevicesFor(backend) + entry = DetectedBackend{Backend: backend, Devices: candidates.devicesFor(backend)} + reasons := make([]string, 0, len(devices)) + probed := false + complete = true + probeEveryDevice := candidates.allocatesAcross(backend) + // Captured before any probe runs: a verdict earned under this generation is + // only worth recording if no invalidation has landed by the time it lands. + generation := hwProbeGeneration() + for _, device := range devices { + if ctx.Err() != nil { + complete = false + break + } + if reason, unprobeable := candidates.unprobeableReason(backend, device); unprobeable { + reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) + continue + } + probed = true + available, reason := ffmpegSupportsBackendContext(ctx, backend, ffmpegPath, device) + if !available && ctx.Err() != nil { + // The walk's own budget ran out while this probe was in flight, so + // the failure describes the deadline and not the hardware. Checking + // only at the top of the loop misses it entirely on the last + // candidate of the last backend, and the report would then publish + // a timeout as a real regression: a new hash, recorded drift, and a + // node resolved to software with nothing wrong with its GPU. + complete = false + break + } + if available { + // Execution has to land on a device a probe passed and not on + // whatever sorts first under /dev/dri, or a report saying "qsv + // verified" is paired with a transcode initializing a GPU the probe + // never touched. + recordVerifiedHWDevice(generation, backend, device) + if !entry.Verified { + entry.Verified = true + entry.Device = device } - return "nvenc" - } else { - slog.Warn("hw_accel=auto: NVIDIA device detected but FFmpeg NVENC probe failed", - "ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", reason) + if !probeEveryDevice { + return entry, complete + } + continue + } + reasons = append(reasons, hwProbeFailureReason(len(devices), device, reason)) + } + if entry.Verified { + // The reasons collected past the first pass belong to devices the + // balancer will now skip, not to a backend that failed. + return entry, complete + } + if len(reasons) == 0 { + reasons = append(reasons, "hardware detection budget exhausted before probing "+backend) + } + entry.Skipped = !probed && len(devices) > 0 && ctx.Err() == nil + entry.Reason = strings.Join(reasons, "; ") + return entry, complete +} + +// hwProbeGeneration reads the current invalidation generation. +func hwProbeGeneration() uint64 { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + return hwProbeCache.generation +} + +// recordVerifiedHWDevice remembers the candidate a backend's smoke encode +// passed on, under the generation that was current when the probe began. +// +// generation is passed in rather than read here because the write happens after +// the probe: an invalidation that landed in between has already discarded this +// verdict, and filing it under the new generation would hand execution a device +// the re-probe was asked to re-verify. A stale generation is dropped instead, +// which reads as "nothing verified yet" — the same state a cold process is in, +// and the one the next walk repairs. +// +// NVENC with no configured device records nothing, because CUDA addresses its +// GPU without a path: there is nothing for execution to adopt. +func recordVerifiedHWDevice(generation uint64, backend, device string) { + if device == "" { + return + } + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + if hwProbeCache.generation != generation { + return + } + key := verifiedHWDeviceKey(generation, backend) + if slices.Contains(hwProbeCache.verifiedDevices[key], device) { + return + } + hwProbeCache.verifiedDevices[key] = append(hwProbeCache.verifiedDevices[key], device) +} + +// VerifiedHWDevice returns the render device this process most recently +// verified the given backend on, or "" when no probe has passed for it. +// +// It exists because auto-detection and execution pick devices independently: +// detection walks a backend's candidates in order and stops at the first that +// passes a smoke encode, while a transcode with no configured playback.hw_device +// falls back to PickRenderDevice, which returns whatever sorts first under +// /dev/dri. On a host whose first render node belongs to a different vendor, +// those are different GPUs, and the transcode initializes hardware that was +// never verified. Reading the verified device closes that gap without making +// every caller of resolution carry a second return value. +func VerifiedHWDevice(backend string) string { + devices := VerifiedHWDevices(backend) + if len(devices) == 0 { + return "" + } + return devices[0] +} + +// VerifiedHWDevices returns every device this process has verified the given +// backend on, in probe order, or nil when no probe has passed for it. +// +// It has more than one entry only for a configured multi-device +// playback.hw_device: detection stops at the first pass when it is choosing a +// backend, but a configured list is *allocated* across, so every entry in it +// has to be probed or the inventory would vouch for cards nothing tested. The +// device balancer intersects its candidates with this set, which is what keeps +// a workload off a card that is present but broken. +func VerifiedHWDevices(backend string) []string { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + return slices.Clone(hwProbeCache.verifiedDevices[verifiedHWDeviceKey(hwProbeCache.generation, backend)]) +} + +func verifiedHWDeviceKey(generation uint64, backend string) string { + return strconv.FormatUint(generation, 10) + "\x00" + backend +} + +// allocatesAcross reports whether every one of a backend's candidate devices is +// one execution may be handed, rather than an alternative detection chooses +// between. +// +// That is true only for a configured multi-device playback.hw_device, which is +// what accessible being non-nil marks. NVENC is never balanced across a list — +// acquireHWDevice hands it the first configured entry — and discovered +// candidates are alternatives, resolved to the single device VerifiedHWDevice +// reports. +func (c hwCandidates) allocatesAcross(backend string) bool { + return backend != transcodeHWNVENC && c.accessible != nil && len(c.devicesFor(backend)) > 1 +} + +// unprobeableReason reports why a candidate cannot be smoke-encoded on, or +// false when it can be. +// +// Both reasons describe the *configuration*, not the hardware, which is why the +// caller records them as skipped rather than failed: neither is evidence that a +// card stopped working. +func (c hwCandidates) unprobeableReason(backend, device string) (string, bool) { + if device == "" { + // NVENC's CUDA default, and the only shape a discovered candidate takes. + return "", false + } + if backend == transcodeHWNVENC { + // A render node path is a perfectly good hw_device for QSV or VAAPI and + // meaningless to CUDA. On a mixed host NVENC stays a candidate through + // hasNVIDIADevice even while the node is deliberately configured for + // QSV, and smoke-encoding CUDA against /dev/dri/renderD128 fails for a + // reason that has nothing to do with the NVIDIA card being fine. + if !isCUDADeviceIdentity(device) { + return "configured hw_device is a render node path, not a CUDA index or GPU uuid", true } + // Otherwise unconditionally probeable: a CUDA index or uuid is not a + // file, so failing to open it is meaningless and only the smoke encode + // can answer. + return "", false + } + if c.accessible == nil { + // Discovered candidates are openable by construction. + return "", false } + if c.accessible[device] { + return "", false + } + return "device not accessible on this node", true +} - if intelDevice != "" { - slog.Info("hw_accel=auto: Intel GPU detected, using QSV", "device", intelDevice) - return "qsv" +// isCUDADeviceIdentity reports whether a configured device names a GPU the way +// CUDA does — an index, a "cuda:N", or a GPU uuid — rather than a DRM render +// node path. +func isCUDADeviceIdentity(device string) bool { + return !strings.ContainsRune(device, '/') +} + +// deviceOpenable mirrors the accessibility filter listRenderDevices applies to +// discovered devices: a device this process cannot open cannot host a probe or +// a transcode. +func deviceOpenable(device string) bool { + f, err := os.Open(device) + if err != nil { + return false } + _ = f.Close() + return true +} - if vaapiDevice != "" { - slog.Info("hw_accel=auto: non-Intel GPU detected, using VAAPI", "device", vaapiDevice) - return "vaapi" +// hwProbeFailureReason attributes a failure to its device only when several +// candidates were tried; a single candidate reads better bare. +func hwProbeFailureReason(candidateCount int, device, reason string) string { + if candidateCount < 2 || device == "" { + return reason } + return device + ": " + reason +} + +// hwBackendProbe verifies one backend against an FFmpeg binary and candidate +// device. commandCount is the number of bounded commands the probe may run and +// budgets the shared deadline. +type hwBackendProbe struct { + commandCount int + run func(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult +} - slog.Info("hw_accel=auto: no compatible GPU devices found, using software encoding") - return "none" +func hwBackendProbeFor(backend string) (hwBackendProbe, bool) { + switch backend { + case transcodeHWNVENC: + return hwBackendProbe{commandCount: 4, run: probeFFmpegNVENCContext}, true + case transcodeHWQSV: + return hwBackendProbe{commandCount: 3, run: probeFFmpegQSVContext}, true + case transcodeHWVAAPI: + return hwBackendProbe{commandCount: 2, run: probeFFmpegVAAPIContext}, true + default: + return hwBackendProbe{}, false + } } -func ffmpegSupportsNVENC(ffmpegPath string) (bool, string) { - return ffmpegSupportsNVENCContext(context.Background(), ffmpegPath) +func ffmpegSupportsBackend(backend, ffmpegPath, device string) (bool, string) { + return ffmpegSupportsBackendContext(context.Background(), backend, ffmpegPath, device) } -func ffmpegSupportsNVENCContext(ctx context.Context, ffmpegPath string) (bool, string) { +// ffmpegSupportsBackendContext verifies one backend, coalescing concurrent cold +// probes and reusing a positive result for the process lifetime. A failure is +// retried once its short negative TTL expires, so a driver or binary repaired +// underneath a running server is picked up without a restart. +func ffmpegSupportsBackendContext(ctx context.Context, backend, ffmpegPath, device string) (bool, string) { if ctx == nil { ctx = context.Background() } + probe, ok := hwBackendProbeFor(backend) + if !ok { + return false, "unsupported hardware backend " + backend + } ffmpegPath = normalizeFFmpegPath(ffmpegPath) - cacheKey := nvencProbeCacheKey(ffmpegPath) - commandTimeout := nvencProbeCommandTimeout - negativeTTL := nvencProbeNegativeTTL - nvencProbeCache.Lock() - if entry, ok := nvencProbeCache.byPath[cacheKey]; ok && nvencProbeCacheEntryCurrent(entry, time.Now()) { - nvencProbeCache.Unlock() + // The flight below outlives an abandoned caller, so every test-mutable seam + // it touches is snapshotted here rather than dereferenced inside it. + commandTimeout := hwProbeCommandTimeout + negativeTTL := hwProbeNegativeTTL + now := hwProbeNow + flightStarted := hwProbeFlightStarted + hwProbeCache.Lock() + // The generation is baked into the key, so an invalidation that lands while + // this probe runs leaves the flight writing to a key nobody will read again + // and sends the next caller to a fresh one. + cacheKey := hwProbeCacheKey(hwProbeCache.generation, ffmpegPath, backend, device) + if entry, ok := hwProbeCache.entries[cacheKey]; ok && hwProbeCacheEntryCurrent(entry, now()) { + hwProbeCache.Unlock() return entry.result.available, entry.result.reason } - nvencProbeCache.Unlock() - - resultCh := nvencProbeCache.group.DoChan(cacheKey, func() (any, error) { - nvencProbeCache.Lock() - cached, ok := nvencProbeCache.byPath[cacheKey] - nvencProbeCache.Unlock() - if ok && nvencProbeCacheEntryCurrent(cached, time.Now()) { + hwProbeCache.Unlock() + + // Raised here, on the calling goroutine, and not inside the function below. + // DoChan schedules that function on a new goroutine and returns without + // waiting for it to run, so a caller whose context is already done takes the + // ctx.Done() branch and returns while the probe has not reached its first + // line. Anything that released its own claim on the encoder when this call + // returned — the transcode node's capability build does exactly that — would + // then hand a re-probe an encoder that is about to be busy. Registering + // before the call closes the window: from here on the flight is counted + // whether or not it has started, and whether or not anyone still waits for + // it. See HWProbesInFlight. + hwProbesInFlight.Add(1) + resultCh := hwProbeCache.group.DoChan(cacheKey, func() (any, error) { + hwProbeCache.Lock() + cached, ok := hwProbeCache.entries[cacheKey] + hwProbeCache.Unlock() + if ok && hwProbeCacheEntryCurrent(cached, now()) { return cached.result, nil } - probeCtx, cancel := context.WithTimeout(context.Background(), 4*commandTimeout+time.Second) + if flightStarted != nil { + flightStarted() + } + probeCtx, cancel := context.WithTimeout(context.Background(), time.Duration(probe.commandCount)*commandTimeout+time.Second) defer cancel() - result := probeFFmpegNVENCContext(probeCtx, ffmpegPath, commandTimeout) - entry := nvencProbeCacheEntry{result: result} + result := probe.run(probeCtx, ffmpegPath, device, commandTimeout) + entry := hwProbeCacheEntry{result: result} if !result.available { - entry.expiresAt = time.Now().Add(negativeTTL) + entry.expiresAt = now().Add(negativeTTL) } - nvencProbeCache.Lock() - nvencProbeCache.byPath[cacheKey] = entry - nvencProbeCache.Unlock() + hwProbeCache.Lock() + hwProbeCache.entries[cacheKey] = entry + hwProbeCache.Unlock() return result, nil }) select { case <-ctx.Done(): + // The flight outlives this caller by design, so the claim goes with it + // rather than with us. DoChan's channel is buffered and every registered + // waiter is served, so this receive always lands and the count always + // comes back down. + go func() { + <-resultCh + hwProbesInFlight.Add(-1) + }() return false, ctx.Err().Error() case shared := <-resultCh: + hwProbesInFlight.Add(-1) if shared.Err != nil { return false, shared.Err.Error() } - result, ok := shared.Val.(nvencProbeResult) + result, ok := shared.Val.(hwProbeResult) if !ok { - return false, "invalid shared NVENC probe result" + return false, "invalid shared hardware probe result" } return result.available, result.reason } } -func nvencProbeCacheEntryCurrent(entry nvencProbeCacheEntry, now time.Time) bool { +// hwProbesInFlight counts hardware smoke encodes running right now, including +// ones whose caller has already given up on them. +var hwProbesInFlight atomic.Int64 + +// HWProbesInFlight reports how many hardware smoke encodes this process has +// claimed the encoder for. +// +// A probe outlives its caller by design: the singleflight task runs on a +// background context so that a canceled request cannot kill work another +// request is waiting on. The consequence is that a component which released its +// own claim on the GPU when its call returned — the transcode node's capability +// build is exactly this — can leave ffmpeg on the card with nothing accounting +// for it. Anything that needs the encoder exclusively must add this to whatever +// else it counts as busy, or it will claim an encoder that is not free and +// publish the collision as a hardware failure. +// +// It counts claims rather than running processes, and deliberately errs high: +// the claim is taken before the probe is dispatched (so a caller can never +// return while its flight is unaccounted for) and released when the result +// lands, and callers that share one flight each hold one. A brief overcount +// costs an operator a 409 and a retry; an undercount costs a false hardware +// regression on a GPU that is fine. +func HWProbesInFlight() int { + count := hwProbesInFlight.Load() + if count < 0 { + return 0 + } + return int(count) +} + +func hwProbeCacheEntryCurrent(entry hwProbeCacheEntry, now time.Time) bool { return entry.result.available || now.Before(entry.expiresAt) } -// nvencProbeCacheKey invalidates cached capability results when an FFmpeg +// InvalidateHWProbeCache drops every cached backend verdict so the next +// detection walk re-runs its FFmpeg smoke encodes against live hardware. +// +// It exists because a positive verdict is cached for the whole process +// lifetime, which is right for routing — a GPU that encoded a frame does not +// stop being able to between two playback requests — but blind to the one event +// that legitimately changes the answer: an operator replacing a driver, moving +// a card, or changing device access underneath a running node. Without this the +// only way to re-verify is a restart. +// +// A probe already in flight is neither canceled nor discarded — canceling +// shared work would fail an unrelated playback request waiting on it — but it +// is superseded: bumping the generation moves every cache and singleflight key, +// so the in-flight probe stores its verdict where nothing will read it and the +// next caller starts a genuinely cold probe rather than joining the old flight. +// That is what makes the operator-facing re-probe honest; without it a re-probe +// racing a background capability fetch would republish the verdict it was asked +// to discard, and report "nothing changed". +func InvalidateHWProbeCache() { + hwProbeCache.Lock() + defer hwProbeCache.Unlock() + hwProbeCache.generation++ + hwProbeCache.entries = make(map[string]hwProbeCacheEntry) + hwProbeCache.verifiedDevices = make(map[string][]string) + // The GPU identity listing goes too. A detection walk drops it on its own, + // but this is exported and nothing here can require that a walk follows — + // the sampler reads identities every few seconds between walks, and an + // operator who re-probes has asked for that to be current now. + resetNVIDIAGPUUUIDs() + // VideoToolbox keeps its own cache, keyed by the FFmpeg binary's identity + // rather than by generation, so it has to be cleared rather than superseded. + // An operator re-probing a Mac is asking the same question everyone else is: + // does this host still encode on hardware. + videoToolboxProbes.Lock() + videoToolboxProbes.byPath = make(map[string]videoToolboxProbeEntry) + videoToolboxProbes.Unlock() +} + +// hwProbeCacheKey separates results per invalidation generation, per backend, +// and per candidate device on top of the FFmpeg binary's identity. +func hwProbeCacheKey(generation uint64, ffmpegPath, backend, device string) string { + return strings.Join([]string{strconv.FormatUint(generation, 10), ffmpegIdentityKey(ffmpegPath), backend, device}, "\x00") +} + +// ffmpegIdentityKey invalidates cached capability results when an FFmpeg // executable is replaced at the same configured path. -func nvencProbeCacheKey(ffmpegPath string) string { +func ffmpegIdentityKey(ffmpegPath string) string { identityPath := ffmpegPath if !strings.ContainsRune(identityPath, os.PathSeparator) { if resolved, err := exec.LookPath(identityPath); err == nil { @@ -387,13 +1250,24 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard } call := &videoToolboxProbeCall{done: make(chan struct{})} videoToolboxProbes.inFlight[cacheKey] = call + // Counted for the life of the smoke encode, not the life of the caller: the + // goroutine below is rooted at Background so an abandoned request cannot + // kill work another is waiting on, and it keeps ffmpeg on the card after + // every caller has returned. Raised here, before the goroutine exists, so + // the claim cannot be observed unraised by anyone this call returns to. + hwProbesInFlight.Add(1) videoToolboxProbes.Unlock() - commandTimeout := nvencProbeCommandTimeout + commandTimeout := hwProbeCommandTimeout retryDelay := videoToolboxProbeRetryDelay + started := videoToolboxProbeStarted go func() { + if started != nil { + started() + } probeCtx, cancel := context.WithTimeout(context.Background(), 4*commandTimeout+time.Second) defer cancel() + defer hwProbesInFlight.Add(-1) result := probeFFmpegVideoToolboxContext(probeCtx, execPath, commandTimeout) entry := videoToolboxProbeEntry{result: result} if !result.available || !result.hevcAvailable { @@ -419,8 +1293,16 @@ func cachedVideoToolboxProbeContext(ctx context.Context, ffmpegPath string) hard // invalidating a cached verdict when that spelling resolves to a replaced // executable. This matters for Homebrew upgrades that swap a symlink target // while Silo remains running. +// +// The invalidation generation leads it, for the same reason the other hardware +// probes bake it into theirs: clearing the cache alone does not supersede a +// probe that is already running. That call stays registered under its key, the +// rebuild joins it instead of starting a cold one, and its completion +// repopulates the map — so an operator's re-probe publishes the very verdict it +// was asked to discard. Moving the key leaves the old flight writing somewhere +// nobody will read. func videoToolboxProbeCacheKey(execPath string) string { - return execPath + "\x00" + nvencProbeCacheKey(execPath) + return strconv.FormatUint(hwProbeGeneration(), 10) + "\x00" + execPath + "\x00" + ffmpegIdentityKey(execPath) } // StartupRetryHWAccel returns the acceleration for the single retry after a @@ -432,7 +1314,7 @@ func videoToolboxProbeCacheKey(execPath string) string { // accel keeps its configured value and moves render devices via AvoidHWDevice. func StartupRetryHWAccel(opts TranscodeOpts) string { if opts.ToneMapMode != tonemap.ModeHardware && - ResolveHWAccelWithFFmpeg(opts.HWAccel, opts.FFmpegPath) == transcodeHWVideoToolbox { + ResolveHWAccelWithFFmpeg(opts.HWAccel, opts.FFmpegPath, opts.HWDevice) == transcodeHWVideoToolbox { return transcodeHWNone } return opts.HWAccel @@ -477,7 +1359,7 @@ func probeFFmpegVideoToolboxContext(ctx context.Context, ffmpegPath string, comm ffmpegFlagHideBanner, ffmpegFlagLogLevel, ffmpegLogLevelError, "-f", "lavfi", - "-i", "testsrc2=size=640x360:rate=1", + "-i", smokeEncodeSource, "-frames:v", "1", "-an", } @@ -523,44 +1405,102 @@ func normalizeFFmpegPath(ffmpegPath string) string { return ffmpegPath } -func probeFFmpegNVENCContext(ctx context.Context, ffmpegPath string, commandTimeout time.Duration) nvencProbeResult { +// probeFFmpegNVENCContext verifies the CUDA decode, scaling, and encode path a +// NVENC transcode depends on. NVENC selects its GPU through CUDA, so the +// candidate device path is not part of the command line. +func probeFFmpegNVENCContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-hwaccels"); err != nil { - return nvencProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} + return hwProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} } else if !ffmpegOutputHasToken(output, "cuda") { - return nvencProbeResult{reason: "cuda hwaccel unavailable"} + return hwProbeResult{reason: "cuda hwaccel unavailable"} } if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { - return nvencProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} - } else if !ffmpegOutputHasToken(output, "h264_nvenc") { - return nvencProbeResult{reason: "h264_nvenc encoder unavailable"} + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, encoderH264NVENC) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264NVENC)} } else if !ffmpegOutputHasToken(output, "hevc_nvenc") { - return nvencProbeResult{reason: "hevc_nvenc encoder unavailable"} + return hwProbeResult{reason: "hevc_nvenc encoder unavailable"} } if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-filters"); err != nil { - return nvencProbeResult{reason: "filters probe failed: " + FormatFFmpegProbeFailure(err, output)} + return hwProbeResult{reason: "filters probe failed: " + FormatFFmpegProbeFailure(err, output)} } else if !ffmpegOutputHasToken(output, "scale_cuda") { - return nvencProbeResult{reason: "scale_cuda filter unavailable"} + return hwProbeResult{reason: "scale_cuda filter unavailable"} } else if !ffmpegOutputHasToken(output, "hwupload_cuda") { - return nvencProbeResult{reason: "hwupload_cuda filter unavailable"} + return hwProbeResult{reason: "hwupload_cuda filter unavailable"} } - if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, - "-hide_banner", - "-loglevel", "error", - "-f", "lavfi", - "-i", "testsrc2=size=640x360:rate=1", - "-frames:v", "1", - "-an", - "-c:v", "h264_nvenc", - "-f", "null", - "-", - ); err != nil { - return nvencProbeResult{reason: "h264_nvenc smoke encode failed: " + FormatFFmpegProbeFailure(err, output)} + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWNVENC, device, commandTimeout) +} + +// probeFFmpegQSVContext verifies the VAAPI-derived QSV chain against a +// candidate Intel render device. Either hwaccel listing is enough: the chain +// initializes a VAAPI display and derives the QSV device from it. +func probeFFmpegQSVContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-hwaccels"); err != nil { + return hwProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, transcodeHWQSV) && !ffmpegOutputHasToken(output, transcodeHWVAAPI) { + return hwProbeResult{reason: "qsv and vaapi hwaccels unavailable"} } - return nvencProbeResult{available: true} + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, encoderH264QSV) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264QSV)} + } else if !ffmpegOutputHasToken(output, "hevc_qsv") { + return hwProbeResult{reason: "hevc_qsv encoder unavailable"} + } + + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWQSV, device, commandTimeout) +} + +// probeFFmpegVAAPIContext verifies the generic VAAPI encode path. VAAPI is the +// last fallback and only needs H.264 encoding to be useful, so the listing gate +// stays narrower than QSV's. +func probeFFmpegVAAPIContext(ctx context.Context, ffmpegPath, device string, commandTimeout time.Duration) hwProbeResult { + if output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, "-hide_banner", "-encoders"); err != nil { + return hwProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)} + } else if !ffmpegOutputHasToken(output, encoderH264VAAPI) { + return hwProbeResult{reason: encoderUnavailableReason(encoderH264VAAPI)} + } + + return smokeEncodeResult(ctx, ffmpegPath, transcodeHWVAAPI, device, commandTimeout) +} + +// smokeEncodeResult runs the backend's bounded single-frame encode, which is +// the only step that exercises the driver rather than FFmpeg's build flags. +func smokeEncodeResult(ctx context.Context, ffmpegPath, backend, device string, commandTimeout time.Duration) hwProbeResult { + output, err := runFFmpegProbe(ctx, commandTimeout, ffmpegPath, hardwareSmokeEncodeArgs(backend, device)...) + if err != nil { + return hwProbeResult{reason: hardwareEncoder(backend) + " smoke encode failed: " + FormatFFmpegProbeFailure(err, output)} + } + return hwProbeResult{available: true} +} + +// H.264 encoder names FFmpeg reports for each hardware backend. +const ( + encoderH264QSV = "h264_qsv" + encoderH264VAAPI = "h264_vaapi" + encoderH264NVENC = "h264_nvenc" +) + +// encoderUnavailableReason reports that a probe's -encoders listing did not +// include the given encoder. +func encoderUnavailableReason(encoder string) string { + return encoder + " encoder unavailable" +} + +// hardwareEncoder returns the H.264 encoder paired with a backend. +func hardwareEncoder(backend string) string { + switch backend { + case transcodeHWQSV: + return encoderH264QSV + case transcodeHWVAAPI: + return encoderH264VAAPI + default: + return encoderH264NVENC + } } func runFFmpegProbe(ctx context.Context, timeout time.Duration, ffmpegPath string, args ...string) ([]byte, error) { @@ -667,7 +1607,15 @@ func detectRenderDevice(driDir string) string { // RenderDeviceInfo describes one render device for operator-facing surfaces. type RenderDeviceInfo struct { - Path string `json:"path"` + Path string `json:"path"` + // PCIAddress is the device's sysfs PCI slot (e.g. 0000:03:00.0). It is + // stable across reboots for a card that stays in its slot, which /dev/dri + // paths are not, so it — not Path — identifies the hardware. + PCIAddress string `json:"pci_address,omitempty"` + // GPUUUID is NVIDIA's own permanent GPU identity, reported only when + // nvidia-smi is installed. It survives a card moving between slots and + // hosts, so it outranks PCIAddress wherever both are present. + GPUUUID string `json:"gpu_uuid,omitempty"` Description string `json:"description"` } @@ -703,14 +1651,250 @@ func readSysfsID(path string) string { return strings.TrimSpace(string(data)) } +// RenderDeviceIdentity is a render device's hardware identity, without the +// probing that a full capability report performs. +type RenderDeviceIdentity struct { + // Path is the render node, e.g. /dev/dri/renderD128. + Path string + // PCIAddress is the sysfs PCI slot, e.g. 0000:03:00.0. + PCIAddress string + // Vendor is "intel", "nvidia", "amd", or empty when sysfs names one we do + // not recognize. + Vendor string +} + +// RenderDeviceIdentities enumerates this host's render devices with the sysfs +// identity of each. +// +// It exists for callers that need to correlate a device across surfaces — +// notably resource sampling, which learns about GPUs by PCI address from DRM +// fdinfo and has to name them the way the rest of the server does. It runs no +// ffmpeg probe and takes no lock: it is a sysfs read, cheap enough to call on a +// sampling interval, unlike DetectHWAccelWithFFmpeg. +func RenderDeviceIdentities() []RenderDeviceIdentity { + if currentGOOS != linuxGOOS { + return nil + } + devices := listRenderDevices(defaultDRIDir) + identities := make([]RenderDeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, RenderDeviceIdentity{ + Path: device, + PCIAddress: renderDevicePCIAddress(device), + Vendor: renderDeviceVendor(device), + }) + } + return identities +} + +// SamplerDeviceIdentities is RenderDeviceIdentities in the shape the resource +// sampler consumes, ready to hand to nodemetrics.Options.DeviceIdentities. +// +// It lives here rather than beside each caller because the conversion is one +// fact, not three: every process that samples resources — the API host and both +// node types — needs the same translation, and three copies would drift the +// moment DeviceIdentity gains a field. The dependency points this way on +// purpose: nodemetrics stays free of any playback import, which is why it takes +// the identities as a provider in the first place. +func SamplerDeviceIdentities() []nodemetrics.DeviceIdentity { + devices := RenderDeviceIdentities() + identities := make([]nodemetrics.DeviceIdentity, 0, len(devices)) + for _, device := range devices { + identities = append(identities, nodemetrics.DeviceIdentity{ + Path: device.Path, + PCIAddress: device.PCIAddress, + Vendor: device.Vendor, + }) + } + return identities +} + +// renderDeviceVendor maps a device's sysfs PCI vendor id to a short label. +func renderDeviceVendor(renderDevPath string) string { + name := filepath.Base(renderDevPath) + switch readSysfsID(filepath.Join(sysClassDRMDir, name, "device", "vendor")) { + case "0x8086": + return "intel" + case "0x10de": + return "nvidia" + case "0x1002": + return "amd" + default: + return "" + } +} + // renderDeviceDetails describes every listed device. func renderDeviceDetails(devices []string) []RenderDeviceInfo { details := make([]RenderDeviceInfo, 0, len(devices)) for _, device := range devices { + pciAddress := renderDevicePCIAddress(device) details = append(details, RenderDeviceInfo{ Path: device, + PCIAddress: pciAddress, + GPUUUID: renderDeviceGPUUUID(device, pciAddress), Description: describeRenderDevice(device), }) } return details } + +// renderDevicePCIAddress resolves the sysfs device symlink behind a render node +// and returns its PCI slot. Best effort: an unresolvable link (a virtual or +// non-PCI device, a restricted sysfs) yields an empty address rather than an +// error, because a missing identity only weakens inventory, never breaks it. +func renderDevicePCIAddress(renderDevPath string) string { + name := filepath.Base(renderDevPath) + devicePath := filepath.Join(sysClassDRMDir, name, "device") + // sysfs exposes this as a symlink into the PCI tree. Anything else is a + // device with no PCI identity, and its own directory name would be "device". + info, err := os.Lstat(devicePath) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + return "" + } + resolved, err := filepath.EvalSymlinks(devicePath) + if err != nil { + return "" + } + return filepath.Base(resolved) +} + +// renderDeviceGPUUUID returns NVIDIA's permanent GPU identity for a render +// device. Only NVIDIA-vendor devices are looked up: no other vendor publishes +// such an id, so querying for them would only cost a subprocess. +func renderDeviceGPUUUID(renderDevPath, pciAddress string) string { + if pciAddress == "" || !isNVIDIADevice(renderDevPath) { + return "" + } + return nvidiaGPUUUIDsByPCIAddress()[normalizePCIAddress(pciAddress)] +} + +// nvidiaSMIQueryTimeout bounds the single nvidia-smi invocation. A wedged +// driver makes nvidia-smi hang, and hardware inventory must not inherit that. +var nvidiaSMIQueryTimeout = 3 * time.Second + +// nvidiaSMIQuery is the execution seam for the GPU uuid listing; tests replace +// it rather than installing a fake binary on PATH. +var nvidiaSMIQuery = runNVIDIASMIQuery + +// nvidiaGPUUUIDs caches the nvidia-smi listing. Within one generation of the +// probe caches a second query could only cost a subprocess to learn the same +// answer, since GPU identities do not change under a running kernel. +// +// It lives for one detection walk, not for the process, which a sync.Once would +// make it. Everything that reads it is asking what hardware this host has right +// now: drift detection compares one walk's answer against the last one, and +// shared-GPU placement groups nodes by it. A listing that outlives its walk +// makes all of those describe a machine that no longer exists — an nvidia-smi +// missing or broken at first call is never asked again, a card swapped into the +// same slot keeps answering to its predecessor's uuid, and a card hot-removed +// from an NVIDIA-only node goes on being reported by every scheduled snapshot +// until someone re-probes by hand. On such a node that uuid is the card's only +// identity, so nothing else in the report would show it gone. +var nvidiaGPUUUIDs struct { + mu sync.Mutex + loaded bool + byPCI map[string]string +} + +// resetNVIDIAGPUUUIDs drops the cached listing so the next lookup re-queries. +func resetNVIDIAGPUUUIDs() { + nvidiaGPUUUIDs.mu.Lock() + defer nvidiaGPUUUIDs.mu.Unlock() + nvidiaGPUUUIDs.loaded = false + nvidiaGPUUUIDs.byPCI = nil +} + +func runNVIDIASMIQuery(ctx context.Context) ([]byte, error) { + path, err := exec.LookPath("nvidia-smi") + if err != nil { + return nil, err + } + return exec.CommandContext(ctx, path, "--query-gpu=uuid,pci.bus_id", "--format=csv,noheader").Output() +} + +func nvidiaGPUUUIDsByPCIAddress() map[string]string { + nvidiaGPUUUIDs.mu.Lock() + defer nvidiaGPUUUIDs.mu.Unlock() + if nvidiaGPUUUIDs.loaded { + return nvidiaGPUUUIDs.byPCI + } + nvidiaGPUUUIDs.loaded = true + ctx, cancel := context.WithTimeout(context.Background(), nvidiaSMIQueryTimeout) + defer cancel() + output, err := nvidiaSMIQuery(ctx) + if err != nil { + // Expected on every host without the NVIDIA toolkit installed, so this + // stays at debug: the report is complete without it. + slog.Debug("nvidia-smi gpu identity query unavailable", "component", "playback", "error", err) + return nil + } + nvidiaGPUUUIDs.byPCI = parseNVIDIAGPUUUIDs(output) + return nvidiaGPUUUIDs.byPCI +} + +// nvidiaGPUUUIDList returns every uuid nvidia-smi reports, sorted and +// deduplicated, independent of whether the card has a readable render node. +func nvidiaGPUUUIDList() []string { + byPCI := nvidiaGPUUUIDsByPCIAddress() + if len(byPCI) == 0 { + return nil + } + uuids := make([]string, 0, len(byPCI)) + for _, uuid := range byPCI { + if uuid != "" && !slices.Contains(uuids, uuid) { + uuids = append(uuids, uuid) + } + } + slices.Sort(uuids) + return uuids +} + +// parseNVIDIAGPUUUIDs reads "csv,noheader" rows of ", " and +// keys them by normalized PCI address. Malformed rows are skipped. +func parseNVIDIAGPUUUIDs(output []byte) map[string]string { + byPCI := make(map[string]string) + for line := range strings.Lines(string(output)) { + uuid, address, ok := strings.Cut(line, ",") + if !ok { + continue + } + uuid = strings.TrimSpace(uuid) + address = normalizePCIAddress(address) + if uuid == "" || address == "" { + continue + } + byPCI[address] = uuid + } + return byPCI +} + +// normalizePCIAddress makes sysfs and nvidia-smi addresses comparable: +// sysfs prints a 16-bit domain (0000:03:00.0) and nvidia-smi a 32-bit one +// (00000000:03:00.0), and neither guarantees a case. +func normalizePCIAddress(address string) string { + address = strings.ToLower(strings.TrimSpace(address)) + domain, rest, ok := strings.Cut(address, ":") + if !ok { + return address + } + value, err := strconv.ParseUint(domain, 16, 64) + if err != nil { + return address + } + return fmt.Sprintf("%04x:%s", value, rest) +} + +// detectBootID reads the kernel's per-boot identity. It is Linux-only and +// best effort: an empty value simply means device identities cannot be scoped +// to a boot on this host. +func detectBootID() string { + if currentGOOS != linuxGOOS { + return "" + } + data, err := os.ReadFile(procBootIDPath) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} diff --git a/internal/playback/gpudetect_invalidate_test.go b/internal/playback/gpudetect_invalidate_test.go new file mode 100644 index 000000000..f66f578fa --- /dev/null +++ b/internal/playback/gpudetect_invalidate_test.go @@ -0,0 +1,55 @@ +package playback + +import ( + "os" + "strings" + "testing" + "time" +) + +// A verified backend is cached for the process lifetime, which is exactly the +// blind spot InvalidateHWProbeCache exists to close: without it, a driver +// upgraded underneath a running node can only be noticed by restarting. The +// observable contract is that ffmpeg is executed again, so this counts +// invocations of the fake binary rather than inspecting the cache. +func TestInvalidateHWProbeCacheForcesAnotherProbe(t *testing.T) { + setupHWAccelTest(t) + // This case counts commands rather than racing a deadline, so it opts out of + // the shared fixture's very short probe timeout: a fake ffmpeg killed by a + // loaded machine would fail here as a probe failure, which is not what is + // under test. Restored by the fixture's own cleanup. + hwProbeCommandTimeout = 5 * time.Second + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + + probeCommands := func() int { + data, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatalf("read probe log: %v", err) + } + return strings.Count(string(data), "\n") + } + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("first probe failed: %s", reason) + } + first := probeCommands() + if first == 0 { + t.Fatal("first probe ran no ffmpeg commands") + } + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("cached probe failed: %s", reason) + } + if got := probeCommands(); got != first { + t.Fatalf("cached probe ran %d commands, want the cached verdict reused (%d)", got-first, first) + } + + InvalidateHWProbeCache() + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("probe after invalidation failed: %s", reason) + } + if got := probeCommands(); got != first*2 { + t.Fatalf("probe after invalidation ran %d commands total, want %d", got, first*2) + } +} diff --git a/internal/playback/gpudetect_publish_test.go b/internal/playback/gpudetect_publish_test.go new file mode 100644 index 000000000..1ccad984b --- /dev/null +++ b/internal/playback/gpudetect_publish_test.go @@ -0,0 +1,822 @@ +package playback + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// A detection walk that ran out of budget marks backends it never reached +// Verified=false, which is byte-identical to a real hardware failure. A node +// that hashed and published that report would tell the API its GPU regressed, +// and the API would persist a capability_drift note, latch it until a clean +// report arrives, and route the node to software in the meantime — all for +// hardware that is fine. So the incompleteness has to reach the publisher. +func TestDetectHWAccelReportsAnIncompleteWalk(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + info, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, ffmpeg.path, "") + if !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete", err) + } + // The report still comes back for an operator-facing surface to show; it is + // only publishing it as this host's capabilities that is refused. + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want an abandoned walk to resolve to software", info.Resolved) + } +} + +// The complement: a walk that reached every candidate backend publishes +// normally, or nothing would ever be inventoried. +func TestDetectHWAccelReportsACompleteWalk(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWQSV { + t.Fatalf("Resolved = %q, want qsv", info.Resolved) + } +} + +// Detection walks a backend's candidates in order and stops at the first that +// passes a smoke encode; execution with no configured playback.hw_device used to +// fall back to PickRenderDevice, which returns whatever sorts first under +// /dev/dri. On a mixed-vendor host those are different GPUs, so a report saying +// "qsv verified" was paired with a transcode initializing a card the probe had +// never touched. +func TestAcquireHWDeviceUsesTheVerifiedRenderDevice(t *testing.T) { + env := setupHWAccelTest(t) + // renderD128 sorts first and is AMD, so it is not a QSV candidate at all; + // only renderD129 can pass the probe. + env.addRenderDevice(t, "renderD128", "0x1002") + env.addRenderDevice(t, "renderD129", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := ResolveHWAccelWithFFmpeg(hwAccelAuto, ffmpeg.path, ""); got != transcodeHWQSV { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv", got) + } + verified := VerifiedHWDevice(transcodeHWQSV) + if want := env.devicePath("renderD129"); verified != want { + t.Fatalf("VerifiedHWDevice(qsv) = %q, want %q", verified, want) + } + // The unverified device is the one auto-detection would otherwise pick. + if fallback := PickRenderDevice(""); fallback == verified { + t.Skip("test setup no longer distinguishes the verified device from the first render node") + } + + device, release := AcquireHWDevice("", transcodeHWQSV) + defer release() + if device != verified { + t.Fatalf("AcquireHWDevice() = %q, want the verified device %q", device, verified) + } + // Counting it is the other half: a default-configured node reported zero + // sessions beside a busy engine because an unnamed device was never counted. + if got := hwDeviceActiveCount(verified); got != 1 { + t.Fatalf("active workloads on %s = %d, want 1", verified, got) + } + release() + if got := hwDeviceActiveCount(verified); got != 0 { + t.Fatalf("active workloads after release = %d, want 0", got) + } + + // An operator re-probe discards the verdicts, so the device they blessed + // goes with them: answering from the old generation would let execution + // keep using a device the re-probe was asked to re-verify. + InvalidateHWProbeCache() + if got := VerifiedHWDevice(transcodeHWQSV); got != "" { + t.Fatalf("VerifiedHWDevice(qsv) after invalidation = %q, want empty", got) + } +} + +// Invalidation has to supersede a probe already in flight, not merely clear the +// map in front of it. The operator-facing re-probe exists to force a cold +// re-verification; if a probe that started before the invalidation could hand +// its pre-invalidation verdict to the caller that invalidated, the action would +// republish exactly what it was asked to discard and report "nothing changed". +func TestInvalidateHWProbeCacheSupersedesAnInFlightProbe(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + device := env.devicePath("renderD128") + + // The race is decided by channel receipts rather than a sleep: the first + // flight parks inside the probe until the invalidation has landed, so this + // cannot pass or fail on how loaded the machine is. + started := make(chan struct{}) + blocked := make(chan struct{}) + var flights atomic.Int32 + hwProbeFlightStarted = func() { + if flights.Add(1) == 1 { + close(started) + <-blocked + } + } + t.Cleanup(func() { hwProbeFlightStarted = nil }) + + var wg sync.WaitGroup + wg.Go(func() { + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Errorf("in-flight probe failed: %s", reason) + } + }) + + <-started + InvalidateHWProbeCache() + close(blocked) + + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Fatalf("post-invalidation probe failed: %s", reason) + } + wg.Wait() + + // Two independent smoke encodes ran: the second call started its own probe + // rather than joining the flight the invalidation superseded, which is the + // whole difference between clearing the map and moving the key. + if got := smokeEncodeCount(t, ffmpeg.logPath); got < 2 { + t.Fatalf("smoke encodes = %d, want the post-invalidation probe to run its own", got) + } +} + +// smokeEncodeCount counts the synthetic single-frame encodes in a fake ffmpeg's +// command log. Every hardware probe ends in exactly one. +func smokeEncodeCount(t *testing.T, logPath string) int { + t.Helper() + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read ffmpeg probe log: %v", err) + } + return strings.Count(string(data), "testsrc2") +} + +// devicePath is the full path of a render device added to this test's /dev/dri +// stand-in. +func (e *hwAccelTestEnv) devicePath(name string) string { + return filepath.Join(e.driDir, name) +} + +// NVENC takes the configured hw_device through to -hwaccel_device, so probing +// with an empty device lets a working GPU 0 verify the backend on behalf of a +// configured GPU 1 that is absent or broken — and the real transcode then fails. +func TestNVENCProbesTheConfiguredCUDADevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + probe := fullyCapableProbe() + // Only the default CUDA device works; the configured one does not. + probe.smokeDeviceFailures = []string{"1"} + ffmpeg := writeFakeFFmpeg(t, probe) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "1") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if backend.Verified { + t.Fatalf("nvenc reported verified while the configured CUDA device fails: %+v", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) +} + +// A CUDA index is not a filesystem path, so the accessibility filter that keeps +// a proxy from probing a render node it cannot open must not silently skip it. +func TestNVENCConfiguredDeviceIsProbedNotSkipped(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "0") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if backend.Skipped { + t.Fatalf("nvenc was skipped for an unopenable CUDA index: %+v", backend) + } + if !backend.Verified { + t.Fatalf("nvenc should verify against a working CUDA device: %+v", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) +} + +// An explicitly configured backend short-circuits resolution, so the detection +// walk never runs and nothing is ever recorded as verified. Without a fallback +// the workload went uncounted and the node reported zero GPU sessions while it +// transcoded — the same reporting hole the auto path had, on the branch that +// never probes. +func TestAcquireHWDeviceCountsTheAutoDetectedDeviceWithoutAProbe(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + + // No walk has run, so nothing is verified — exactly the state a host with + // hw_accel=qsv and no hw_device is in. + if got := VerifiedHWDevice(transcodeHWQSV); got != "" { + t.Fatalf("VerifiedHWDevice = %q, want nothing verified for this test", got) + } + + device, release := AcquireHWDevice("", transcodeHWQSV) + defer release() + + want := env.devicePath("renderD128") + if device != want { + t.Fatalf("AcquireHWDevice() = %q, want the device execution will pick, %q", device, want) + } + if got := hwDeviceActiveCount(want); got != 1 { + t.Fatalf("active workloads on %s = %d, want the transcode counted", want, got) + } + release() + if got := hwDeviceActiveCount(want); got != 0 { + t.Fatalf("active workloads after release = %d, want 0", got) + } +} + +// The identity listing used to be a sync.Once, so an nvidia-smi that was +// missing at first call was never asked again and a card swapped into the same +// slot kept answering to its predecessor's uuid — both of which feed drift +// detection and shared-GPU placement. A re-probe is exactly when either becomes +// true, so it drops the listing too. +func TestInvalidateHWProbeCacheRequeriesNVIDIAIdentities(t *testing.T) { + setupHWAccelTest(t) + + queries := 0 + answer := "" + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + queries++ + if answer == "" { + return nil, errors.New("nvidia-smi not installed") + } + return []byte(answer), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + + if got := nvidiaGPUUUIDsByPCIAddress(); len(got) != 0 { + t.Fatalf("identities = %v, want none while nvidia-smi is unavailable", got) + } + if nvidiaGPUUUIDsByPCIAddress(); queries != 1 { + t.Fatalf("queries = %d, want the failure cached within a generation", queries) + } + + // The toolkit is installed, or the card is replaced. Only a re-probe should + // make the process notice. + answer = "GPU-new, 00000000:03:00.0\n" + if got := nvidiaGPUUUIDsByPCIAddress(); len(got) != 0 { + t.Fatalf("identities = %v, want the cached answer until the caches are dropped", got) + } + + InvalidateHWProbeCache() + got := nvidiaGPUUUIDsByPCIAddress() + if got["0000:03:00.0"] != "GPU-new" { + t.Fatalf("identities = %v, want the re-probe to pick up the new uuid", got) + } +} + +// On a mixed NVIDIA/Intel host NVENC stays a candidate through hasNVIDIADevice +// even when the node is deliberately configured for QSV on a render node. Smoke +// encoding CUDA against /dev/dri/renderD128 fails for a reason that has nothing +// to do with the NVIDIA card, and a non-skipped failure latches a drift warning +// that cannot clear while the QSV policy stands. +func TestNVENCSkippedWhenTheConfiguredDeviceIsARenderPath(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addNVIDIADevice(t, "nvidia0") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult( + context.Background(), hwAccelAuto, ffmpeg.path, env.devicePath("renderD128")) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if !backend.Skipped { + t.Fatalf("nvenc = %+v, want it skipped rather than failed for a render-path device", backend) + } + if backend.Verified { + t.Fatalf("nvenc = %+v, want no verification claimed", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) +} + +// A CUDA identity is still probed: skipping is about the device being the wrong +// *kind* of name, not about avoiding NVENC. +func TestNVENCProbedWhenTheConfiguredDeviceIsACUDAIdentity(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + for _, device := range []string{"0", "cuda:1", "GPU-a1b2c3d4"} { + t.Run(device, func(t *testing.T) { + InvalidateHWProbeCache() + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, device) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend != transcodeHWNVENC { + continue + } + if backend.Skipped || !backend.Verified { + t.Fatalf("nvenc = %+v, want a CUDA identity probed and verified", backend) + } + return + } + t.Fatalf("no nvenc entry in %+v", info.DetectedBackends) + }) + } +} + +// A configured multi-device hw_device is a set the balancer allocates *across*, +// not a list of alternatives detection picks from. Stopping the walk at the +// first pass left every later entry untested while the report said the backend +// was verified, and acquireHWDevice balanced onto them regardless — so a share +// of the node's transcodes started on a card that had already failed its smoke +// encode, and each of them died at ffmpeg init. +func TestConfiguredDeviceListIsProbedInFullAndBalancedOnlyAcrossPasses(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + probe := successfulQSVProbe() + probe.smokeDeviceFailures = []string{"renderD129"} + ffmpeg := writeFakeFFmpeg(t, probe) + + working, broken := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := working + "," + broken + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, configured) + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWQSV { + t.Fatalf("resolved = %q, want qsv from the device that passed", info.Resolved) + } + + if got := VerifiedHWDevices(transcodeHWQSV); !slices.Equal(got, []string{working}) { + t.Fatalf("verified devices = %v, want only the card whose probe passed", got) + } + + // Ten acquisitions is far more than the balancer needs to reach a second + // device: with both present it alternates on the very next one. + releases := make([]func(), 0, 10) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for i := range 10 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + if device != working { + t.Fatalf("acquisition %d selected %q, want the verified device %q", i, device, working) + } + } +} + +// With nothing verified — a cold process, or hw_accel named explicitly so the +// walk never ran — the balancer must not narrow to an empty set and must keep +// using every present device exactly as before. +func TestBalancingIsUnchangedWhenNoDeviceHasBeenVerified(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + first, second := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := first + "," + second + + selected := map[string]bool{} + releases := make([]func(), 0, 2) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range 2 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + selected[device] = true + } + if len(selected) != 2 { + t.Fatalf("selected %v, want both devices used when no probe has ruled either out", selected) + } +} + +// The narrowing above is only safe because the whole configured list is probed. +// If the walk stopped at the first pass, every other card in the set would be +// unverified and the balancer would collapse a multi-GPU node onto one device — +// trading a correctness bug for a capacity one. +func TestEveryConfiguredDeviceThatPassesStaysInTheBalancer(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + + first, second := env.devicePath("renderD128"), env.devicePath("renderD129") + configured := first + "," + second + + if _, err := DetectHWAccelWithFFmpegContextResult( + context.Background(), hwAccelAuto, ffmpeg.path, configured); err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if got := VerifiedHWDevices(transcodeHWQSV); !slices.Equal(got, []string{first, second}) { + t.Fatalf("verified devices = %v, want both cards in the configured set", got) + } + + selected := map[string]bool{} + releases := make([]func(), 0, 2) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range 2 { + device, _, release := acquireHWDevice(configured, transcodeHWQSV, "") + releases = append(releases, release) + selected[device] = true + } + if len(selected) != 2 { + t.Fatalf("selected %v, want the workload spread across both verified cards", selected) + } +} + +// An NVIDIA container is routinely given /dev/nvidia* and the toolkit with no +// /dev/dri at all: NVENC works and render_device_details is empty. Without a +// standalone uuid list the whole host contributes no hardware identity, so two +// such containers on one card look like two independent GPUs to the planner — +// which is the deployment where GPU sharing is most common and the placement +// mistake most expensive. +func TestCUDAOnlyHostPublishesItsGPUIdentities(t *testing.T) { + env := setupHWAccelTest(t) + env.addNVIDIADevice(t, "nvidia0") + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + return []byte("GPU-aaa, 00000000:03:00.0\nGPU-bbb, 00000000:04:00.0\n"), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if len(info.RenderDeviceDetails) != 0 { + t.Fatalf("render devices = %+v, want none on a container with no /dev/dri", info.RenderDeviceDetails) + } + if !slices.Equal(info.NVIDIAGPUUUIDs, []string{"GPU-aaa", "GPU-bbb"}) { + t.Fatalf("nvidia gpu uuids = %v, want both cards nvidia-smi reported", info.NVIDIAGPUUUIDs) + } + + // A card appearing or disappearing has to move the hash, or a node that + // gained or lost one is never refetched. + withOne := info + withOne.NVIDIAGPUUUIDs = []string{"GPU-aaa"} + if ComputeCapabilityHash(info) == ComputeCapabilityHash(withOne) { + t.Fatal("capability hash ignores the GPU identity list; a lost card would never trigger a refetch") + } +} + +// The walk's budget can also run out *inside* a probe rather than between two of +// them. Checking the context only at the top of the loop misses that entirely on +// the last candidate of the last backend: the probe returns a context error, the +// loop ends normally, and the report goes out claiming a verified regression — +// a new hash, a recorded drift note, and a node routed to software, for a GPU +// that is fine and merely slow to answer. +func TestDetectHWAccelReportsADeadlineInsideTheFinalProbe(t *testing.T) { + env := setupHWAccelTest(t) + // An AMD card so VAAPI is the only backend with candidates: with QSV also in + // play, the walk's between-backends check would mask the gap. + env.addRenderDevice(t, "renderD128", "0x1002") + probe := successfulVAAPIProbe() + probe.hang = true + ffmpeg := writeFakeFFmpeg(t, probe) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + t.Cleanup(cancel) + + info, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, ffmpeg.path, "") + if !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete for a deadline inside the probe", err) + } + for _, backend := range info.DetectedBackends { + if backend.Backend == transcodeHWVAAPI && backend.Verified { + t.Fatalf("vaapi = %+v, want no verification claimed", backend) + } + } +} + +// A probe outlives its caller by design, so a component that released its own +// claim on the GPU when its call returned can leave ffmpeg on the card with +// nothing accounting for it. The count is what lets the transcode node's +// re-probe gate see that. +func TestHWProbesInFlightCountsADetachedProbe(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + device := env.devicePath("renderD128") + + // Asserted as a floor rather than an exact count: the process-global counter + // is shared with any probe an earlier test detached, and those drain on + // their own schedule. What this test owns is that its own flight is counted + // while it runs and released when it lands. + // + // Decided by channel receipts, not a sleep: the flight parks inside the + // probe until this test has read the count. + started := make(chan struct{}) + release := make(chan struct{}) + hwProbeFlightStarted = func() { + close(started) + <-release + } + t.Cleanup(func() { hwProbeFlightStarted = nil }) + + var wg sync.WaitGroup + wg.Go(func() { + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, device); !ok { + t.Errorf("probe failed: %s", reason) + } + }) + + <-started + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d while a smoke encode is running, want at least 1", got) + } + close(release) + wg.Wait() + awaitNoProbesInFlight(t) +} + +// The claim has to be taken on the calling goroutine, not inside the function +// singleflight schedules. DoChan returns without waiting for that goroutine to +// run, so a caller whose context is already done returns immediately — and a +// component that released its own claim on the encoder when this call returned +// would hand a re-probe a card that is about to be busy. +// +// Checked the instant the call returns, which is the only instant that matters: +// that is when the caller's own claim goes away. +func TestHWProbeClaimIsHeldBeforeAnAbandonedCallerReturns(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + device := env.devicePath("renderD128") + + // The flight parks so it cannot finish and decrement before the assertion. + release := make(chan struct{}) + var released sync.Once + hwProbeFlightStarted = func() { <-release } + t.Cleanup(func() { + hwProbeFlightStarted = nil + released.Do(func() { close(release) }) + }) + + awaitNoProbesInFlight(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if ok, _ := ffmpegSupportsBackendContext(ctx, transcodeHWVAAPI, ffmpeg.path, device); ok { + t.Fatal("an already-canceled probe reported success") + } + + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d the moment the caller returned, want at least 1: "+ + "the detached flight was unaccounted for", got) + } + + released.Do(func() { close(release) }) + // The claim comes back down once the flight lands, with no caller left to + // receive it. + awaitNoProbesInFlight(t) +} + +// awaitNoProbesInFlight waits for every claim on the encoder to be released, +// including ones detached from a caller that has already returned. Waiting on +// the counter itself rather than on a fixed delay is what keeps these tests from +// depending on how loaded the machine is. +func awaitNoProbesInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if got := HWProbesInFlight(); got == 0 { + return + } else if time.Now().After(deadline) { + t.Fatalf("HWProbesInFlight() = %d, want every detached flight released", got) + } + runtime.Gosched() + } +} + +// Clearing the VideoToolbox cache does not supersede a probe already running: +// that call stays registered under its key, so a rebuild joins it rather than +// starting a cold one, and its completion repopulates the map — the re-probe +// then publishes the verdict it was asked to discard. The generation in the key +// is what moves the rebuild onto a fresh flight. +func TestInvalidateHWProbeCacheSupersedesAnInFlightVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + + // The first flight parks inside the probe and stays there while the second + // call is made, so "did the second start its own flight" is decided by + // channel receipts rather than by how the two happen to be scheduled. + var starts atomic.Int32 + blocked := make(chan struct{}) + firstStarted := make(chan struct{}) + previous := videoToolboxProbeStarted + videoToolboxProbeStarted = func() { + if starts.Add(1) == 1 { + close(firstStarted) + <-blocked + } + } + t.Cleanup(func() { videoToolboxProbeStarted = previous }) + + var wg sync.WaitGroup + wg.Go(func() { + if result := cachedVideoToolboxProbe(ffmpeg.path); !result.available { + t.Errorf("in-flight probe failed: %s", result.reason) + } + }) + <-firstStarted + + InvalidateHWProbeCache() + + second := make(chan hardwareProbeResult, 1) + wg.Go(func() { second <- cachedVideoToolboxProbe(ffmpeg.path) }) + + // The second call must reach its own flight while the first is still parked. + deadline := time.Now().Add(5 * time.Second) + for starts.Load() < 2 { + if time.Now().After(deadline) { + t.Fatal("the re-probe joined the flight it was supposed to supersede") + } + runtime.Gosched() + } + + close(blocked) + if result := <-second; !result.available { + t.Fatalf("post-invalidation probe failed: %s", result.reason) + } + wg.Wait() +} + +// A capable Mac was publishing resolved:"none" with no detected backends: the +// snapshot builder only walked backends on Linux, so the VideoToolbox probe +// that resolution uses never ran here. The API stored that as the node's +// durable inventory — software-only, planned for software tone mapping, with an +// operator re-probe that could not verify otherwise. +func TestDetectHWAccelPublishesVideoToolboxOnDarwin(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != transcodeHWVideoToolbox { + t.Fatalf("Resolved = %q, want videotoolbox", info.Resolved) + } + if len(info.DetectedBackends) != 1 || info.DetectedBackends[0].Backend != transcodeHWVideoToolbox || + !info.DetectedBackends[0].Verified { + t.Fatalf("DetectedBackends = %+v, want a verified videotoolbox entry", info.DetectedBackends) + } +} + +// A Mac whose probe fails publishes the failure rather than silence, so an +// operator can see why — but a probe cut short by the caller's deadline is not +// a verdict about the hardware and must not be hashed as one. +func TestDetectHWAccelReportsAFailedVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + + info, err := DetectHWAccelWithFFmpegContextResult(context.Background(), hwAccelAuto, ffmpeg.path, "") + if err != nil { + t.Fatalf("DetectHWAccelWithFFmpegContextResult: %v", err) + } + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) != 1 || info.DetectedBackends[0].Verified || + info.DetectedBackends[0].Reason == "" { + t.Fatalf("DetectedBackends = %+v, want an unverified entry carrying a reason", info.DetectedBackends) + } + + hung := writeFakeFFmpeg(t, fakeFFmpegProbe{hang: true}) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + t.Cleanup(cancel) + if _, err := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, hung.path, ""); !errors.Is(err, ErrHardwareDetectionIncomplete) { + t.Fatalf("error = %v, want ErrHardwareDetectionIncomplete for a deadline inside the probe", err) + } +} + +// The VideoToolbox flight outlives its caller like the others, so it has to be +// counted like the others — otherwise a re-probe sees an idle encoder and a +// new-generation probe starts beside the one still running. +func TestHWProbesInFlightCountsADetachedVideoToolboxProbe(t *testing.T) { + setupHWAccelTest(t) + currentGOOS = darwinGOOS + ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) + awaitNoProbesInFlight(t) + + release := make(chan struct{}) + var released, announced sync.Once + started := make(chan struct{}) + previous := videoToolboxProbeStarted + videoToolboxProbeStarted = func() { + announced.Do(func() { close(started) }) + <-release + } + t.Cleanup(func() { + videoToolboxProbeStarted = previous + released.Do(func() { close(release) }) + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + _ = cachedVideoToolboxProbeContext(ctx, ffmpeg.path) + }() + <-started + cancel() + <-done + + if got := HWProbesInFlight(); got < 1 { + t.Fatalf("HWProbesInFlight() = %d with a detached VideoToolbox probe running, want at least 1", got) + } + released.Do(func() { close(release) }) + awaitNoProbesInFlight(t) +} + +// A scheduled capability snapshot is how a long-running node notices its +// hardware changing, and on an NVIDIA-only node — /dev/nvidia* and the toolkit, +// no /dev/dri — a card's uuid is the only trace of it in the report. A listing +// cached past the walk that took it would republish a hot-removed card in every +// snapshot until someone re-probed by hand. +func TestCapabilityWalkRequeriesNVIDIAIdentities(t *testing.T) { + previous := nvidiaSMIQuery + answer := "GPU-aaa, 00000000:03:00.0\n" + queries := 0 + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + queries++ + return []byte(answer), nil + } + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAGPUUUIDs() + }) + resetNVIDIAGPUUUIDs() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + first, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, "", "") + if !slices.Contains(first.NVIDIAGPUUUIDs, "GPU-aaa") { + t.Fatalf("first walk reported %v, want the card nvidia-smi named", first.NVIDIAGPUUUIDs) + } + + // The card is hot-removed. Nothing else in this node's report mentions it. + answer = "" + second, _ := DetectHWAccelWithFFmpegContextResult(ctx, hwAccelAuto, "", "") + if slices.Contains(second.NVIDIAGPUUUIDs, "GPU-aaa") { + t.Fatalf("second walk still reported %v for a card that is gone", second.NVIDIAGPUUUIDs) + } + if queries < 2 { + t.Fatalf("nvidia-smi queried %d times across two walks, want one per walk", queries) + } +} diff --git a/internal/playback/gpudetect_test.go b/internal/playback/gpudetect_test.go index 7eecc272c..fa8cfdd6a 100644 --- a/internal/playback/gpudetect_test.go +++ b/internal/playback/gpudetect_test.go @@ -2,9 +2,12 @@ package playback import ( "context" + "encoding/json" "fmt" "os" "path/filepath" + "slices" + "strconv" "strings" "sync" "testing" @@ -21,16 +24,27 @@ type hwAccelTestEnv struct { type fakeFFmpegProbe struct { cuda bool + qsvHWAccel bool + vaapiHWAccel bool h264NVENC bool hevcNVENC bool + h264QSV bool + hevcQSV bool + h264VAAPI bool scaleCUDA bool uploadCUDA bool videotoolbox bool h264VT bool hevcVT bool smokeOK bool - hang bool - delay time.Duration + // smokeFailures names encoders whose smoke encode fails even when smokeOK + // is set, modeling a listed encoder with no working driver behind it. + smokeFailures []string + // smokeDeviceFailures names render devices (by basename) whose smoke encode + // fails, modeling one broken GPU on a host that has another working one. + smokeDeviceFailures []string + hang bool + delay time.Duration } type fakeFFmpegBinary struct { @@ -42,9 +56,9 @@ func TestResolveHWAccelWithFFmpegAutoPrefersNVENCOverIntel(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x8086") env.addRenderDevice(t, "renderD129", "0x10de") - ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) } } @@ -53,9 +67,9 @@ func TestResolveHWAccelWithFFmpegFallsBackToIntelWhenNVENCProbeFails(t *testing. env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x8086") env.addRenderDevice(t, "renderD129", "0x10de") - ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "qsv" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "qsv" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv", got) } } @@ -64,19 +78,208 @@ func TestResolveHWAccelWithFFmpegFallsBackToVAAPIWhenNVENCProbeFails(t *testing. env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") env.addRenderDevice(t, "renderD129", "0x1002") - ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi", got) + } +} + +func TestResolveHWAccelWithFFmpegFallsBackToVAAPIWhenQSVListingFails(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + probe := successfulVAAPIProbe() + probe.qsvHWAccel = true + probe.h264QSV = true + // hevc_qsv is missing, so the QSV listing gate rejects an Intel GPU that + // VAAPI can still drive. + ffmpeg := writeFakeFFmpeg(t, probe) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "vaapi" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi", got) } } +func TestResolveHWAccelWithFFmpegTriesEveryCandidateDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + env.addRenderDevice(t, "renderD129", "0x1002") + probe := successfulVAAPIProbe() + // The GPU that sorts first has no working driver; the second one does. + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi from the working device", got) + } + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + vaapi := info.DetectedBackends[len(info.DetectedBackends)-1] + if vaapi.Backend != "vaapi" || !vaapi.Verified { + t.Fatalf("vaapi entry = %+v, want verified", vaapi) + } + if device := filepath.Base(vaapi.Device); device != "renderD129" { + t.Fatalf("verified device = %q, want renderD129", device) + } +} + +func TestResolveHWAccelWithFFmpegSkipsNVIDIANodesAsVAAPIDevices(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x10de") + env.addRenderDevice(t, "renderD129", "0x1002") + probe := successfulVAAPIProbe() + // An NVIDIA render node has no libva driver; probing it would reject a + // backend the AMD card can drive. + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "vaapi" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want vaapi from the AMD device", got) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), "vaapi=hw:"+filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("VAAPI probe used the NVIDIA render node; log:\n%s", logData) + } +} + +func TestResolveHWAccelWithFFmpegProbesTheConfiguredHWDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + probe := successfulQSVProbe() + probe.smokeDeviceFailures = []string{"renderD128"} + ffmpeg := writeFakeFFmpeg(t, probe) + + // The operator pinned the working GPU, which is the device a transcode + // opens; auto resolution has to verify that one rather than renderD128. + pinned := filepath.Join(env.driDir, "renderD129") + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, pinned); got != "qsv" { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want qsv on the pinned device", got) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("probe touched an unconfigured device; log:\n%s", logData) + } +} + +func TestDetectHWAccelReportsHostInventoryBehindAPinnedDevice(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x1002") + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, filepath.Join(env.driDir, "renderD129")) + if info.Resolved != "vaapi" { + t.Fatalf("Resolved = %q, want vaapi from the pinned AMD device", info.Resolved) + } + // Pinning a device narrows what is probed, never what is reported. + if !info.IntelDetected { + t.Fatal("IntelDetected = false, want the host's Intel GPU still reported") + } + if len(info.RenderDevices) != 2 { + t.Fatalf("RenderDevices = %v, want the full host inventory", info.RenderDevices) + } +} + +// A proxy node reads the cluster-wide hw_device meant for the transcode nodes: +// the paths and their sysfs vendor entries are visible, but the devices cannot +// be opened. Detection must skip the probes entirely — no ffmpeg spawn, no +// alarming driver error — and say why. +func TestDetectHWAccelSkipsConfiguredDevicesItCannotOpen(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + configured := filepath.Join(env.driDir, "renderD128") + "," + filepath.Join(env.driDir, "renderD129") + for _, name := range []string{"renderD128", "renderD129"} { + if err := os.Remove(filepath.Join(env.driDir, name)); err != nil { + t.Fatal(err) + } + } + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, configured) + if info.Resolved != HWAccelNone { + t.Fatalf("Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) == 0 { + t.Fatal("DetectedBackends is empty, want skipped qsv/vaapi entries") + } + for _, backend := range info.DetectedBackends { + if !backend.Skipped { + t.Fatalf("backend %q Skipped = false, want true", backend.Backend) + } + if backend.Verified { + t.Fatalf("backend %q Verified = true, want false", backend.Backend) + } + if !strings.Contains(backend.Reason, "not accessible") { + t.Fatalf("backend %q Reason = %q, want an accessibility reason", backend.Backend, backend.Reason) + } + } + if logData, err := os.ReadFile(ffmpeg.logPath); err == nil && len(strings.TrimSpace(string(logData))) > 0 { + t.Fatalf("ffmpeg was spawned for inaccessible devices; log:\n%s", logData) + } +} + +// One configured device is gone, the other works: the accessible one must +// still be probed and win, and the missing one must not be smoke-encoded. +func TestDetectHWAccelProbesOnlyTheAccessibleConfiguredDevices(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x8086") + if err := os.Remove(filepath.Join(env.driDir, "renderD128")); err != nil { + t.Fatal(err) + } + configured := filepath.Join(env.driDir, "renderD128") + "," + filepath.Join(env.driDir, "renderD129") + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, configured) + if info.Resolved != "qsv" { + t.Fatalf("Resolved = %q, want qsv from the accessible device", info.Resolved) + } + var qsv *DetectedBackend + for i := range info.DetectedBackends { + if info.DetectedBackends[i].Backend == "qsv" { + qsv = &info.DetectedBackends[i] + } + } + if qsv == nil || qsv.Skipped || !qsv.Verified { + t.Fatalf("qsv entry = %+v, want verified and not skipped", qsv) + } + if qsv.Device != filepath.Join(env.driDir, "renderD129") { + t.Fatalf("qsv Device = %q, want the accessible renderD129", qsv.Device) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logData), filepath.Join(env.driDir, "renderD128")) { + t.Fatalf("probe touched the inaccessible device; log:\n%s", logData) + } +} + +func TestResolveHWAccelWithFFmpegReturnsNoneWhenVAAPISmokeEncodeFails(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x1002") + probe := successfulVAAPIProbe() + probe.smokeFailures = []string{"h264_vaapi"} + ffmpeg := writeFakeFFmpeg(t, probe) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != HWAccelNone { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) + } +} + func TestResolveHWAccelWithFFmpegReturnsNoneWhenNVENCProbeFailsWithoutFallback(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "none" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) } } @@ -86,16 +289,140 @@ func TestResolveHWAccelWithFFmpegUsesNVIDIADeviceNodesWithoutDRM(t *testing.T) { env.addNVIDIADevice(t, "nvidia0") ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) } } -func TestExplicitNVENCBypassesFFmpegProbe(t *testing.T) { +func TestResolveHWAccelPassesThroughConfiguredBackends(t *testing.T) { setupHWAccelTest(t) - if got := ResolveHWAccelWithFFmpeg("nvenc", "/does/not/exist/ffmpeg"); got != "nvenc" { - t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want nvenc", got) + for _, configured := range []string{"nvenc", "qsv", "vaapi", "none", "custom"} { + t.Run(configured, func(t *testing.T) { + if got := ResolveHWAccelWithFFmpeg(configured, "/does/not/exist/ffmpeg", ""); got != configured { + t.Fatalf("ResolveHWAccelWithFFmpeg(%q) = %q, want unchanged", configured, got) + } + }) + } +} + +// Windows rather than macOS: macOS has its own hardware path through +// VideoToolbox, so it is no longer a platform with nothing to probe. +func TestResolveHWAccelAutoIsNoneOnAPlatformWithNoHardwarePath(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + currentGOOS = windowsGOOS + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != HWAccelNone { + t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) + } + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + if info.Resolved != HWAccelNone { + t.Fatalf("DetectHWAccelWithFFmpeg().Resolved = %q, want none", info.Resolved) + } + if len(info.DetectedBackends) != 0 { + t.Fatalf("DetectHWAccelWithFFmpeg().DetectedBackends = %+v, want empty off Linux", info.DetectedBackends) + } + if _, err := os.Stat(ffmpeg.logPath); !os.IsNotExist(err) { + t.Fatalf("off-Linux detection ran FFmpeg probes (stat err = %v)", err) + } +} + +func TestDetectHWAccelReportsEveryCandidateBackend(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + env.addRenderDevice(t, "renderD129", "0x10de") + probe := fullyCapableProbe() + probe.h264NVENC = false + probe.smokeFailures = []string{"h264_vaapi"} + ffmpeg := writeFakeFFmpeg(t, probe) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + if info.Resolved != "qsv" { + t.Fatalf("Resolved = %q, want qsv", info.Resolved) + } + if len(info.DetectedBackends) != 3 { + t.Fatalf("DetectedBackends = %+v, want one entry per candidate backend", info.DetectedBackends) + } + // The NVIDIA render node carries no libva driver, so it is not a VAAPI + // candidate even though it is a render device. + want := []DetectedBackend{ + {Backend: "nvenc", Verified: false, Devices: []string{"/dev/dri/renderD129"}}, + {Backend: "qsv", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "vaapi", Verified: false, Devices: []string{"/dev/dri/renderD128"}}, + } + for i, expected := range want { + got := info.DetectedBackends[i] + if got.Backend != expected.Backend || got.Verified != expected.Verified { + t.Fatalf("DetectedBackends[%d] = %+v, want backend %q verified=%v", i, got, expected.Backend, expected.Verified) + } + if !slices.Equal(stripDevicePrefix(got.Devices), stripDevicePrefix(expected.Devices)) { + t.Fatalf("DetectedBackends[%d].Devices = %v, want %v", i, got.Devices, expected.Devices) + } + if expected.Verified && got.Reason != "" { + t.Fatalf("DetectedBackends[%d].Reason = %q, want empty for a verified backend", i, got.Reason) + } + if !expected.Verified && got.Reason == "" { + t.Fatalf("DetectedBackends[%d].Reason is empty, want a failure explanation", i) + } + } + if device := filepath.Base(info.DetectedBackends[1].Device); device != "renderD128" { + t.Fatalf("qsv verified device = %q, want the Intel render node", device) + } + if reason := info.DetectedBackends[0].Reason; reason != "h264_nvenc encoder unavailable" { + t.Fatalf("nvenc reason = %q, want the missing encoder", reason) + } + if reason := info.DetectedBackends[2].Reason; !strings.HasPrefix(reason, "h264_vaapi smoke encode failed") { + t.Fatalf("vaapi reason = %q, want the failed smoke encode", reason) + } +} + +func TestDetectHWAccelOmitsBackendsWithoutCandidateHardware(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + backends := make([]string, 0, len(info.DetectedBackends)) + for _, entry := range info.DetectedBackends { + backends = append(backends, entry.Backend) + } + if !slices.Equal(backends, []string{"qsv", "vaapi"}) { + t.Fatalf("detected backends = %v, want qsv and vaapi only", backends) + } + if !info.IntelDetected { + t.Fatal("IntelDetected = false, want true") + } +} + +func TestDetectedBackendJSONShape(t *testing.T) { + encoded, err := json.Marshal(HWAccelInfo{ + Resolved: "qsv", + DetectedBackends: []DetectedBackend{ + {Backend: "qsv", Verified: true, Devices: []string{"/dev/dri/renderD128"}}, + {Backend: "nvenc", Verified: false, Reason: "h264_nvenc encoder unavailable"}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"detected_backends":[`, + `{"backend":"qsv","verified":true,"devices":["/dev/dri/renderD128"]}`, + `{"backend":"nvenc","verified":false,"reason":"h264_nvenc encoder unavailable"}`, + } { + if !strings.Contains(string(encoded), want) { + t.Fatalf("HWAccelInfo JSON = %s, missing %s", encoded, want) + } + } + + empty, err := json.Marshal(HWAccelInfo{Resolved: HWAccelNone}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(empty), "detected_backends") { + t.Fatalf("HWAccelInfo JSON = %s, want detected_backends omitted when empty", empty) } } @@ -112,7 +439,7 @@ func TestResolveHWAccelWithFFmpegContextHonorsCallerDeadline(t *testing.T) { // below distinguishes it from. ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) started := time.Now() - got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path) + got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path, "") cancel() if got != HWAccelNone { t.Fatalf("ResolveHWAccelWithFFmpegContext() = %q, want none", got) @@ -125,7 +452,7 @@ func TestResolveHWAccelWithFFmpegContextHonorsCallerDeadline(t *testing.T) { } retryCtx, retryCancel := context.WithTimeout(context.Background(), 60*time.Millisecond) - _ = ResolveHWAccelWithFFmpegContext(retryCtx, "auto", ffmpeg.path) + _ = ResolveHWAccelWithFFmpegContext(retryCtx, "auto", ffmpeg.path, "") retryCancel() logData, err := os.ReadFile(ffmpeg.logPath) if err != nil { @@ -147,7 +474,14 @@ func TestNormalizeProbeRequestTimeout(t *testing.T) { {name: "negative uses caller fallback", millis: -1, fallback: 2 * time.Minute, want: 2 * time.Minute}, {name: "too small", millis: time.Second.Milliseconds(), fallback: 2 * time.Minute, want: 5 * time.Second}, {name: "advertised", millis: (137 * time.Second).Milliseconds(), fallback: 2 * time.Minute, want: 137 * time.Second}, - {name: "too large", millis: (10 * time.Minute).Milliseconds(), fallback: 2 * time.Minute, want: 5 * time.Minute}, + { + // The ceiling is derived from the probe formula rather than picked, + // so the assertion is too. + name: "too large", + millis: (24 * time.Hour).Milliseconds(), + fallback: 2 * time.Minute, + want: MaxCapabilityRequestTimeout(), + }, } { t.Run(test.name, func(t *testing.T) { if got := NormalizeProbeRequestTimeout(test.millis, test.fallback); got != test.want { @@ -258,24 +592,117 @@ func TestFFmpegSupportsNVENCRequiresCUDAEncodersFiltersAndSmoke(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - resetNVENCProbeCacheForTest() + resetHWProbeCacheForTest() ffmpeg := writeFakeFFmpeg(t, tt.probe) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); ok { - t.Fatalf("ffmpegSupportsNVENC() = true, want false") + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); ok { + t.Fatalf("ffmpegSupportsBackend(nvenc) = true, want false") } else if reason == "" { - t.Fatalf("ffmpegSupportsNVENC() reason is empty") + t.Fatalf("ffmpegSupportsBackend(nvenc) reason is empty") } }) } } +func TestFFmpegSupportsQSVRequiresListingsAndSmoke(t *testing.T) { + setupHWAccelTest(t) + tests := []struct { + name string + probe fakeFFmpegProbe + want string + }{ + { + name: "missing qsv and vaapi hwaccels", + probe: fakeFFmpegProbe{h264QSV: true, hevcQSV: true, smokeOK: true}, + want: "qsv and vaapi hwaccels unavailable", + }, + { + name: "missing h264 qsv encoder", + probe: fakeFFmpegProbe{qsvHWAccel: true, hevcQSV: true, smokeOK: true}, + want: "h264_qsv encoder unavailable", + }, + { + name: "missing hevc qsv encoder", + probe: fakeFFmpegProbe{qsvHWAccel: true, h264QSV: true, smokeOK: true}, + want: "hevc_qsv encoder unavailable", + }, + { + name: "smoke encode failure", + probe: fakeFFmpegProbe{vaapiHWAccel: true, h264QSV: true, hevcQSV: true}, + want: "h264_qsv smoke encode failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, tt.probe) + ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, "/dev/dri/renderD128") + if ok { + t.Fatal("ffmpegSupportsBackend(qsv) = true, want false") + } + if !strings.HasPrefix(reason, tt.want) { + t.Fatalf("reason = %q, want prefix %q", reason, tt.want) + } + }) + } + + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, successfulQSVProbe()) + if ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("ffmpegSupportsBackend(qsv) = false, want true (reason=%q)", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + logText := string(logData) + for _, want := range []string{ + "vaapi=va:/dev/dri/renderD128,driver=iHD,kernel_driver=i915,vendor_id=0x8086", + "qsv=qs@va", + "testsrc2=size=640x360:rate=1", + } { + if !strings.Contains(logText, want) { + t.Fatalf("QSV smoke command missing %q; log:\n%s", want, logText) + } + } +} + +func TestFFmpegSupportsVAAPIRequiresEncoderAndSmoke(t *testing.T) { + setupHWAccelTest(t) + + resetHWProbeCacheForTest() + missing := writeFakeFFmpeg(t, fakeFFmpegProbe{vaapiHWAccel: true, smokeOK: true}) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, missing.path, "/dev/dri/renderD128"); ok { + t.Fatal("ffmpegSupportsBackend(vaapi) = true, want false") + } else if reason != "h264_vaapi encoder unavailable" { + t.Fatalf("reason = %q, want the missing encoder", reason) + } + + resetHWProbeCacheForTest() + ffmpeg := writeFakeFFmpeg(t, successfulVAAPIProbe()) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("ffmpegSupportsBackend(vaapi) = false, want true (reason=%q)", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + logText := string(logData) + if !strings.Contains(logText, "vaapi=hw:/dev/dri/renderD128") { + t.Fatalf("VAAPI smoke command missing its init chain; log:\n%s", logText) + } + if strings.Count(logText, "\n") != 2 { + t.Fatalf("VAAPI probe ran %d commands, want an encoders listing and one smoke encode; log:\n%s", + strings.Count(logText, "\n"), logText) + } +} + func TestFFmpegSupportsNVENCCachesByFFmpegPath(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) for i := 0; i < 2; i++ { - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "nvenc" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "nvenc" { t.Fatalf("ResolveHWAccelWithFFmpeg() call %d = %q, want nvenc", i+1, got) } } @@ -289,6 +716,50 @@ func TestFFmpegSupportsNVENCCachesByFFmpegPath(t *testing.T) { } } +func TestHWProbeCacheSeparatesBackendsAndDevices(t *testing.T) { + setupHWAccelTest(t) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + keys := map[string]string{ + "nvenc": hwProbeCacheKey(0, ffmpeg.path, transcodeHWNVENC, ""), + "qsv-128": hwProbeCacheKey(0, ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD128"), + "qsv-129": hwProbeCacheKey(0, ffmpeg.path, transcodeHWQSV, "/dev/dri/renderD129"), + "vaapi-128": hwProbeCacheKey(0, ffmpeg.path, transcodeHWVAAPI, "/dev/dri/renderD128"), + "identity-eq": hwProbeCacheKey(0, ffmpeg.path, transcodeHWNVENC, ""), + } + if keys["nvenc"] != keys["identity-eq"] { + t.Fatal("identical backend and device produced different cache keys") + } + seen := map[string]string{} + for name, key := range keys { + if name == "identity-eq" { + continue + } + if other, ok := seen[key]; ok { + t.Fatalf("cache keys for %s and %s collided", name, other) + } + seen[key] = name + } + + // Each distinct key runs its own probe command set: 3 for QSV on two + // devices, 2 for VAAPI. + for _, device := range []string{"/dev/dri/renderD128", "/dev/dri/renderD129"} { + if ok, reason := ffmpegSupportsBackend(transcodeHWQSV, ffmpeg.path, device); !ok { + t.Fatalf("QSV probe on %s failed: %s", device, reason) + } + } + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpeg.path, "/dev/dri/renderD128"); !ok { + t.Fatalf("VAAPI probe failed: %s", reason) + } + logData, err := os.ReadFile(ffmpeg.logPath) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(string(logData), "\n"); got != 8 { + t.Fatalf("probe command count = %d, want 8 across three distinct cache keys; log:\n%s", got, logData) + } +} + func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { env := setupHWAccelTest(t) env.addRenderDevice(t, "renderD128", "0x10de") @@ -304,7 +775,7 @@ func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { go func() { defer wg.Done() <-start - results <- ResolveHWAccelWithFFmpeg("auto", ffmpeg.path) + results <- ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, "") }() } close(start) @@ -328,7 +799,7 @@ func TestFFmpegSupportsNVENCCoalescesConcurrentColdProbes(t *testing.T) { func TestFFmpegSupportsNVENCInvalidatesWhenBinaryChangesInPlace(t *testing.T) { setupHWAccelTest(t) ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); !ok { + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); !ok { t.Fatalf("initial NVENC probe failed: %s", reason) } @@ -340,12 +811,12 @@ func TestFFmpegSupportsNVENCInvalidatesWhenBinaryChangesInPlace(t *testing.T) { if err := os.Chtimes(ffmpeg.path, changedAt, changedAt); err != nil { t.Fatalf("advance replacement timestamp: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpeg.path); ok { + if ok, _ := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); ok { t.Fatal("replaced FFmpeg binary reused a stale positive NVENC result") } } -func TestNVENCProbeCacheKeyIncludesResolvedPATHIdentity(t *testing.T) { +func TestFFmpegIdentityKeyIncludesResolvedPATHIdentity(t *testing.T) { firstDir := t.TempDir() secondDir := t.TempDir() stamp := time.Unix(100, 0) @@ -360,29 +831,26 @@ func TestNVENCProbeCacheKeyIncludesResolvedPATHIdentity(t *testing.T) { } t.Setenv("PATH", firstDir) - firstKey := nvencProbeCacheKey("ffmpeg") + firstKey := ffmpegIdentityKey("ffmpeg") t.Setenv("PATH", secondDir) - secondKey := nvencProbeCacheKey("ffmpeg") + secondKey := ffmpegIdentityKey("ffmpeg") if firstKey == secondKey { t.Fatalf("PATH-resolved FFmpeg identities collided: %q", firstKey) } } -func TestFFmpegSupportsNVENCNegativeResultExpires(t *testing.T) { +func TestHWProbeNegativeResultExpires(t *testing.T) { setupHWAccelTest(t) - oldTTL := nvencProbeNegativeTTL - nvencProbeNegativeTTL = 20 * time.Millisecond - t.Cleanup(func() { nvencProbeNegativeTTL = oldTTL }) + clock := time.Now() + hwProbeNow = func() time.Time { return clock } dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - markerPath := filepath.Join(dir, "nvenc-ready") + markerPath := filepath.Join(dir, "vaapi-ready") script := fmt.Sprintf(`#!/bin/sh case "$*" in - *-hwaccels*) echo cuda; exit 0 ;; - *-encoders*) echo 'h264_nvenc hevc_nvenc'; exit 0 ;; - *-filters*) echo 'scale_cuda hwupload_cuda'; exit 0 ;; + *-encoders*) echo 'h264_vaapi'; exit 0 ;; *) test -e %q ;; esac `, markerPath) @@ -390,27 +858,37 @@ esac t.Fatalf("write marker-controlled FFmpeg: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpegPath); ok { - t.Fatal("initial NVENC probe unexpectedly succeeded") + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); ok { + t.Fatal("initial VAAPI probe unexpectedly succeeded") } if err := os.WriteFile(markerPath, []byte("ready"), 0o600); err != nil { - t.Fatalf("enable NVENC smoke probe: %v", err) + t.Fatalf("enable VAAPI smoke probe: %v", err) } - if ok, _ := ffmpegSupportsNVENC(ffmpegPath); ok { - t.Fatal("negative result was not retained during its short TTL") + clock = clock.Add(hwProbeNegativeTTL - time.Millisecond) + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); ok { + t.Fatal("negative result was not retained during its TTL") } - time.Sleep(2 * nvencProbeNegativeTTL) - if ok, reason := ffmpegSupportsNVENC(ffmpegPath); !ok { + clock = clock.Add(2 * time.Millisecond) + if ok, reason := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); !ok { t.Fatalf("expired negative result was not retried: %s", reason) } + // A positive result is kept for the process lifetime, so removing the + // marker after success must not change the answer. + if err := os.Remove(markerPath); err != nil { + t.Fatal(err) + } + clock = clock.Add(time.Hour) + if ok, _ := ffmpegSupportsBackend(transcodeHWVAAPI, ffmpegPath, "/dev/dri/renderD128"); !ok { + t.Fatal("positive probe result expired, want process-lifetime caching") + } } func TestFFmpegSupportsNVENCSmokeProbeUsesSafeFrameDimensions(t *testing.T) { setupHWAccelTest(t) ffmpeg := writeFakeFFmpeg(t, successfulNVENCProbe()) - if ok, reason := ffmpegSupportsNVENC(ffmpeg.path); !ok { - t.Fatalf("ffmpegSupportsNVENC() = false, want true (reason=%q)", reason) + if ok, reason := ffmpegSupportsBackend(transcodeHWNVENC, ffmpeg.path, ""); !ok { + t.Fatalf("ffmpegSupportsBackend(nvenc) = false, want true (reason=%q)", reason) } logData, err := os.ReadFile(ffmpeg.logPath) @@ -434,6 +912,49 @@ func successfulNVENCProbe() fakeFFmpegProbe { } } +func successfulQSVProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + qsvHWAccel: true, + h264QSV: true, + hevcQSV: true, + smokeOK: true, + } +} + +func successfulVAAPIProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + vaapiHWAccel: true, + h264VAAPI: true, + smokeOK: true, + } +} + +func fullyCapableProbe() fakeFFmpegProbe { + return fakeFFmpegProbe{ + cuda: true, + qsvHWAccel: true, + vaapiHWAccel: true, + h264NVENC: true, + hevcNVENC: true, + h264QSV: true, + hevcQSV: true, + h264VAAPI: true, + scaleCUDA: true, + uploadCUDA: true, + smokeOK: true, + } +} + +// stripDevicePrefix compares device lists by basename so expectations stay +// readable against the test's temporary /dev/dri stand-in. +func stripDevicePrefix(devices []string) []string { + names := make([]string, 0, len(devices)) + for _, device := range devices { + names = append(names, filepath.Base(device)) + } + return names +} + func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { t.Helper() @@ -442,8 +963,9 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { oldNVIDIADeviceGlob := defaultNVIDIADeviceGlob oldSysClassDRMDir := sysClassDRMDir oldGOOS := currentGOOS - oldProbeTimeout := nvencProbeCommandTimeout - resetNVENCProbeCacheForTest() + oldProbeTimeout := hwProbeCommandTimeout + oldProbeNow := hwProbeNow + resetHWProbeCacheForTest() tmp := t.TempDir() env := &hwAccelTestEnv{ @@ -456,7 +978,7 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { defaultNVIDIADeviceGlob = filepath.Join(env.devDir, "nvidia[0-9]*") sysClassDRMDir = env.sysDir currentGOOS = "linux" - nvencProbeCommandTimeout = 200 * time.Millisecond + hwProbeCommandTimeout = 200 * time.Millisecond if err := os.MkdirAll(env.driDir, 0o755); err != nil { t.Fatalf("create test dri dir: %v", err) @@ -471,8 +993,9 @@ func setupHWAccelTest(t *testing.T) *hwAccelTestEnv { defaultNVIDIADeviceGlob = oldNVIDIADeviceGlob sysClassDRMDir = oldSysClassDRMDir currentGOOS = oldGOOS - nvencProbeCommandTimeout = oldProbeTimeout - resetNVENCProbeCacheForTest() + hwProbeCommandTimeout = oldProbeTimeout + hwProbeNow = oldProbeNow + resetHWProbeCacheForTest() }) return env @@ -525,6 +1048,12 @@ func writeFakeFFmpegScript(t *testing.T, path, logPath string, probe fakeFFmpegP if probe.cuda { script += " echo 'cuda'\n" } + if probe.qsvHWAccel { + script += " echo 'qsv'\n" + } + if probe.vaapiHWAccel { + script += " echo 'vaapi'\n" + } if probe.videotoolbox { script += " echo 'videotoolbox'\n" } @@ -536,6 +1065,15 @@ func writeFakeFFmpegScript(t *testing.T, path, logPath string, probe fakeFFmpegP if probe.hevcNVENC { script += " echo ' V..... hevc_nvenc NVIDIA NVENC hevc encoder'\n" } + if probe.h264QSV { + script += " echo ' V..... h264_qsv H.264 QSV encoder'\n" + } + if probe.hevcQSV { + script += " echo ' V..... hevc_qsv HEVC QSV encoder'\n" + } + if probe.h264VAAPI { + script += " echo ' V..... h264_vaapi H.264 VAAPI encoder'\n" + } if probe.h264VT { script += " echo ' V..... h264_videotoolbox VideoToolbox H.264 encoder'\n" } @@ -551,13 +1089,31 @@ func writeFakeFFmpegScript(t *testing.T, path, logPath string, probe fakeFFmpegP script += " echo ' ... hwupload_cuda V->V upload CUDA frames'\n" } script += " exit 0 ;;\n" - script += " *)\n" - if probe.smokeOK { - script += " exit 0 ;;\n" - } else { - script += " echo 'no capable devices found' >&2\n" - script += " exit 1 ;;\n" + for _, encoder := range []string{"h264_nvenc", "h264_qsv", "h264_vaapi", "h264_videotoolbox", "hevc_videotoolbox"} { + script += fmt.Sprintf(" *%s*)\n", encoder) + if probe.smokeOK && !slices.Contains(probe.smokeFailures, encoder) { + // The init chain carries the device path, so a broken GPU is modeled + // by matching the command rather than by ignoring the argument. + if len(probe.smokeDeviceFailures) > 0 { + patterns := make([]string, 0, len(probe.smokeDeviceFailures)) + for _, device := range probe.smokeDeviceFailures { + patterns = append(patterns, "*"+device+"*") + } + script += " case \"$*\" in\n" + script += fmt.Sprintf(" %s)\n", strings.Join(patterns, "|")) + script += fmt.Sprintf(" echo 'no capable devices found for %s' >&2\n", encoder) + script += " exit 1 ;;\n" + script += " esac\n" + } + script += " exit 0 ;;\n" + } else { + script += fmt.Sprintf(" echo 'no capable devices found for %s' >&2\n", encoder) + script += " exit 1 ;;\n" + } } + script += " *)\n" + script += " echo 'unexpected probe command' >&2\n" + script += " exit 1 ;;\n" script += "esac\n" if err := os.WriteFile(path, []byte(script), 0o755); err != nil { @@ -565,13 +1121,11 @@ func writeFakeFFmpegScript(t *testing.T, path, logPath string, probe fakeFFmpegP } } -func resetNVENCProbeCacheForTest() { - nvencProbeCache.Lock() - nvencProbeCache.byPath = make(map[string]nvencProbeCacheEntry) - nvencProbeCache.Unlock() - videoToolboxProbes.Lock() - videoToolboxProbes.byPath = make(map[string]videoToolboxProbeEntry) - videoToolboxProbes.Unlock() +// resetHWProbeCacheForTest clears the probe cache between cases. It delegates +// to the exported invalidation so tests exercise the same seam the re-probe +// action uses rather than a second, drifting implementation. +func resetHWProbeCacheForTest() { + InvalidateHWProbeCache() } func successfulVideoToolboxProbe() fakeFFmpegProbe { @@ -583,7 +1137,7 @@ func TestResolveHWAccelWithFFmpegDarwinUsesVideoToolbox(t *testing.T) { currentGOOS = "darwin" ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "videotoolbox" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "videotoolbox" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got) } } @@ -596,7 +1150,7 @@ func TestResolveHWAccelWithFFmpegContextDarwinHonorsCallerDeadline(t *testing.T) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) defer cancel() started := time.Now() - if got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path); got != HWAccelNone { + if got := ResolveHWAccelWithFFmpegContext(ctx, "auto", ffmpeg.path, ""); got != HWAccelNone { t.Fatalf("ResolveHWAccelWithFFmpegContext() = %q, want none", got) } if elapsed := time.Since(started); elapsed >= 150*time.Millisecond { @@ -614,7 +1168,7 @@ func TestResolveHWAccelWithFFmpegDarwinFallsBackToNoneWhenProbeFails(t *testing. currentGOOS = "darwin" ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{}) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "none" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got) } } @@ -624,7 +1178,7 @@ func TestResolveHWAccelWithFFmpegDarwinAllowsH264OnlyVideoToolbox(t *testing.T) currentGOOS = "darwin" ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{videotoolbox: true, h264VT: true, smokeOK: true}) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "videotoolbox" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "videotoolbox" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox for H.264-only Mac", got) } if ok, _ := videoToolboxSupportsTargetCodec(ffmpeg.path, "hevc"); ok { @@ -637,7 +1191,7 @@ func TestResolveHWAccelWithFFmpegDarwinFallsBackToNoneWhenSmokeEncodeFails(t *te currentGOOS = "darwin" ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{videotoolbox: true, h264VT: true, hevcVT: true}) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "none" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none when smoke encode fails", got) } } @@ -647,7 +1201,7 @@ func TestVideoToolboxProbeSmokesBothEncodersInPortableBitrateMode(t *testing.T) currentGOOS = "darwin" ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe()) - if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "videotoolbox" { + if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path, ""); got != "videotoolbox" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got) } log, err := os.ReadFile(ffmpeg.logPath) @@ -669,7 +1223,7 @@ func TestVideoToolboxProbeSmokesBothEncodersInPortableBitrateMode(t *testing.T) func TestExplicitVideoToolboxBypassesFFmpegProbe(t *testing.T) { setupHWAccelTest(t) - if got := ResolveHWAccelWithFFmpeg("videotoolbox", "/does/not/exist/ffmpeg"); got != "videotoolbox" { + if got := ResolveHWAccelWithFFmpeg("videotoolbox", "/does/not/exist/ffmpeg", ""); got != "videotoolbox" { t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got) } } @@ -778,3 +1332,109 @@ func TestCachedVideoToolboxProbeInvalidatesWhenExecutableIsReplaced(t *testing.T t.Fatal("replacement binary reused the previous executable's positive verdict") } } + +// The walk deadline has to cover the matrix it will actually run, which grows +// with the device set: every configured render device is probed for both QSV +// and VAAPI. A fixed thirty seconds marked a three-device host incomplete while +// every individual command was still inside its own budget, and +// /hw-capabilities then answered 503 for a node that was working. +func TestHWAccelWalkTimeoutScalesWithTheDeviceSet(t *testing.T) { + one := hwCandidates{ + intel: []string{"/dev/dri/renderD128"}, + vaapi: []string{"/dev/dri/renderD128"}, + } + three := hwCandidates{ + intel: []string{"/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/dri/renderD130"}, + vaapi: []string{"/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/dri/renderD130"}, + } + + oneDevice, threeDevices := hwAccelWalkTimeout(one), hwAccelWalkTimeout(three) + if threeDevices <= oneDevice { + t.Fatalf("three-device walk %v is not longer than the one-device %v", threeDevices, oneDevice) + } + + // It covers what it will run: every command the matrix allows, at its own + // per-command bound, plus spawn slack. + wantThree := time.Duration(3*(3+2))*hwProbeCommandTimeout + hwAccelWalkSlack + if threeDevices != wantThree { + t.Fatalf("three-device walk = %v, want %v", threeDevices, wantThree) + } + if threeDevices <= 30*time.Second { + t.Fatalf("three-device walk = %v, which the old fixed 30s bound would have cut short", threeDevices) + } + + // A host with nothing to probe still gets a usable, non-zero deadline. + if got := hwAccelWalkTimeout(hwCandidates{}); got <= 0 { + t.Fatalf("empty walk timeout = %v, want a positive bound", got) + } +} + +// The walk is capped at the same device ceiling the tone-map matrix is, because +// the budget every caller allows is derived from that ceiling. Probing past it +// would guarantee the capability request is canceled before the walk finishes. +func TestCollectHWCandidatesCapsTheProbedDeviceSet(t *testing.T) { + env := setupHWAccelTest(t) + configured := make([]string, 0, tonemap.MaxProbedDevices+3) + for i := range tonemap.MaxProbedDevices + 3 { + name := "renderD" + strconv.Itoa(128+i) + env.addRenderDevice(t, name, "0x8086") + configured = append(configured, env.devicePath(name)) + } + + candidates := collectHWCandidates(strings.Join(configured, ",")) + if got := len(candidates.probeDevicesFor(transcodeHWQSV)); got != tonemap.MaxProbedDevices { + t.Fatalf("qsv probe devices = %d, want the %d cap", got, tonemap.MaxProbedDevices) + } + + // The walk therefore stays inside the budget its callers allow, which is the + // property the cap exists for. + if walk, ceiling := hwAccelWalkTimeout(candidates), MaxCapabilityRequestTimeout(); walk >= ceiling { + t.Fatalf("walk budget %v is not below the %v callers allow", walk, ceiling) + } + + // Every configured device is still reported, capped or not: the inventory is + // what an operator reads, and truncating it would hide hardware that exists. + if got := len(candidates.devicesFor(transcodeHWQSV)); got != tonemap.MaxProbedDevices { + t.Fatalf("qsv reported devices = %d, want the probed set", got) + } + if got := len(candidates.renderDevices); got != len(configured) { + t.Fatalf("render devices = %d, want all %d enumerated", got, len(configured)) + } + + // A set inside the cap is untouched. + small := configured[:3] + if got := len(collectHWCandidates(strings.Join(small, ",")).probeDevicesFor(transcodeHWQSV)); got != 3 { + t.Fatalf("qsv probe devices = %d, want the 3 configured", got) + } +} + +// The ceiling is clamped against on an API replica, which does not have the +// remote node's cards. Pricing a synthetic device list there classified the +// fabricated paths as VAAPI-only — no sysfs vendor to read — so the ceiling came +// out below what a node with a dozen Intel devices legitimately advertises, and +// the clamp then canceled that node before its own matrix could finish. +func TestMaxCapabilityRequestTimeoutDoesNotDependOnTheLocalHost(t *testing.T) { + env := setupHWAccelTest(t) + bare := MaxCapabilityRequestTimeout() + + // Same process, now with Intel cards present: classification would change if + // the ceiling consulted sysfs at all. + for i := range 4 { + env.addRenderDevice(t, "renderD"+strconv.Itoa(128+i), "0x8086") + } + if got := MaxCapabilityRequestTimeout(); got != bare { + t.Fatalf("ceiling moved from %v to %v when local hardware appeared", bare, got) + } + + // And it covers the largest matrix the cap allows: a full set of Intel + // devices, which is the classification that draws two backends per device. + devices := make([]string, 0, tonemap.MaxProbedDevices) + for i := range tonemap.MaxProbedDevices { + name := "renderD" + strconv.Itoa(200+i) + env.addRenderDevice(t, name, "0x8086") + devices = append(devices, env.devicePath(name)) + } + if advertised := CapabilityRequestTimeout(hwAccelAuto, strings.Join(devices, ",")); advertised > bare { + t.Fatalf("a full Intel set advertises %v, above the %v ceiling that clamps it", advertised, bare) + } +} diff --git a/internal/playback/gpuidentity_test.go b/internal/playback/gpuidentity_test.go new file mode 100644 index 000000000..d0437ee45 --- /dev/null +++ b/internal/playback/gpuidentity_test.go @@ -0,0 +1,183 @@ +package playback + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +// addPCIRenderDevice models the sysfs layout a real render node has: the drm +// entry's "device" is a symlink into the PCI tree, and the vendor id lives on +// the far side of it. +func (e *hwAccelTestEnv) addPCIRenderDevice(t *testing.T, name, vendor, pciAddress string) { + t.Helper() + if err := os.WriteFile(filepath.Join(e.driDir, name), []byte{}, 0o600); err != nil { + t.Fatalf("create render device: %v", err) + } + pciDir := filepath.Join(e.sysDir, "pci", pciAddress) + if err := os.MkdirAll(pciDir, 0o755); err != nil { + t.Fatalf("create pci dir: %v", err) + } + if err := os.WriteFile(filepath.Join(pciDir, "vendor"), []byte(vendor+"\n"), 0o644); err != nil { + t.Fatalf("write vendor file: %v", err) + } + drmDir := filepath.Join(e.sysDir, name) + if err := os.MkdirAll(drmDir, 0o755); err != nil { + t.Fatalf("create drm dir: %v", err) + } + if err := os.Symlink(pciDir, filepath.Join(drmDir, "device")); err != nil { + t.Fatalf("link drm device: %v", err) + } +} + +func (e *hwAccelTestEnv) setBootID(t *testing.T, bootID string) { + t.Helper() + path := filepath.Join(t.TempDir(), "boot_id") + if err := os.WriteFile(path, []byte(bootID+"\n"), 0o600); err != nil { + t.Fatalf("write boot id: %v", err) + } + previous := procBootIDPath + procBootIDPath = path + t.Cleanup(func() { procBootIDPath = previous }) +} + +// stubNVIDIASMI replaces the nvidia-smi invocation and clears its process-wide +// cache, which is otherwise computed once and would leak between tests. +func stubNVIDIASMI(t *testing.T, output string, err error) { + t.Helper() + previous := nvidiaSMIQuery + nvidiaSMIQuery = func(context.Context) ([]byte, error) { + if err != nil { + return nil, err + } + return []byte(output), nil + } + resetNVIDIAUUIDCacheForTest() + t.Cleanup(func() { + nvidiaSMIQuery = previous + resetNVIDIAUUIDCacheForTest() + }) +} + +func resetNVIDIAUUIDCacheForTest() { + resetNVIDIAGPUUUIDs() +} + +func renderDeviceDetail(t *testing.T, info HWAccelInfo, path string) RenderDeviceInfo { + t.Helper() + for _, detail := range info.RenderDeviceDetails { + if detail.Path == path { + return detail + } + } + t.Fatalf("no render device detail for %s in %+v", path, info.RenderDeviceDetails) + return RenderDeviceInfo{} +} + +// The device paths under /dev/dri are assigned by enumeration order and move +// when hardware is added or removed. PCI address, GPU uuid and boot id are what +// let an operator (and the node inventory) say the GPU behind a path is still +// the same GPU. +func TestDetectHWAccelReportsHardwareIdentity(t *testing.T) { + env := setupHWAccelTest(t) + env.addPCIRenderDevice(t, "renderD128", "0x10de", "0000:03:00.0") + env.addPCIRenderDevice(t, "renderD129", "0x8086", "0000:04:00.0") + env.setBootID(t, "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111") + // nvidia-smi prints a 32-bit PCI domain in upper case; sysfs prints 16 bits. + stubNVIDIASMI(t, "GPU-11112222-3333-4444-5555-666677778888, 00000000:03:00.0\n"+ + "GPU-99990000-1111-2222-3333-444455556666, 00000000:04:00.0\n", nil) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + info := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "") + + if info.BootID != "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111" { + t.Fatalf("BootID = %q", info.BootID) + } + nvidia := renderDeviceDetail(t, info, filepath.Join(env.driDir, "renderD128")) + if nvidia.PCIAddress != "0000:03:00.0" { + t.Fatalf("nvidia PCIAddress = %q, want 0000:03:00.0", nvidia.PCIAddress) + } + if nvidia.GPUUUID != "GPU-11112222-3333-4444-5555-666677778888" { + t.Fatalf("nvidia GPUUUID = %q", nvidia.GPUUUID) + } + intel := renderDeviceDetail(t, info, filepath.Join(env.driDir, "renderD129")) + if intel.PCIAddress != "0000:04:00.0" { + t.Fatalf("intel PCIAddress = %q, want 0000:04:00.0", intel.PCIAddress) + } + // nvidia-smi listed this address, but the device is Intel: attributing an + // NVIDIA uuid to it would merge two distinct GPUs into one inventory entry. + if intel.GPUUUID != "" { + t.Fatalf("intel GPUUUID = %q, want empty", intel.GPUUUID) + } +} + +// A host without the NVIDIA toolkit is the common case, not a failure: the +// report must still describe the hardware it can see. +func TestDetectHWAccelOmitsGPUUUIDWhenNVIDIASMIUnavailable(t *testing.T) { + env := setupHWAccelTest(t) + env.addPCIRenderDevice(t, "renderD128", "0x10de", "0000:03:00.0") + stubNVIDIASMI(t, "", errors.New("exec: \"nvidia-smi\": executable file not found in $PATH")) + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + detail := renderDeviceDetail(t, DetectHWAccelWithFFmpeg("auto", ffmpeg.path, ""), filepath.Join(env.driDir, "renderD128")) + + if detail.PCIAddress != "0000:03:00.0" { + t.Fatalf("PCIAddress = %q, want the sysfs address even without nvidia-smi", detail.PCIAddress) + } + if detail.GPUUUID != "" { + t.Fatalf("GPUUUID = %q, want empty", detail.GPUUUID) + } +} + +// A device with no PCI symlink (virtual, or a restricted sysfs) reports no +// address rather than the literal directory name behind the lookup. +func TestDetectHWAccelOmitsPCIAddressWithoutSysfsLink(t *testing.T) { + env := setupHWAccelTest(t) + env.addRenderDevice(t, "renderD128", "0x8086") + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + detail := renderDeviceDetail(t, DetectHWAccelWithFFmpeg("auto", ffmpeg.path, ""), filepath.Join(env.driDir, "renderD128")) + + if detail.PCIAddress != "" { + t.Fatalf("PCIAddress = %q, want empty", detail.PCIAddress) + } +} + +func TestDetectHWAccelOmitsBootIDOffLinux(t *testing.T) { + env := setupHWAccelTest(t) + env.setBootID(t, "2f6e7a8b-9c0d-4a2b-8c3d-5b2c1f0e1111") + currentGOOS = "darwin" + ffmpeg := writeFakeFFmpeg(t, fullyCapableProbe()) + + if got := DetectHWAccelWithFFmpeg("auto", ffmpeg.path, "").BootID; got != "" { + t.Fatalf("BootID = %q, want empty off Linux", got) + } +} + +func TestNormalizePCIAddress(t *testing.T) { + tests := []struct{ in, want string }{ + {"0000:03:00.0", "0000:03:00.0"}, + {"00000000:03:00.0", "0000:03:00.0"}, + {"00000000:0A:00.0", "0000:0a:00.0"}, + {" 0000:03:00.0 ", "0000:03:00.0"}, + {"not-an-address", "not-an-address"}, + {"", ""}, + } + for _, tt := range tests { + if got := normalizePCIAddress(tt.in); got != tt.want { + t.Fatalf("normalizePCIAddress(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestParseNVIDIAGPUUUIDsSkipsMalformedRows(t *testing.T) { + parsed := parseNVIDIAGPUUUIDs([]byte("GPU-aaa, 00000000:03:00.0\n\nmissing-separator\n, 00000000:05:00.0\nGPU-bbb, \n")) + if len(parsed) != 1 { + t.Fatalf("parsed = %v, want exactly the one well-formed row", parsed) + } + if parsed["0000:03:00.0"] != "GPU-aaa" { + t.Fatalf("parsed = %v", parsed) + } +} diff --git a/internal/playback/hwdevice.go b/internal/playback/hwdevice.go index f51f0f997..e53a9bf48 100644 --- a/internal/playback/hwdevice.go +++ b/internal/playback/hwdevice.go @@ -3,6 +3,8 @@ package playback import ( "log/slog" "os" + "slices" + "strconv" "strings" "sync" ) @@ -17,7 +19,8 @@ import ( // index/UUID rather than render-node path, so a multi-entry list falls back // to its first entry. A single configured value keeps the historical // pass-through contract for every accelerator, so existing deployments are -// unaffected. +// unaffected — but it is still counted, because per-device session reporting +// has to work on the one-GPU node that most deployments are. // HWDeviceSet is the parsed form of the playback.hw_device setting: an // ordered list of device entries. Order is priority order — ties in load @@ -71,6 +74,56 @@ var hwDeviceLoad = struct { counts map[string]int }{counts: map[string]int{}} +// DefaultNVENCDevice is the accounting key for an NVENC workload that named no +// device. NVENC addresses GPUs through the CUDA runtime rather than a render +// node, and ffmpeg defaults to CUDA device 0, so that is what an unqualified +// NVENC job is actually occupying. +const DefaultNVENCDevice = "cuda:0" + +// HWDeviceLoadSnapshot returns a copy of the active workload count per device. +// +// This exists so node metrics can report how many of this process's GPU jobs +// are pinned to each device — a number no driver can supply, because the driver +// sees processes and not sessions. It is a copy taken under the same lock the +// allocator uses, so a reader never observes a half-applied reservation and +// never holds up one. +func HWDeviceLoadSnapshot() map[string]int { + hwDeviceLoad.mu.Lock() + defer hwDeviceLoad.mu.Unlock() + counts := make(map[string]int, len(hwDeviceLoad.counts)) + for device, count := range hwDeviceLoad.counts { + if count > 0 { + counts[device] = count + } + } + return counts +} + +// nvencAccountingDevice is the key an NVENC workload is counted under. +// +// A configured multi-entry list resolves to its first entry, matching the +// device NVENC will actually use; an unconfigured value counts against the CUDA +// default. Accounting only — selection is untouched, because counting a +// workload and balancing on the count are different decisions and NVENC is +// deliberately excluded from the second. +// +// A bare CUDA index is rewritten to "cuda:N", because the count is only useful +// if it joins with the name resource sampling publishes for the same GPU, and +// that is "cuda:N" (or the render path, which is matched against the same +// index). Counting "1" would leave every explicitly-selected NVIDIA GPU +// reporting zero sessions while it transcodes. A GPU UUID passes through: the +// sampler knows nvidia-smi's UUID for each card and matches on it. +func nvencAccountingDevice(configured string) string { + first := ParseHWDeviceSet(configured).First() + if first == "" { + return DefaultNVENCDevice + } + if index, err := strconv.Atoi(first); err == nil && index >= 0 { + return "cuda:" + strconv.Itoa(index) + } + return first +} + // hwAccelBalancesRenderDevices reports whether the resolved acceleration mode // selects GPUs by render-device path, which is what the balancer hands out. // NVENC addresses GPUs by CUDA index/UUID and is deliberately excluded. @@ -122,16 +175,27 @@ func newHWDeviceRelease(device string) func() { } } +// countHWDeviceWorkload records one active workload against a device without +// influencing selection. It is the accounting half of a reservation, used where +// the device was decided elsewhere (NVENC, which picks by CUDA index, and +// session restarts that keep their original device). +func countHWDeviceWorkload(device string) func() { + hwDeviceLoad.mu.Lock() + hwDeviceLoad.counts[device]++ + hwDeviceLoad.mu.Unlock() + return newHWDeviceRelease(device) +} + // reserveConcreteHWDevice reserves a device that was selected for an earlier // process in the same transcode session. Restarts keep device affinity rather // than running the least-loaded selection again. func reserveConcreteHWDevice(device string) func() { + release := countHWDeviceWorkload(device) hwDeviceLoad.mu.Lock() - hwDeviceLoad.counts[device]++ count := hwDeviceLoad.counts[device] hwDeviceLoad.mu.Unlock() slog.Info("GPU workload device reserved", "device", device, "active_workloads", count) - return newHWDeviceRelease(device) + return release } var nvencMultiDeviceWarnOnce sync.Once @@ -140,41 +204,91 @@ var nvencMultiDeviceWarnOnce sync.Once // device for one GPU workload. resolvedHWAccel must already be resolved (no // "auto"). The returned release must be called exactly once when the ffmpeg // process for this workload has exited; it is idempotent and a no-op for -// workloads that did not reserve (empty or single-device value, or an -// accelerator the balancer does not manage). +// workloads that were not counted (a software accelerator, or a GPU accelerator +// with no configured device to name). // // - Empty value: returns "" so downstream auto-detection applies. -// - Single value: passes through unchanged for every accelerator. +// - Single value: passes through unchanged for every accelerator, and is +// counted for the GPU accelerators, because a node with one GPU is the +// common deployment and its sessions have to be reportable too. // - Multi-device value with QSV/VAAPI: reserves the present device with the // fewest active workloads (ties keep list order) until release. -// - Multi-device value with any other accelerator (including NVENC, which -// addresses GPUs by CUDA index/UUID, not render-node path): falls back to -// the first entry without reserving. +// - NVENC: selection is unchanged — the first entry, or empty for +// auto-detection — but the workload is counted against the device it will +// occupy so per-device reporting covers NVIDIA nodes too. NVENC addresses +// GPUs by CUDA index/UUID rather than render-node path, so it is still +// never balanced across a list. +// - Any other accelerator: falls back to the first entry without counting. func AcquireHWDevice(configured, resolvedHWAccel string) (string, func()) { - return acquireHWDevice(configured, resolvedHWAccel, "") + device, _, release := acquireHWDevice(configured, resolvedHWAccel, "") + return device, release } // acquireHWDevice applies the normal allocator while optionally excluding one // previously failed render device when another present device is available. // The selected device is still reserved through the same accounting path. -func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (string, func()) { +// +// It also returns the key the workload was counted under ("" when it was not +// counted), so a caller that outlives the ffmpeg process — a transcode session +// that restarts one — can re-count the replacement against the same device +// without restating the rule for which workloads count. +func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (device, workloadDevice string, release func()) { noop := func() {} set := ParseHWDeviceSet(configured) - if !set.Multi() { - return set.First(), noop - } - if !hwAccelBalancesRenderDevices(resolvedHWAccel) { - if resolvedHWAccel == "nvenc" { + if resolvedHWAccel == transcodeHWNVENC { + if set.Multi() { nvencMultiDeviceWarnOnce.Do(func() { slog.Warn("multi-device hw_device is not supported with NVENC (devices are CUDA index/UUID, not render paths); using the first entry", "hw_device", configured, "using", set.First()) }) } - return set.First(), noop + accounted := nvencAccountingDevice(configured) + return set.First(), accounted, countHWDeviceWorkload(accounted) + } + if !hwAccelBalancesRenderDevices(resolvedHWAccel) { + // A software workload occupies no GPU, so counting it would both + // misreport the device and skew the balancer that reads the counts. + return set.First(), "", noop + } + if !set.Multi() { + // Nothing to balance, but there is still a workload on a known device. + // Selection and accounting are separate decisions: skipping the second + // with the first is what left every single-GPU QSV/VAAPI node reporting + // zero sessions beside a busy engine. + if first := set.First(); first != "" { + return first, first, countHWDeviceWorkload(first) + } + // No configured device. Resolving it here rather than leaving it to + // PickRenderDevice downstream fixes two things at once: the workload + // runs on the render node auto-detection actually verified this backend + // on — not on whatever sorts first under /dev/dri, which on a + // mixed-vendor host is a different GPU — and it becomes countable, so a + // default-configured node stops reporting zero sessions beside a busy + // engine. With no verified device (nothing probed yet, or a backend + // named explicitly and never walked) this falls through unchanged and + // ffmpeg picks one downstream exactly as before. + if verified := VerifiedHWDevice(resolvedHWAccel); verified != "" { + return verified, verified, countHWDeviceWorkload(verified) + } + // Nothing verified. That is not only the cold-process case: an + // explicitly configured backend short-circuits resolution, so a host + // running hw_accel=qsv with no hw_device never walks its hardware at all + // and would otherwise never name a device here — reporting zero GPU + // sessions for every transcode it runs. Fall back to the device + // execution is about to pick anyway, which is this same first render + // node; appendHWAccelArgs resolves an empty value the same way. + if detected := detectRenderDevice(defaultDRIDir); detected != "" { + return detected, detected, countHWDeviceWorkload(detected) + } + return "", "", noop } // Select and reserve in one critical section so concurrent workload starts // observe each other's reservations instead of piling onto one device. - present := presentHWDevices(set.List()) + // The same ceiling the probe matrix stops at: nothing past it has a verdict + // behind it, so nothing past it takes work. Applied to the parsed list in + // the order it was configured, which is the order the walk probed, so both + // sides keep exactly the same devices. + present := verifiedHWDevices(resolvedHWAccel, presentHWDevices(UsableHWDevices(set.List()))) if len(present) > 1 && avoidDevice != "" { eligible := make([]string, 0, len(present)-1) for _, device := range present { @@ -187,13 +301,48 @@ func acquireHWDevice(configured, resolvedHWAccel, avoidDevice string) (string, f } } hwDeviceLoad.mu.Lock() - device := leastLoadedHWDeviceLocked(present) - hwDeviceLoad.counts[device]++ - count := hwDeviceLoad.counts[device] + selected := leastLoadedHWDeviceLocked(present) + hwDeviceLoad.counts[selected]++ + count := hwDeviceLoad.counts[selected] hwDeviceLoad.mu.Unlock() - slog.Info("GPU workload device selected", "device", device, "active_workloads", count) + slog.Info("GPU workload device selected", "device", selected, "active_workloads", count) - return device, newHWDeviceRelease(device) + return selected, selected, newHWDeviceRelease(selected) +} + +// verifiedHWDevices narrows a configured device list to the entries whose smoke +// encode actually passed. +// +// Presence is not fitness. A device that exists and can be opened can still fail +// to initialize the backend — a card in a bad state, a driver mismatch, a +// container that mapped the node without the matching libraries — and detection +// already found that out. Balancing across every present entry would hand a +// share of the node's workloads to that card and fail each of them at startup, +// while the capability report shows the backend verified. +// +// It narrows only when there is something to narrow to. An empty verified set +// means detection never ran for this backend (a cold process, or hw_accel named +// explicitly so resolution short-circuited), which is no evidence against any +// device, so the full present list stands. +func verifiedHWDevices(resolvedHWAccel string, present []string) []string { + verified := VerifiedHWDevices(resolvedHWAccel) + if len(verified) == 0 { + return present + } + eligible := make([]string, 0, len(present)) + for _, device := range present { + if slices.Contains(verified, device) { + eligible = append(eligible, device) + } + } + if len(eligible) == 0 { + // Every present device failed its probe, or the probe set and the + // configured set have drifted apart. Excluding everything would leave + // the balancer with nothing to pick, which is worse than letting ffmpeg + // try and report a real error. + return present + } + return eligible } // hwDeviceActiveCount reports the active workload count for one device; test diff --git a/internal/playback/hwdevice_test.go b/internal/playback/hwdevice_test.go index 92f76dbaf..b66f2f40e 100644 --- a/internal/playback/hwdevice_test.go +++ b/internal/playback/hwdevice_test.go @@ -2,8 +2,12 @@ package playback import ( "os" + "strconv" + "strings" "sync" "testing" + + "github.com/Silo-Server/silo-server/internal/tonemap" ) // fakeDeviceStat installs a stat function that reports only the given paths as @@ -58,12 +62,20 @@ func TestParseHWDeviceSet(t *testing.T) { } } -func TestAcquireHWDeviceEmptyValueStaysEmpty(t *testing.T) { +// An empty setting resolves to nothing only when there is nothing to resolve to. +// Pointed at an empty device directory for the same reason as the counting test +// below: with a real /dev/dri present this now returns the device execution +// would pick, which is the point of resolving it here. +func TestAcquireHWDeviceEmptyValueStaysEmptyWithoutDevices(t *testing.T) { resetDeviceLoad(t) + original := defaultDRIDir + defaultDRIDir = t.TempDir() + t.Cleanup(func() { defaultDRIDir = original }) + device, release := AcquireHWDevice("", "qsv") defer release() if device != "" { - t.Fatalf("device = %q, want empty so auto-detection applies", device) + t.Fatalf("device = %q, want empty so auto-detection applies downstream", device) } } @@ -79,6 +91,47 @@ func TestAcquireHWDeviceSingleValuePassesThrough(t *testing.T) { } } +// The single-GPU node is the common deployment. Having nothing to balance is no +// reason to leave its workloads uncounted: sessions would read 0 on a node +// whose engine busy percentage says it is transcoding. +func TestAcquireHWDeviceSingleRenderDeviceIsCounted(t *testing.T) { + for _, accel := range []string{"qsv", "vaapi"} { + t.Run(accel, func(t *testing.T) { + resetDeviceLoad(t) + fakeDeviceStat(t) // the device need not exist for the count to be right + _, releaseFirst := AcquireHWDevice("/dev/dri/renderD128", accel) + _, releaseSecond := AcquireHWDevice("/dev/dri/renderD128", accel) + + if got := HWDeviceLoadSnapshot()["/dev/dri/renderD128"]; got != 2 { + t.Fatalf("snapshot count = %d, want both workloads counted (%v)", got, HWDeviceLoadSnapshot()) + } + releaseFirst() + releaseSecond() + if got := HWDeviceLoadSnapshot(); len(got) != 0 { + t.Fatalf("snapshot after release = %v, want empty", got) + } + }) + } +} + +// With no render device to name, the workload stays uncounted rather than being +// attributed to a device that does not exist. The device directory is pointed at +// an empty temp dir rather than left at the real /dev/dri: unconfigured +// acquisition now falls back to the device execution would pick, so on a host +// that actually has a GPU this would otherwise assert the opposite of the truth. +func TestAcquireHWDeviceUnconfiguredRenderDeviceIsNotCounted(t *testing.T) { + resetDeviceLoad(t) + original := defaultDRIDir + defaultDRIDir = t.TempDir() + t.Cleanup(func() { defaultDRIDir = original }) + + _, release := AcquireHWDevice("", "vaapi") + defer release() + if got := HWDeviceLoadSnapshot(); len(got) != 0 { + t.Fatalf("snapshot = %v, want no count for an unresolved device", got) + } +} + func TestAcquireHWDeviceBalancesAcrossList(t *testing.T) { resetDeviceLoad(t) fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") @@ -128,16 +181,98 @@ func TestAcquireHWDeviceAllMissingFallsBackToFirst(t *testing.T) { } } -func TestAcquireHWDeviceNVENCMultiListUsesFirstWithoutReserving(t *testing.T) { +// NVENC is counted but never balanced: a multi-entry list still resolves to its +// first entry, because CUDA indexes are not render-node paths and the balancer +// has no way to compare them. +func TestAcquireHWDeviceNVENCMultiListUsesFirstWithoutBalancing(t *testing.T) { resetDeviceLoad(t) fakeDeviceStat(t) // NVENC entries are CUDA indexes/UUIDs, never present as paths - device, release := AcquireHWDevice("0,1", "nvenc") - defer release() - if device != "0" { - t.Fatalf("device = %q, want first NVENC entry", device) + first, releaseFirst := AcquireHWDevice("0,1", "nvenc") + defer releaseFirst() + second, releaseSecond := AcquireHWDevice("0,1", "nvenc") + defer releaseSecond() + + if first != "0" || second != "0" { + t.Fatalf("devices = %q, %q; want both on the first NVENC entry", first, second) } - if got := hwDeviceActiveCount("0"); got != 0 { - t.Fatalf("active count = %d, want no reservation for NVENC", got) + // ffmpeg is handed the bare CUDA index, but the count is keyed the way + // resource sampling names that GPU — otherwise the join drops it. + if got := hwDeviceActiveCount("cuda:0"); got != 2 { + t.Fatalf("active count = %d, want both NVENC workloads counted under cuda:0", got) + } +} + +// NVENC selects GPUs by CUDA index, and the sampler publishes them as "cuda:N". +// Counting the raw configured value would report zero sessions on every +// explicitly-selected NVIDIA GPU while it transcodes. +func TestNVENCAccountingDeviceUsesSamplerNamespace(t *testing.T) { + for _, tc := range []struct{ configured, want string }{ + {"", DefaultNVENCDevice}, + {"0", "cuda:0"}, + {"1", "cuda:1"}, + {"1,0", "cuda:1"}, + {"cuda:1", "cuda:1"}, + {"GPU-1234abcd", "GPU-1234abcd"}, + {"/dev/dri/renderD128", "/dev/dri/renderD128"}, + } { + if got := nvencAccountingDevice(tc.configured); got != tc.want { + t.Fatalf("nvencAccountingDevice(%q) = %q, want %q", tc.configured, got, tc.want) + } + } +} + +// An unconfigured NVENC workload lands on the CUDA default, which is what +// ffmpeg will actually use, so per-device reporting is not blank on the most +// common NVIDIA deployment. +func TestAcquireHWDeviceNVENCUnconfiguredCountsCUDADefault(t *testing.T) { + resetDeviceLoad(t) + device, release := AcquireHWDevice("", "nvenc") + if device != "" { + t.Fatalf("device = %q, want empty so auto-detection applies", device) + } + if got := hwDeviceActiveCount(DefaultNVENCDevice); got != 1 { + t.Fatalf("active count = %d, want the workload counted against %s", got, DefaultNVENCDevice) + } + release() + if got := hwDeviceActiveCount(DefaultNVENCDevice); got != 0 { + t.Fatalf("active count after release = %d, want 0", got) + } +} + +// The snapshot is what node metrics report per device; it must show live +// workloads and drop devices back out once they are released. +func TestHWDeviceLoadSnapshot(t *testing.T) { + resetDeviceLoad(t) + fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") + _, releaseA := AcquireHWDevice("/dev/dri/renderD128,/dev/dri/renderD129", "qsv") + _, releaseB := AcquireHWDevice("/dev/dri/renderD128,/dev/dri/renderD129", "qsv") + _, releaseNVENC := AcquireHWDevice("", "nvenc") + + snapshot := HWDeviceLoadSnapshot() + want := map[string]int{ + "/dev/dri/renderD128": 1, + "/dev/dri/renderD129": 1, + DefaultNVENCDevice: 1, + } + if len(snapshot) != len(want) { + t.Fatalf("snapshot = %v, want %v", snapshot, want) + } + for device, count := range want { + if snapshot[device] != count { + t.Fatalf("snapshot[%q] = %d, want %d (snapshot %v)", device, snapshot[device], count, snapshot) + } + } + + // The copy must not track later changes, or a reader would see counts move + // underneath a snapshot it already published. + releaseA() + releaseB() + releaseNVENC() + if snapshot["/dev/dri/renderD128"] != 1 { + t.Fatal("snapshot changed after release; it is not a copy") + } + if remaining := HWDeviceLoadSnapshot(); len(remaining) != 0 { + t.Fatalf("snapshot after all releases = %v, want empty", remaining) } } @@ -185,15 +320,18 @@ func TestAcquireHWDeviceAvoidsFailedRenderDeviceAndReservesAlternate(t *testing. resetDeviceLoad(t) fakeDeviceStat(t, "/dev/dri/renderD128", "/dev/dri/renderD129") configured := "/dev/dri/renderD128,/dev/dri/renderD129" - got, release := acquireHWDevice(configured, "qsv", "/dev/dri/renderD128") + got, workload, release := acquireHWDevice(configured, "qsv", "/dev/dri/renderD128") defer release() if got != "/dev/dri/renderD129" { t.Fatalf("alternate device = %q, want renderD129", got) } + if workload != got { + t.Fatalf("workload device = %q, want the selected device %q", workload, got) + } if active := hwDeviceActiveCount(got); active != 1 { t.Fatalf("alternate device active count = %d, want 1", active) } - if got, releaseNVENC := acquireHWDevice(configured, "nvenc", "/dev/dri/renderD128"); got != "/dev/dri/renderD128" { + if got, _, releaseNVENC := acquireHWDevice(configured, "nvenc", "/dev/dri/renderD128"); got != "/dev/dri/renderD128" { releaseNVENC() t.Fatalf("NVENC retry device = %q, want first configured device", got) } else { @@ -302,3 +440,36 @@ func TestAcquireHWDeviceConcurrentStartsBalanceExactly(t *testing.T) { t.Fatalf("concurrent workload split = %v, want exact %d/%d", counts, workloads/2, workloads/2) } } + +// The probe matrix stops at a ceiling, so past it there is no verdict to +// dispatch on. Balancing over the full configured list would hand a share of the +// node's transcodes to a device the walk never reached, and the failure would +// land after the session started — while the published capabilities say nothing +// about that device either way. +func TestAcquireHWDeviceNeverSelectsPastTheProbeCeiling(t *testing.T) { + resetDeviceLoad(t) + devices := make([]string, 0, tonemap.MaxProbedDevices+1) + for i := range tonemap.MaxProbedDevices + 1 { + devices = append(devices, "/dev/dri/renderD"+strconv.Itoa(128+i)) + } + fakeDeviceStat(t, devices...) + beyond := devices[tonemap.MaxProbedDevices] + configured := strings.Join(devices, ",") + + // Every probed device has to be handed a workload before the one past the + // ceiling could come up, so run enough starts to cover the whole list twice. + var releases []func() + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range len(devices) * 2 { + selected, release := AcquireHWDevice(configured, "qsv") + releases = append(releases, release) + if selected == beyond { + t.Fatalf("workload dispatched to %q, which is past the %d-device probe ceiling", + beyond, tonemap.MaxProbedDevices) + } + } +} diff --git a/internal/playback/registry_capability_timeout_test.go b/internal/playback/registry_capability_timeout_test.go new file mode 100644 index 000000000..68c43bc83 --- /dev/null +++ b/internal/playback/registry_capability_timeout_test.go @@ -0,0 +1,79 @@ +package playback + +import ( + "os" + "testing" +) + +// The registry-only budget is advertised on a capability report and covered by +// its hash, so it must not be derived from the host. A proxy reports what its +// ffmpeg can do and walks no hardware; if this number tracked /dev/dri, a GPU +// appearing on a machine that also runs a proxy would move that proxy's +// capability hash — costing the API a refetch and a planning-cache drop to +// announce capabilities that did not change. +func TestRegistryCapabilityTimeoutIgnoresHostDevices(t *testing.T) { + bare := RegistryCapabilityRequestTimeout() + bareEndpoint := RegistryCapabilityEndpointTimeout() + + withRenderDevices(t) + + if got := RegistryCapabilityRequestTimeout(); got != bare { + t.Fatalf("registry request budget = %s with render devices present, want the host-free %s", got, bare) + } + if got := RegistryCapabilityEndpointTimeout(); got != bareEndpoint { + t.Fatalf("registry endpoint budget = %s with render devices present, want the host-free %s", got, bareEndpoint) + } + // The control: the hardware-aware budget does read the host, which is why + // asking it for the software backend was not good enough. + if CapabilityEndpointTimeout(HWAccelNone, "") == bareEndpoint { + t.Fatal("CapabilityEndpointTimeout did not grow with the host's devices; " + + "the fixture proves nothing about the registry budget's independence") + } +} + +// The advertised budget is what a caller waits out, so it has to leave room for +// the response on top of the endpoint's own work — the same ordering +// CapabilityRequestTimeout has — and it has to be a real budget rather than +// something a caller would clamp away. +func TestRegistryCapabilityRequestTimeoutCoversTheEndpoint(t *testing.T) { + endpoint := RegistryCapabilityEndpointTimeout() + request := RegistryCapabilityRequestTimeout() + if request <= endpoint { + t.Fatalf("request budget %s does not exceed the endpoint budget %s", request, endpoint) + } + if request < probeRequestMinTimeout { + t.Fatalf("request budget %s is under the floor a caller clamps to (%s)", request, probeRequestMinTimeout) + } + if ceiling := MaxCapabilityRequestTimeout(); request > ceiling { + t.Fatalf("request budget %s is above the ceiling a caller believes (%s)", request, ceiling) + } +} + +// withRenderDevices points device discovery at a temp /dev/dri holding two +// classifiable render nodes, restored when the test ends. +func withRenderDevices(t *testing.T) { + t.Helper() + driDir := t.TempDir() + sysDir := t.TempDir() + for name, ids := range map[string][2]string{ + "renderD128": {"0x8086", "0x56a6"}, + "renderD129": {"0x10de", "0x2489"}, + } { + if err := os.WriteFile(driDir+"/"+name, nil, 0o644); err != nil { + t.Fatal(err) + } + devDir := sysDir + "/" + name + "/device" + if err := os.MkdirAll(devDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(devDir+"/vendor", []byte(ids[0]+"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(devDir+"/device", []byte(ids[1]+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + origDRI, origSys := defaultDRIDir, sysClassDRMDir + defaultDRIDir, sysClassDRMDir = driDir, sysDir + t.Cleanup(func() { defaultDRIDir, sysClassDRMDir = origDRI, origSys }) +} diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index f1c0bee53..e60a241d7 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -172,6 +172,9 @@ const ( transcodeResolution1080p = "1080p" transcodeResolution2160p = "2160p" qsvHWMapFilter = "hwmap=derive_device=qsv" + // vaapiHWDeviceAlias names the VAAPI device every non-QSV hardware command + // line declares; filter graphs and probes reference it by this alias. + vaapiHWDeviceAlias = "hw" ) // TranscodeSession manages a running ffmpeg HLS transcode process. @@ -202,10 +205,11 @@ type TranscodeSession struct { // written by a previous generation (or a previous session sharing the // directory) and describes media this process has not produced yet. generationStartedAt time.Time - // reserveHWDeviceOnRestart is true when StartTranscode selected and reserved - // one device from a multi-device QSV/VAAPI setting. Each replacement ffmpeg - // process reacquires that same concrete device. - reserveHWDeviceOnRestart bool + // hwWorkloadDevice is the device this session's GPU workload is counted + // against, or empty when it holds none. Each replacement ffmpeg process + // reacquires this same device rather than re-running selection, so a restart + // keeps its GPU affinity and stays visible in per-device reporting. + hwWorkloadDevice string } // NewTranscodeSessionForTest exposes only the output directory needed by tests @@ -358,11 +362,11 @@ func StartTranscode(ctx context.Context, opts TranscodeOpts) (*TranscodeSession, if err := validateToneMapOpts(opts); err != nil { return nil, err } - configuredHWDevices := ParseHWDeviceSet(opts.HWDevice) - reserveHWDeviceOnRestart := configuredHWDevices.Multi() && hwAccelBalancesRenderDevices(opts.HWAccel) // Resolve a multi-device hw_device list to one concrete GPU. Restarts reuse // the selected device, but each ffmpeg process owns its own reservation. - hwDevice, releaseHWDevice := acquireHWDevice(opts.HWDevice, opts.HWAccel, opts.AvoidHWDevice) + // hwWorkloadDevice is whatever the allocator counted this workload under, so + // the rule for which workloads are counted lives in one place. + hwDevice, hwWorkloadDevice, releaseHWDevice := acquireHWDevice(opts.HWDevice, opts.HWAccel, opts.AvoidHWDevice) opts.HWDevice = hwDevice opts.AvoidHWDevice = "" if err := validateToneMapSource(ctx, opts); err != nil { @@ -385,14 +389,14 @@ func StartTranscode(ctx context.Context, opts TranscodeOpts) (*TranscodeSession, // transcode process outlives a disconnected manifest request. ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) s := &TranscodeSession{ - cancel: cancel, - opts: opts, - outputDir: opts.OutputDir, - running: true, - done: make(chan struct{}), - stderr: newBoundedTailBuffer(stderrTailMaxBytes), - lastRequestedSegment: opts.StartSegmentNumber, - reserveHWDeviceOnRestart: reserveHWDeviceOnRestart, + cancel: cancel, + opts: opts, + outputDir: opts.OutputDir, + running: true, + done: make(chan struct{}), + stderr: newBoundedTailBuffer(stderrTailMaxBytes), + lastRequestedSegment: opts.StartSegmentNumber, + hwWorkloadDevice: hwWorkloadDevice, } args := buildFFmpegArgs(opts) @@ -629,7 +633,7 @@ func ResolveToneMapExecutor(ctx context.Context, opts TranscodeOpts) (TranscodeO opts.ToneMapSourceRevision.IsZero() { return opts, fmt.Errorf("incomplete tone-map recipe") } - backend := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath) + backend := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath, opts.HWDevice) capabilities, err := tonemap.Probe(ctx, ResolveFFmpegPath(opts.FFmpegPath), backend, opts.HWDevice) if err != nil { return opts, fmt.Errorf("%w: probe tone-map executor: %w", ErrToneMapExecutorUnavailable, err) @@ -859,7 +863,10 @@ func resolveEffectiveTranscodeHWAccel(opts TranscodeOpts) string { } func resolveEffectiveTranscodeHWAccelContext(ctx context.Context, opts TranscodeOpts) string { - hwAccel := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath) + // The device goes with the backend: resolution probes it, so a host whose + // first render node belongs to another vendor is not verified on hardware + // the transcode will never open. + hwAccel := ResolveHWAccelWithFFmpegContext(ctx, opts.HWAccel, opts.FFmpegPath, opts.HWDevice) if hwAccel == "" { return "" } @@ -1008,10 +1015,7 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { hwDevice = "/dev/dri/renderD128" // last-resort fallback } // VAAPI→QSV hardware pipeline: derive QSV from VAAPI device. - args = append(args, - "-init_hw_device", qsvVAAPIInitDevice(hwDevice), - "-init_hw_device", "qsv=qs@va", - ) + args = append(args, tonemap.QSVInitDeviceArgs(hwDevice)...) if opts.ToneMapMode == tonemap.ModeHardware && opts.ToneMapFilter == tonemap.HardwareFilterOpenCL { args = append(args, "-init_hw_device", "opencl=ocl@va") } @@ -1025,10 +1029,8 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { if vaapiDevice == "" { vaapiDevice = "/dev/dri/renderD128" // last-resort fallback } - args = append(args, - "-init_hw_device", fmt.Sprintf("vaapi=hw:%s", vaapiDevice), - "-filter_hw_device", "hw", - ) + args = append(args, tonemap.VAAPIInitDeviceArgs(vaapiHWDeviceAlias, vaapiDevice)...) + args = append(args, "-filter_hw_device", vaapiHWDeviceAlias) if !opts.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } @@ -1060,10 +1062,6 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string { return args } -func qsvVAAPIInitDevice(device string) string { - return fmt.Sprintf("vaapi=va:%s,driver=iHD,kernel_driver=i915,vendor_id=0x8086", device) -} - // videoPreset returns an encoder-compatible preset. CPU encoders use a faster // fast-start preset for initial playback, while QSV stays on the fastest // preset family it supports. @@ -1462,9 +1460,10 @@ const ( // the source channels before FFmpeg sums them would still allow the final // stereo signal to clip. The limiter's input gain is +6.0206 dB; its -2 dBFS // sample ceiling leaves headroom for lossy-codec and inter-sample overshoot. -// async=1 removes sub-frame input timestamp jitter while retaining the source -// clock and first packet timestamp. Without it, fixed-duration AAC packets can -// carry small PTS gaps that Firefox renders as audible zero-fill crackle. +// async=1 enables FFmpeg's timestamp-matching fill/trim behavior while +// retaining the source clock and first packet timestamp. On the affected +// inputs, the resulting fixed-duration AAC packets no longer carry the small +// PTS gaps that Firefox renders as audible zero-fill crackle. func appendStereoDownmixBoostArgs(args []string, sourceChannels, outputChannels int) []string { if sourceChannels <= 2 || outputChannels != 2 { return args @@ -2868,7 +2867,7 @@ func (s *TranscodeSession) restart( s.stderr.Reset() } s.restartCount++ - reserveHWDevice := s.reserveHWDeviceOnRestart + hwWorkloadDevice := s.hwWorkloadDevice s.mu.Unlock() previousOpts := opts @@ -2925,8 +2924,8 @@ func (s *TranscodeSession) restart( // this session on the same concrete GPU while accounting for the replacement // process as a new active workload. releaseHWDevice := func() {} - if reserveHWDevice { - releaseHWDevice = reserveConcreteHWDevice(opts.HWDevice) + if hwWorkloadDevice != "" { + releaseHWDevice = reserveConcreteHWDevice(hwWorkloadDevice) } // As in StartTranscode, stamp the generation before the process can write. diff --git a/internal/playback/transcode_args_test.go b/internal/playback/transcode_args_test.go index 9b440e08f..027f62bc2 100644 --- a/internal/playback/transcode_args_test.go +++ b/internal/playback/transcode_args_test.go @@ -1669,8 +1669,8 @@ func TestAppendAudioArgsNormalizesEveryAACEncodeAndBoostsOnlySurroundToStereo(t // builder's encoder probe must succeed on Linux CI as well as macOS. func videoToolboxTestFFmpeg(t *testing.T) string { t.Helper() - resetNVENCProbeCacheForTest() - t.Cleanup(resetNVENCProbeCacheForTest) + resetHWProbeCacheForTest() + t.Cleanup(resetHWProbeCacheForTest) return writeFakeFFmpeg(t, successfulVideoToolboxProbe()).path } diff --git a/internal/playback/transcode_manifest_test.go b/internal/playback/transcode_manifest_test.go index 574976472..8d948ec03 100644 --- a/internal/playback/transcode_manifest_test.go +++ b/internal/playback/transcode_manifest_test.go @@ -1164,7 +1164,17 @@ func TestTranscodeThrottlerIgnoresOutputFromAnEarlierGeneration(t *testing.T) { } // Once this generation writes its own manifest, throttling works normally. + // + // The mtime is set rather than inherited from the write. Staleness is + // decided by ManifestModTime.Before(GenerationStartedAt), and a filesystem + // with coarse mtime granularity — which a CI runner's overlay can have and a + // developer's APFS does not — truncates a write made at `now` back below it, + // so the fresh manifest reads as older than the generation that produced it. + fresh := now.Add(time.Second) writeManifestRange(t, tempDir, 225, 293, ".ts") + if err := os.Chtimes(filepath.Join(tempDir, "stream.m3u8"), fresh, fresh); err != nil { + t.Fatalf("chtimes manifest: %v", err) + } throttler.CheckOnce() if !throttler.paused { t.Fatal("expected throttler to pause on this generation's own produced head") diff --git a/internal/playback/transformations_v3.go b/internal/playback/transformations_v3.go index 900dd6980..ed3938688 100644 --- a/internal/playback/transformations_v3.go +++ b/internal/playback/transformations_v3.go @@ -53,24 +53,42 @@ func ProbeTransformationRegistryWithToneMapV3Result(ctx context.Context, ffmpegP encoders, encoderErr := exec.CommandContext(encoderCtx, ffmpegPath, "-hide_banner", "-encoders").Output() encoderContextErr := encoderCtx.Err() cancelEncoders() - audioRecipeCtx, cancelAudioRecipe := context.WithTimeout(ctx, 3*time.Second) - audioRecipeErr := exec.CommandContext(audioRecipeCtx, ffmpegPath, - "-hide_banner", "-loglevel", "error", - "-f", "lavfi", "-i", "anullsrc=r=8000:cl=5.1", - "-frames:a", "1", "-af", stereoDownmixBoostFilterV3, - "-f", "null", "-", - ).Run() - audioRecipeContextErr := audioRecipeCtx.Err() - cancelAudioRecipe() + normalizeRecipeErr, normalizeRecipeContextErr := probeAudioRecipeFilterV3(ctx, ffmpegPath, "stereo", aacTimestampNormalizeFilterV3) + downmixRecipeErr, downmixRecipeContextErr := probeAudioRecipeFilterV3(ctx, ffmpegPath, "5.1", stereoDownmixBoostFilterV3) _, ffmpegErr := exec.LookPath(ffmpegPath) registry := NewTransformationRegistryV3([]TransformationSpecV3{ {Name: TransformationServerDV7HDR10V3, RecipeVersion: TransformationServerDV7HDR10RecipeVersionV3, Available: bytes.Contains(bsfs, []byte("dovi_rpu")) && bytes.Contains(bsfs, []byte("filter_units")), RequiredCapability: "ffmpeg_bsf:dovi_rpu+filter_units", PromisedDynamicRange: DynamicRangeHDR10V3, ValidatedClaims: DV7ToHDR10ClaimsV3(), TerminalReason: TerminalDVConversionUnsupportedV3}, {Name: TransformationServerDV8BaseV3, RecipeVersion: TransformationServerDV8BaseRecipeVersionV3, Available: bytes.Contains(bsfs, []byte("dovi_rpu")) && bytes.Contains(bsfs, []byte("filter_units")), RequiredCapability: "ffmpeg_bsf:dovi_rpu+filter_units", ValidatedClaims: DV8ToBaseLayerClaimsV3(""), TerminalReason: TerminalDVConversionUnsupportedV3}, - {Name: TransformationAudioToAACV3, RecipeVersion: TransformationAudioToAACRecipeVersionV3, Available: ffmpegErr == nil && bytes.Contains(encoders, []byte(" aac ")) && audioRecipeErr == nil, RequiredCapability: "ffmpeg_encoder:aac+ffmpeg_filter_smoke:timestamp_normalization_v4", ValidatedClaims: []string{ClaimAudioDecodeV3}, TerminalReason: TerminalAudioConversionUnsupportedV3}, + {Name: TransformationAudioToAACV3, RecipeVersion: TransformationAudioToAACRecipeVersionV3, Available: ffmpegErr == nil && bytes.Contains(encoders, []byte(" aac ")) && normalizeRecipeErr == nil && downmixRecipeErr == nil, RequiredCapability: "ffmpeg_encoder:aac+ffmpeg_filter_smoke:timestamp_normalization_and_stereo_downmix_v4", ValidatedClaims: []string{ClaimAudioDecodeV3}, TerminalReason: TerminalAudioConversionUnsupportedV3}, {Name: TransformationVideoToH264V3, RecipeVersion: TransformationVideoToH264RecipeVersionV3, Available: ffmpegErr == nil && h264EncoderAvailableV3(encoders), RequiredCapability: "ffmpeg_encoder:h264", PromisedDynamicRange: DynamicRangeSDRV3, ValidatedClaims: []string{ClaimH264DecodeV3}, TerminalReason: TerminalVideoConversionUnsupportedV3}, {Name: TransformationHDRToSDRToneMapV3, RecipeVersion: TransformationHDRToSDRToneMapRecipeVersionV3, Available: len(toneMapCapabilities) > 0, RequiredCapability: "ffmpeg_filter:hdr_to_sdr_tonemap", PromisedDynamicRange: DynamicRangeSDRV3, ValidatedClaims: []string{ClaimHDRMetadataRemovedV3, ClaimSDRBT709OutputV3}, TerminalReason: TerminalHDRTranscodeUnsupportedV3}, }) - return registry, errors.Join(bsfErr, encoderErr, bsfContextErr, encoderContextErr, audioRecipeProbeInfrastructureError(audioRecipeErr), audioRecipeContextErr) + return registry, errors.Join( + bsfErr, + encoderErr, + bsfContextErr, + encoderContextErr, + audioRecipeProbeInfrastructureError(normalizeRecipeErr), + normalizeRecipeContextErr, + audioRecipeProbeInfrastructureError(downmixRecipeErr), + downmixRecipeContextErr, + ) +} + +// probeAudioRecipeFilterV3 smoke-tests one filter branch used by the frozen +// AAC recipe and distinguishes unsupported graphs from probe infrastructure +// failures at the registry boundary. +func probeAudioRecipeFilterV3(ctx context.Context, ffmpegPath, channelLayout, filter string) (error, error) { + probeCtx, cancelProbe := context.WithTimeout(ctx, 3*time.Second) + probeErr := exec.CommandContext(probeCtx, ffmpegPath, + "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "anullsrc=r=8000:cl="+channelLayout, + "-frames:a", "1", "-af", filter, + "-f", "null", "-", + ).Run() + probeContextErr := probeCtx.Err() + cancelProbe() + return probeErr, probeContextErr } // An ordinary non-zero FFmpeg exit means the installed filter graph is not a diff --git a/internal/playback/transformations_v3_test.go b/internal/playback/transformations_v3_test.go index 155c9f2b4..024bd35dd 100644 --- a/internal/playback/transformations_v3_test.go +++ b/internal/playback/transformations_v3_test.go @@ -3,6 +3,7 @@ package playback import ( "context" "errors" + "fmt" "os" "path/filepath" "testing" @@ -120,28 +121,37 @@ func TestProbeTransformationRegistryV3RequiresBothDV7BaseLayerFilters(t *testing } } -func TestProbeTransformationRegistryV3RequiresVersion3AudioFilterGraph(t *testing.T) { - ffmpeg := filepath.Join(t.TempDir(), "ffmpeg") - // Model an older FFmpeg that lists both filters but rejects one of the v3 - // graph options (notably out_chlayout or alimiter latency compensation). - script := "#!/bin/sh\ncase \"$2\" in\n-bsfs) : ;;\n-encoders) echo ' A....D aac AAC' ;;\n-filters) echo ' ... aresample A->A'; echo ' T.C alimiter A->A' ;;\nesac\ncase \" $* \" in\n*\" -f lavfi \"*) exit 1 ;;\nesac\n" - if err := os.WriteFile(ffmpeg, []byte(script), 0o755); err != nil { - t.Fatal(err) - } +func TestProbeTransformationRegistryV3RequiresBothVersion4AudioFilterGraphs(t *testing.T) { + for _, test := range []struct { + name string + rejectFilter string + }{ + {name: "timestamp normalization", rejectFilter: aacTimestampNormalizeFilterV3}, + {name: "surround downmix", rejectFilter: stereoDownmixBoostFilterV3}, + } { + t.Run(test.name, func(t *testing.T) { + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg") + script := fmt.Sprintf("#!/bin/sh\ncase \"$2\" in\n-bsfs) : ;;\n-encoders) echo ' A....D aac AAC' ;;\nesac\ncase \" $* \" in\n*\" -af %s \"*) exit 1 ;;\nesac\n", test.rejectFilter) + if err := os.WriteFile(ffmpeg, []byte(script), 0o755); err != nil { + t.Fatal(err) + } - registry, err := ProbeTransformationRegistryWithToneMapV3Result(context.Background(), ffmpeg, nil) - if err != nil { - t.Fatalf("unsupported graph should be a cacheable capability result: %v", err) - } - if registry.Available(TransformationAudioToAACV3) { - t.Fatal("audio_to_aac advertised when the exact version 3 graph was rejected") + registry, err := ProbeTransformationRegistryWithToneMapV3Result(context.Background(), ffmpeg, nil) + if err != nil { + t.Fatalf("unsupported graph should be a cacheable capability result: %v", err) + } + if registry.Available(TransformationAudioToAACV3) { + t.Fatalf("audio_to_aac advertised when %s was rejected", test.name) + } + }) } - script = "#!/bin/sh\ncase \"$2\" in\n-bsfs) : ;;\n-encoders) echo ' A....D aac AAC' ;;\n-filters) echo ' ... aresample A->A'; echo ' T.C alimiter A->A' ;;\nesac\n" + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg") + script := "#!/bin/sh\ncase \"$2\" in\n-bsfs) : ;;\n-encoders) echo ' A....D aac AAC' ;;\nesac\n" if err := os.WriteFile(ffmpeg, []byte(script), 0o755); err != nil { t.Fatal(err) } - registry = ProbeTransformationRegistryV3(context.Background(), ffmpeg) + registry := ProbeTransformationRegistryV3(context.Background(), ffmpeg) for _, transformation := range registry.Advertised() { if transformation.Name == TransformationAudioToAACV3 { if transformation.RecipeVersion != TransformationAudioToAACRecipeVersionV3 { @@ -150,5 +160,5 @@ func TestProbeTransformationRegistryV3RequiresVersion3AudioFilterGraph(t *testin return } } - t.Fatal("audio_to_aac was not advertised with the complete version 3 toolchain") + t.Fatal("audio_to_aac was not advertised with the complete version 4 toolchain") } diff --git a/internal/proxy/capabilities_test.go b/internal/proxy/capabilities_test.go index 0bf263e21..ca06f40db 100644 --- a/internal/proxy/capabilities_test.go +++ b/internal/proxy/capabilities_test.go @@ -15,7 +15,7 @@ import ( // and the mismatch only surfaces as a failed stream. func TestProxyAdvertisesTransformationCapabilities(t *testing.T) { const secret = "capability-secret" - server := newDownloadProxyServer(t, secret) + server := newCapabilityProxyServer(t, secret) request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) request.Header.Set("Authorization", "Bearer "+secret) @@ -39,7 +39,7 @@ func TestProxyAdvertisesTransformationCapabilities(t *testing.T) { } func TestProxyCapabilitiesRequireBearer(t *testing.T) { - server := newDownloadProxyServer(t, "capability-secret") + server := newCapabilityProxyServer(t, "capability-secret") recorder := httptest.NewRecorder() server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil)) diff --git a/internal/proxy/capability_snapshot_test.go b/internal/proxy/capability_snapshot_test.go new file mode 100644 index 000000000..5bd1547ad --- /dev/null +++ b/internal/proxy/capability_snapshot_test.go @@ -0,0 +1,301 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/playback" +) + +func decodeProxyHealth(t *testing.T, server *Server) healthResponse { + t.Helper() + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var health healthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &health); err != nil { + t.Fatalf("decode health: %v", err) + } + return health +} + +// A proxy executes remux recipes, so the API tracks its capabilities the same +// way it tracks a transcode node's: by the hash health advertises. Until the +// first snapshot the field must be absent rather than a hash of nothing, which +// the sweep would treat as a real report. +func TestProxyHealthPublishesCapabilityHashOnlyAfterSnapshot(t *testing.T) { + server := newCapabilityProxyServer(t, "capability-secret") + + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q before any snapshot, want empty", got) + } + + server.refreshCapabilitySnapshot(context.Background()) + + hash := decodeProxyHealth(t, server).CapabilitiesHash + if hash == "" { + t.Fatal("capabilities_hash is still empty after a snapshot") + } + // A second snapshot of an unchanged ffmpeg must not move the hash, or the + // sweep would refetch this proxy's report forever. + server.refreshCapabilitySnapshot(context.Background()) + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != hash { + t.Fatalf("capabilities_hash changed without hardware changing: %q then %q", hash, got) + } +} + +// The endpoint and the background snapshot share one assembly, so a served +// report carries the same hash health publishes. +func TestProxyCapabilitiesPublishCapabilityHash(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash == "" { + t.Fatal("served capability report carries no capability_hash") + } + served := info + served.CapabilityHash = "" + if want := playback.ComputeCapabilityHash(served); want != info.CapabilityHash { + t.Fatalf("capability_hash = %s, want %s for the served payload", info.CapabilityHash, want) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != info.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the just-served %q", got, info.CapabilityHash) + } +} + +// A probe that did not finish hashes differently from the same ffmpeg probed +// successfully, so publishing it would announce a capability change that never +// happened — and cost the API a full capability refetch plus a planning-cache +// drop. A caller that gives up must leave the published hash alone. +func TestProxyCapabilitiesRejectsIncompleteProbeWithoutPublishing(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published before the canceled request") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil).WithContext(canceled) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusServiceUnavailable, recorder.Body.String()) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished probe, want the previous %q", got, published) + } +} + +// The background snapshot has the same duty: a probe it could not finish is not +// evidence the proxy's ffmpeg lost anything. +func TestProxySnapshotKeepsPreviousHashWhenProbeCannotFinish(t *testing.T) { + server := newCapabilityProxyServer(t, "capability-secret") + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published by the first snapshot") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + server.refreshCapabilitySnapshot(canceled) + + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished snapshot, want the previous %q", got, published) + } +} + +// newCapabilityProxyServer builds a proxy whose configured ffmpeg is a script +// with a known, successful answer for every listing the capability assembly +// runs. +// +// The capability tests must not depend on the host's toolchain. Left +// unconfigured, the probes shell out to whatever `ffmpeg` is on PATH — which +// asserts a 200 on a developer's machine and a 503 on CI, where no ffmpeg is +// installed, because ProbeTransformationRegistryWithToneMapV3Result reports the +// exec failure. Scripting the binary also makes the published capability hash +// deterministic, which is what the stability assertions here depend on. +func newCapabilityProxyServer(t *testing.T, secret string) *Server { + t.Helper() + server, _ := newCapabilityProxyServerRecordingFFmpeg(t, secret, true) + return server +} + +// newCapabilityProxyServerRecordingFFmpeg is newCapabilityProxyServer with the +// scripted binary's argv appended to a log, and returns the log's path so a test +// can assert what the assembly did and did not run. +// +// aacAvailable scripts whether `-encoders` lists the AAC encoder, which is the +// cheapest way to make this proxy's advertised transformations genuinely differ. +func newCapabilityProxyServerRecordingFFmpeg(t *testing.T, secret string, aacAvailable bool) (*Server, string) { + t.Helper() + dir := t.TempDir() + ffmpegPath := filepath.Join(dir, "ffmpeg") + invocations := filepath.Join(dir, "invocations.log") + encoders := " V..... libx264 H.264\\n" + if aacAvailable { + encoders += " A..... aac AAC\\n" + } + script := "#!/bin/sh\n" + + "printf '%s\\n' \"$*\" >> " + invocations + "\n" + + "case \"$*\" in\n" + + " *-bsfs*) echo 'dovi_rpu'; exit 0 ;;\n" + + " *-encoders*) printf '" + encoders + "'; exit 0 ;;\n" + + "esac\n" + + "exit 0\n" + if err := os.WriteFile(ffmpegPath, []byte(script), 0o700); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = secret + cfg.Playback.FFmpegPath = ffmpegPath + w.SetConfigForTest(cfg) + return NewServer(w, nil), invocations +} + +// fetchProxyCapabilities reads the served capability report. +func fetchProxyCapabilities(t *testing.T, server *Server, secret string) playback.HWAccelInfo { + t.Helper() + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + return info +} + +// A proxy relays streams and runs identity/remux recipes; it never executes a +// hardware transcode, and nothing on the API side reads its acceleration +// fields. So it must not run the detection walk at all — the report carries no +// backends, no devices and no host identity, and the only ffmpeg it execs is the +// transformation registry's own listings. +func TestProxyCapabilitiesReportNoHardware(t *testing.T) { + const secret = "capability-secret" + server, invocations := newCapabilityProxyServerRecordingFFmpeg(t, secret, true) + + info := fetchProxyCapabilities(t, server, secret) + + if info.Resolved != playback.HWAccelNone { + t.Fatalf("resolved = %q, want %q on a proxy", info.Resolved, playback.HWAccelNone) + } + if len(info.DetectedBackends) != 0 { + t.Fatalf("detected_backends = %#v, want none on a proxy", info.DetectedBackends) + } + if len(info.RenderDevices) != 0 || len(info.RenderDeviceDetails) != 0 { + t.Fatalf("render devices = %v / %#v, want none on a proxy", info.RenderDevices, info.RenderDeviceDetails) + } + if len(info.NVIDIAGPUUUIDs) != 0 { + t.Fatalf("nvidia_gpu_uuids = %v, want none on a proxy", info.NVIDIAGPUUUIDs) + } + if info.IntelDetected { + t.Fatal("intel_detected is set on a proxy that probed no hardware") + } + // No boot id is the reboot half of the contract: everything a reboot can + // move is out of the report, so the hash tracks only what this proxy can do. + // A hash that moved on reboot cost the API a refetch and a planning-cache + // drop for a proxy whose abilities were identical. + if info.BootID != "" { + t.Fatalf("boot_id = %q, want none: a reboot must not move a proxy's hash", info.BootID) + } + if len(info.Transformations) == 0 { + t.Fatal("no transformations advertised; the report has nothing the planner can use") + } + + // The walk's smoke encodes are what this change exists to stop paying for, + // on every proxy, every fifteen minutes. + recorded, err := os.ReadFile(invocations) + if err != nil { + t.Fatalf("read ffmpeg invocations: %v", err) + } + for _, line := range strings.Split(strings.TrimSpace(string(recorded)), "\n") { + if line == "" { + continue + } + if strings.Contains(line, "-init_hw_device") || strings.Contains(line, "-hwaccel") { + t.Fatalf("proxy ran a hardware probe: ffmpeg %s", line) + } + } +} + +// The hash still has to move for the one thing it now tracks, or a proxy whose +// ffmpeg lost the AAC encoder would keep being planned for audio remuxes it can +// no longer run. +func TestProxyCapabilityHashTracksTransformations(t *testing.T) { + const secret = "capability-secret" + capable, _ := newCapabilityProxyServerRecordingFFmpeg(t, secret, true) + degraded, _ := newCapabilityProxyServerRecordingFFmpeg(t, secret, false) + + capableInfo := fetchProxyCapabilities(t, capable, secret) + degradedInfo := fetchProxyCapabilities(t, degraded, secret) + + if len(capableInfo.Transformations) == len(degradedInfo.Transformations) { + t.Fatalf("both ffmpeg builds advertised %d transformations; the fixture proves nothing", + len(capableInfo.Transformations)) + } + if capableInfo.CapabilityHash == degradedInfo.CapabilityHash { + t.Fatalf("capability_hash = %s for both builds, but their transformations differ", + capableInfo.CapabilityHash) + } +} + +// A proxy's capability read is cheap now, but the budget still has to be +// advertised: a caller that guesses cancels the read, and the proxy's stored +// report falls as far behind as it would after a failure. +func TestProxyCapabilitiesAdvertiseTheProbeBudget(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + + info := fetchProxyCapabilities(t, server, secret) + // Sized for a registry-only probe. A proxy that advertised the cluster's + // acceleration budget would hold a caller's connection open for minutes of + // hardware work it no longer does — and one that sized the budget from the + // host's device set would put that count inside its capability hash. + want := playback.RegistryCapabilityRequestTimeout().Milliseconds() + if info.ProbeRequestTimeoutMillis != want { + t.Fatalf("probe_request_timeout_ms = %d, want the proxy's own %d", + info.ProbeRequestTimeoutMillis, want) + } + + // It is inside the hash, so a build that needs longer reaches the sweep + // rather than sitting behind an unchanged identity. + served := info + served.ProbeRequestTimeoutMillis = want + 1_000 + served.CapabilityHash = "" + if playback.ComputeCapabilityHash(served) == info.CapabilityHash { + t.Fatal("the advertised budget does not move the capability hash") + } +} diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go new file mode 100644 index 000000000..8ed0b693c --- /dev/null +++ b/internal/proxy/metrics_test.go @@ -0,0 +1,118 @@ +package proxy + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +func newMetricsProxyServer(t *testing.T) *Server { + t.Helper() + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = "proxy-metrics-secret" + cfg.Playback.TranscodeDir = t.TempDir() + w.SetConfigForTest(cfg) + return NewServer(w, nil) +} + +// newFakeSampler answers with a fixed reading so the handlers under test are +// exercised without a Linux host beneath them. +func newFakeSampler() *nodemetrics.Sampler { + video := 8 + return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: time.Now(), + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 1, + VideoBusyPct: &video, Source: nodemetrics.SourceFdinfo, + }}, + }) +} + +// A proxy runs ffmpeg too (remux, Dolby Vision RPU strip), so it reports the +// same resource fields a transcode node does — and, like a transcode node, it +// reads them from a published snapshot rather than measuring on the request. +func TestProxyHealthIncludesResourceSampleWithoutBlocking(t *testing.T) { + server := newMetricsProxyServer(t) + server.metrics = newFakeSampler() + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + answered <- recorder + }() + + var recorder *httptest.ResponseRecorder + select { + case recorder = <-answered: + case <-time.After(5 * time.Second): + t.Fatal("health handler blocked") + } + + var body struct { + Status string `json:"status"` + System *struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + } `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + if body.Status != "ok" { + t.Fatalf("status = %q", body.Status) + } + if body.System == nil || body.System.CPUPct != 41 { + t.Fatalf("system = %+v", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" { + t.Fatalf("gpu = %+v", body.GPU) + } +} + +// Without a sampler the response is byte-for-byte what it always was. +func TestProxyHealthOmitsResourceFieldsWithoutASampler(t *testing.T) { + server := newMetricsProxyServer(t) + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + for _, key := range []string{"system", "gpu"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sampler: %s", key, recorder.Body) + } + } +} + +func TestProxyMetricsEndpointIsMountedAndUnauthenticated(t *testing.T) { + server := newMetricsProxyServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("GET /metrics = %d, want 200", recorder.Code) + } + if recorder.Body.Len() == 0 { + t.Fatal("GET /metrics returned an empty body") + } +} diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go new file mode 100644 index 000000000..e5af7d983 --- /dev/null +++ b/internal/proxy/reprobe.go @@ -0,0 +1,99 @@ +package proxy + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// reprobeCapabilitiesResponse mirrors the transcode node's re-probe answer, so +// an operator action does not have to know which node type it is talking to. +type reprobeCapabilitiesResponse struct { + // Resolved is the backend this proxy would now use. + Resolved string `json:"resolved"` + // CapabilityHash identifies the snapshot this re-probe published. + CapabilityHash string `json:"capability_hash"` +} + +// handleReprobeCapabilities discards this proxy's cached probe verdicts and +// rebuilds the capability snapshot against the live ffmpeg binary. +// +// A proxy runs ffmpeg for remux and Dolby Vision RPU strips, so an operator who +// swaps the binary under a running proxy needs a way to say so now instead of +// waiting out the 15-minute snapshot tick. It is deliberately *not* a hardware +// re-verification: a proxy never executes a hardware transcode, so its report +// carries no acceleration inventory to re-check — see +// buildCapabilitySnapshotLocked. The rebuild is therefore cheap, and one that +// does not finish still keeps the previously published hash. +// +// Unlike the transcode node this does not refuse while *jobs* are running. That +// guard is about encoder sessions: a proxy's jobs are remuxes and RPU strips, +// which hold no encoder slot for a probe's smoke encode to lose a race against. +func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { + // Held across the invalidation and the rebuild together, so a scheduled + // snapshot cannot interleave with them and publish a hash for a half-cleared + // cache. + s.capabilityBuildMu.Lock() + defer s.capabilityBuildMu.Unlock() + + // The mutex is not enough on its own: a probe outlives its caller by design + // — the singleflights run on background contexts so a canceled request + // cannot kill work another request is waiting on — so a capability request + // abandoned mid-probe releases this mutex while ffmpeg is still running. + // Invalidating then would start a second matrix beside the first. A proxy no + // longer launches the hardware walk that made this expensive, so in practice + // the count is zero; the gate stays because it is the shared contract with + // the transcode node's route and costs one comparison. + if busy := s.probesInFlight(); busy > 0 { + slog.InfoContext(r.Context(), "proxy capability re-probe refused while probes are still running", + "component", "proxy", "probes_in_flight", busy) + http.Error(w, fmt.Sprintf( + "node is running %d probe(s); starting another beside them would report working hardware as failed. Retry shortly.", + busy), http.StatusConflict) + return + } + + playback.InvalidateHWProbeCache() + tonemap.InvalidateProbeCache() + // The resource sampler retires nvidia-smi after repeated failure, and a + // driver that was broken at start is exactly what an operator reaches for + // this route after. A proxy samples the same GPU a transcode node does — it + // reports utilization on /health even though it never transcodes — so + // without this nudge it keeps reporting no GPU until the breaker's own retry + // interval. + s.metrics.RetrySources() + + // buildCapabilitySnapshotLocked owns the probe deadline, so a re-probe can + // never cost more than a cold capability fetch already may. + info, err := s.buildCapabilitySnapshotLocked(r.Context()) + if err != nil { + slog.WarnContext(r.Context(), "proxy capability re-probe incomplete", "component", "proxy", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return + } + previous := s.storedCapabilityHash() + s.storeCapabilityHash(info.CapabilityHash) + slog.InfoContext(r.Context(), "proxy capabilities re-probed", "component", "proxy", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(reprobeCapabilitiesResponse{ + Resolved: info.Resolved, + CapabilityHash: info.CapabilityHash, + }); err != nil { + slog.WarnContext(r.Context(), "encode proxy re-probe result", "component", "proxy", "error", err) + } +} + +// probesInFlight counts the hardware and tone-map probes this process has +// claimed the encoder for. It is a method so a test can drive the refusal +// without reaching into either package's unexported singleflight. +func (s *Server) probesInFlight() int { + if s.countProbesInFlight != nil { + return s.countProbesInFlight() + } + return playback.HWProbesInFlight() + tonemap.ProbesInFlight() +} diff --git a/internal/proxy/reprobe_test.go b/internal/proxy/reprobe_test.go new file mode 100644 index 000000000..3e11fa775 --- /dev/null +++ b/internal/proxy/reprobe_test.go @@ -0,0 +1,109 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// A proxy runs ffmpeg for remux recipes, so it gets the same escape hatch a +// transcode node has: re-probe that binary now, publish the result, and let +// health advertise it immediately instead of at the next 15-minute tick. It is +// not a hardware re-verification — a proxy reports no hardware — but the route +// and its publishing contract are unchanged. +func TestProxyReprobeCapabilitiesRecomputesAndStoresHash(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + server.storeCapabilityHash("sha256:stale") + + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var result reprobeCapabilitiesResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode re-probe result: %v", err) + } + if result.CapabilityHash == "" || result.CapabilityHash == "sha256:stale" { + t.Fatalf("capability_hash = %q, want a recomputed hash", result.CapabilityHash) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != result.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the re-probed %q", got, result.CapabilityHash) + } +} + +// A re-probe that cannot finish must answer 503 and keep the published hash: an +// unfinished probe is not evidence the proxy's ffmpeg lost anything. +func TestProxyReprobeCapabilitiesKeepsHashOnIncompleteProbe(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + server.refreshCapabilitySnapshot(context.Background()) + published := decodeProxyHealth(t, server).CapabilitiesHash + if published == "" { + t.Fatal("no capability hash was published before the canceled re-probe") + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil).WithContext(canceled) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503; body = %s", recorder.Code, recorder.Body.String()) + } + if got := decodeProxyHealth(t, server).CapabilitiesHash; got != published { + t.Fatalf("health capabilities_hash = %q after an unfinished re-probe, want %q", got, published) + } +} + +// The route executes ffmpeg, so it stays inside the bearer-authed admin group. +func TestProxyReprobeCapabilitiesRequiresBearer(t *testing.T) { + server := newCapabilityProxyServer(t, "capability-secret") + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil)) + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) + } +} + +// A probe outlives its caller by design, so a capability request abandoned +// mid-probe releases capabilityBuildMu while ffmpeg is still running. A +// re-probe arriving then would start a second matrix beside the first, and two +// contending for one card publish a hardware failure for hardware that is fine. +// A proxy no longer starts the walk that made this expensive, but the gate is +// the shared contract with the transcode node's route and still holds here. +func TestProxyReprobeCapabilitiesRefusedWhileProbesAreRunning(t *testing.T) { + const secret = "capability-secret" + server := newCapabilityProxyServer(t, secret) + server.storeCapabilityHash("sha256:previous") + server.countProbesInFlight = func() int { return 1 } + + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+secret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while a probe is still running", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } + + // And it is not a permanent refusal: once the detached probe lands, the + // re-probe an operator asked for goes through. + server.countProbesInFlight = func() int { return 0 } + recorder = httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d after the probes finished, want 200", recorder.Code) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 49f1afadb..ae8556273 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -13,15 +13,18 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/go-chi/chi/v5" "github.com/go-chi/cors" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/downloads" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" @@ -53,6 +56,25 @@ type Server struct { downloadBandwidth *downloads.BandwidthManager downloadServerBPS int64 downloadUserBPS int64 + + // capabilityHash is the last computed capability snapshot's hash, published + // by /health without probing. Nil until the first snapshot or capability + // request completes. + capabilityHash atomic.Pointer[string] + + // metrics samples host and GPU resources in the background. Nil until + // StartMetricsSampler runs, which leaves health exactly as it was before. + metrics *nodemetrics.Sampler + + // capabilityBuildMu serializes capability assemblies with each other, so an + // operator re-probe cannot run its ffmpeg probes beside the scheduled + // snapshot's. The probe caches no longer coalesce the two — bumping the + // invalidation generation is what makes the re-probe honest — so without + // this they would genuinely run at once. + capabilityBuildMu sync.Mutex + // countProbesInFlight overrides the detached-probe count the re-probe route + // refuses on. Tests set it; production leaves it nil. + countProbesInFlight func() int } type remoteArtifactMissReporter interface { @@ -154,6 +176,10 @@ func (s *Server) Handler() http.Handler { MaxAge: 86400, })) r.Get("/api/v1/health", s.handleHealth) + // Unauthenticated, matching the API listener's own /metrics posture: a + // scrape target that needs a credential is a scrape target that goes + // unmonitored, and the exposure is host resource counters, not media. + r.Method(http.MethodGet, "/metrics", promhttp.Handler()) r.Group(func(r chi.Router) { // Streaming and download bytes count toward the node's measured egress. r.Use(s.meterEgress) @@ -185,6 +211,8 @@ func (s *Server) Handler() http.Handler { r.Use(s.requireBearer) r.Get("/hw-capabilities", s.handleHWCapabilities) r.Post("/admin/force-reload", s.handleForceReload) + r.Post("/admin/reload-config", s.handleReloadConfig) + r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) }) return r @@ -198,23 +226,175 @@ func (s *Server) Handler() http.Handler { // whether the proxy it just picked can run the transformations a plan froze, so // a pool whose proxies carry a different ffmpeg build (a rolling upgrade, a // custom image) would fail at stream time rather than at selection time. +// +// The report is deliberately hardware-free. A proxy relays bytes and runs +// identity/remux recipes; it never executes a hardware transcode, and nothing +// on the API side reads a proxy's acceleration fields. Probing them anyway cost +// every proxy a full GPU smoke-encode matrix every 15 minutes to produce an +// answer no planner consults. func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { - ffmpegPath := "" - if cfg := s.watcher.Config(); cfg != nil { - ffmpegPath = cfg.Playback.FFmpegPath + info, err := s.buildCapabilitySnapshot(r.Context()) + if err != nil { + // An incomplete probe would hash differently from the same ffmpeg probed + // successfully, so serving it would announce a capability change that did + // not happen. + slog.WarnContext(r.Context(), "proxy capability probe incomplete", "component", "proxy", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return } - info := playback.DetectHWAccelWithFFmpeg(ffmpegPath) - info.Transformations = playback.ProbeTransformationRegistryV3(r.Context(), ffmpegPath).Advertised() + // A served report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(info); err != nil { slog.WarnContext(r.Context(), "encode proxy capabilities", "component", "proxy", "error", err) } } +// buildCapabilitySnapshot assembles this proxy's capability report and its +// identity hash. It is the single assembly used by both the capability endpoint +// and the background snapshot, so the hash a health response advertises always +// describes the payload the endpoint would serve. +// +// An error means the probe did not finish — a caller that gave up, or an ffmpeg +// slower than the probe deadline — not that the proxy lost a capability. The +// caller must keep the previous hash rather than publish the partial report, +// exactly as a transcode node does. +func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { + s.capabilityBuildMu.Lock() + defer s.capabilityBuildMu.Unlock() + return s.buildCapabilitySnapshotLocked(ctx) +} + +// buildCapabilitySnapshotLocked is buildCapabilitySnapshot's body. Callers must +// hold capabilityBuildMu; the re-probe takes it itself so its cache +// invalidation and its rebuild are one step no other builder can interleave +// with. +func (s *Server) buildCapabilitySnapshotLocked(ctx context.Context) (playback.HWAccelInfo, error) { + ffmpegPath := "" + if cfg := s.watcher.Config(); cfg != nil { + ffmpegPath = cfg.Playback.FFmpegPath + } + // Hardware acceleration is not probed here, and the report says so rather + // than leaving the fields unset by accident. A proxy relays streams and runs + // identity/remux recipes on ffmpeg; it never executes a hardware transcode, + // and the only field anything reads off this report is Transformations — + // planIdentityProxySessionV3 filters proxies by their advertised + // transformations and consults nothing else. So there is no inventory to + // report and nothing a GPU smoke-encode matrix could tell the planner. + // + // The consequence worth stating: the hash now tracks only what this proxy + // can *do*. A reboot, a renumbered render node, or a card appearing on the + // host no longer moves it, so the API refetches a proxy's report exactly + // when its ffmpeg's abilities changed. Nothing derived from the host may + // enter this report, the advertised budget below included — see + // playback.RegistryCapabilityEndpointTimeout. + info := playback.HWAccelInfo{ + Resolved: playback.HWAccelNone, + Source: "local", + } + // One deadline over the registry probe, matching the transcode node: it has + // its own internal per-command bounds, but only a shared budget keeps the + // whole rebuild inside the window a caller was told to allow, and only a + // deadline the builder owns bounds the background snapshot, whose context + // lives as long as the process. It is the registry-only budget — the same + // formula the transcode node uses, with no hardware in it — and the same one + // advertised below, so a caller's allowance and this deadline cannot drift. + ctx, cancel := context.WithTimeout(ctx, playback.RegistryCapabilityEndpointTimeout()) + defer cancel() + // A registry probe that ran out of budget is refused rather than published: + // it marks transformations unavailable, which is byte-identical to an ffmpeg + // that genuinely cannot run them, so hashing it would announce a change that + // did not happen and drop this proxy out of remux eligibility. + registry, err := playback.ProbeTransformationRegistryWithToneMapV3Result(ctx, ffmpegPath, nil) + if err != nil { + return playback.HWAccelInfo{}, err + } + info.Transformations = registry.Advertised() + // Advertised before the hash is taken, because it is part of what the hash + // covers: a build that needs longer reaches the sweep rather than sitting + // behind an unchanged identity. + info.ProbeRequestTimeoutMillis = playback.RegistryCapabilityRequestTimeout().Milliseconds() + info.CapabilityHash = playback.ComputeCapabilityHash(info) + return info, nil +} + +// capabilitySnapshotInterval is how often the proxy recomputes its capability +// snapshot. It exists to notice the ffmpeg underneath a long-running proxy +// changing — a swapped binary, a rolling image upgrade — without waiting for a +// restart. The transformation registry re-execs ffmpeg every time, which is the +// other reason a snapshot that did not finish must not be published. +const capabilitySnapshotInterval = 15 * time.Minute + +// StartCapabilitySnapshots keeps the capability hash published by /health +// current, in the background, until ctx is canceled. +func (s *Server) StartCapabilitySnapshots(ctx context.Context) { + if s == nil || ctx == nil { + return + } + go func() { + s.refreshCapabilitySnapshot(ctx) + ticker := time.NewTicker(capabilitySnapshotInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.refreshCapabilitySnapshot(ctx) + } + } + }() +} + +func (s *Server) refreshCapabilitySnapshot(ctx context.Context) { + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + // Keep the previous hash: a failed probe is not evidence this proxy's + // ffmpeg changed, and republishing a degraded one would make the API + // refetch a report that lost nothing. + slog.WarnContext(ctx, "proxy capability snapshot incomplete", "component", "proxy", "error", err) + return + } + if previous := s.storedCapabilityHash(); previous != "" && previous != info.CapabilityHash { + slog.InfoContext(ctx, "proxy capabilities changed", "component", "proxy", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + } + s.storeCapabilityHash(info.CapabilityHash) +} + +// storedCapabilityHash returns the last published capability hash, or empty +// when none has been computed yet. +func (s *Server) storedCapabilityHash() string { + if hash := s.capabilityHash.Load(); hash != nil { + return *hash + } + return "" +} + +func (s *Server) storeCapabilityHash(hash string) { + s.capabilityHash.Store(&hash) +} + type healthResponse struct { Status string `json:"status"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` + // CapabilitiesHash identifies this proxy's last computed capability + // snapshot. It is read from the stored snapshot only — health must stay a + // cheap liveness answer, so it never triggers a probe — and is empty until + // the first background snapshot completes. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` + // System and GPU are this proxy's last resource sample, read from the + // published snapshot for the same reason as the hash above. A proxy runs + // ffmpeg too (remux, Dolby Vision RPU strip), so it reports GPU usage on the + // same code path a transcode node does. + // + // This route takes no credential, so the sample is path-free: disk entries + // carry their role and their fill, never where they are mounted. See + // nodemetrics.Snapshot.RedactPaths. + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { @@ -222,14 +402,49 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { if s.tracker != nil { activeJobs = s.tracker.ActiveCount() } + snapshot := s.metrics.Snapshot().RedactPaths() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(healthResponse{ - Status: "ok", - ActiveJobs: activeJobs, - EgressKbps: s.egress.RateKbps(), + Status: "ok", + ActiveJobs: activeJobs, + EgressKbps: s.egress.RateKbps(), + CapabilitiesHash: s.storedCapabilityHash(), + System: snapshot.System, + GPU: snapshot.GPU, }) } +// StartMetricsSampler begins background resource sampling until ctx is +// canceled, and publishes the readings on /health, /status and /metrics. +// +// A proxy's only working directory is the subtitle/remux scratch under the +// configured transcode dir, so that is the mount it samples; media roots belong +// to the API host, which is the process that knows what the library is. +func (s *Server) StartMetricsSampler(ctx context.Context) { + if s == nil || ctx == nil { + return + } + // Read per sample, not captured here: playback.transcode_dir is + // hot-reloadable, and a proxy that snapshotted it at startup would go on + // measuring a volume nothing writes to. + scratchDir := func() string { + if s.watcher == nil { + return "" + } + cfg := s.watcher.Config() + if cfg == nil { + return "" + } + return cfg.Playback.TranscodeDir + } + s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ + ScratchDir: scratchDir, + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: playback.SamplerDeviceIdentities, + }) + s.metrics.Start(ctx) +} + // requireBearer checks Authorization: Bearer {secret} for admin endpoints. func (s *Server) requireBearer(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -705,6 +920,14 @@ func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, cl io.Copy(w, resp.Body) } +// handleReloadConfig re-reads this proxy's configuration. A proxy's force +// reload is already config-only — it holds no transcode sessions to tear down — +// so this is the same work under the name the control plane uses on both node +// types, which saves the API branching on node type for its own housekeeping. +func (s *Server) handleReloadConfig(w http.ResponseWriter, r *http.Request) { + s.handleForceReload(w, r) +} + func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { if err := s.watcher.ForceReload(r.Context()); err != nil { http.Error(w, "reload failed: "+err.Error(), http.StatusInternalServerError) @@ -715,12 +938,23 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { } type statusResponse struct { - ActiveSessions int `json:"active_sessions"` + ActiveSessions int `json:"active_sessions"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { + // NewServer accepts a nil tracker and handleHealth already tolerates one; + // this must too, or the same construction that answers /health panics here. + activeSessions := 0 + if s.tracker != nil { + activeSessions = s.tracker.ActiveCount() + } + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(statusResponse{ - ActiveSessions: s.tracker.ActiveCount(), + ActiveSessions: activeSessions, + System: snapshot.System, + GPU: snapshot.GPU, }) } diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index e102bf277..d6bdc4a30 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -1,9 +1,12 @@ # fixture 1 POST /admin/force-reload non-media +POST /admin/reload-config non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media GET /stream/direct/{token} media playback viewer_egress true true HEAD /stream/direct/{token} media playback viewer_egress true true @@ -23,10 +26,13 @@ HEAD /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true GET /stream/v3/{session_id}/segment/{name} media playback viewer_egress true true # fixture 2 POST /admin/force-reload non-media +POST /admin/reload-config non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media GET /downloads/file/{token} media transfer viewer_egress false true HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media GET /stream/direct/{token} media playback viewer_egress true true HEAD /stream/direct/{token} media playback viewer_egress true true diff --git a/internal/scanner/audiobook_test.go b/internal/scanner/audiobook_test.go index f4805bba9..f6c9ba719 100644 --- a/internal/scanner/audiobook_test.go +++ b/internal/scanner/audiobook_test.go @@ -77,9 +77,7 @@ func (f *fakeScannerCoverCacher) CacheAudiobookCover(_ context.Context, data []b func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nprintf cover-bytes\n"), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, "#!/bin/sh\nprintf cover-bytes\n") exec := &fakeAudiobookPosterExec{} cacher := &fakeScannerCoverCacher{} @@ -122,9 +120,7 @@ func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { func TestApplyAudiobookEmbeddedCoverPreservesExistingPoster(t *testing.T) { dir := t.TempDir() ffmpegPath := filepath.Join(dir, "ffmpeg") - if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nprintf cover-bytes\n"), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, "#!/bin/sh\nprintf cover-bytes\n") reader := &fakeAudiobookPosterReader{posterPath: "local/audiobooks/content-1/poster/original.webp"} exec := &fakeAudiobookPosterExec{} diff --git a/internal/scanner/fake_tool_test.go b/internal/scanner/fake_tool_test.go new file mode 100644 index 000000000..141f14014 --- /dev/null +++ b/internal/scanner/fake_tool_test.go @@ -0,0 +1,35 @@ +package scanner + +import ( + "os" + "syscall" + "testing" +) + +// writeFakeTool writes an executable stub for ffmpeg or ffprobe and fails the +// test if it cannot. +// +// The fork lock is what makes this different from a plain os.WriteFile, and it +// is the whole point of the helper. execve refuses a file that anyone still has +// open for writing, with ETXTBSY — "text file busy". Go opens the file +// O_CLOEXEC, so the fd is not meant to outlive an exec, but O_CLOEXEC only +// takes effect at the child's execve: a fork landing between this open and its +// close leaves that child holding a copy of the writing fd for the whole time +// it takes to exec something else. A test that then runs the stub it just +// wrote fails, and it fails for whichever test happened to be next to a fork — +// never the same one twice, and never on a laptop running one package at a +// time. +// +// syscall.ForkLock is the lock os/exec takes around fork for exactly this class +// of problem. Holding it for the write means no fork can observe the fd at all, +// so the window does not exist rather than being waited out: no retry, no +// sleep, no exec of a stub whose side effects a test is about to assert on. +func writeFakeTool(t *testing.T, path, script string) { + t.Helper() + syscall.ForkLock.Lock() + err := os.WriteFile(path, []byte(script), 0o755) + syscall.ForkLock.Unlock() + if err != nil { + t.Fatalf("writing fake tool %s: %v", path, err) + } +} diff --git a/internal/scanner/probe_duration_test.go b/internal/scanner/probe_duration_test.go index 121b4206c..fc1506cbc 100644 --- a/internal/scanner/probe_duration_test.go +++ b/internal/scanner/probe_duration_test.go @@ -315,9 +315,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "long.mp4") if err != nil { @@ -439,9 +437,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") if err != nil { @@ -469,9 +465,7 @@ case " $* " in ;; esac ` - if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { - t.Fatalf("writing fake ffprobe: %v", err) - } + writeFakeTool(t, ffprobePath, script) probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") if err != nil { diff --git a/internal/scanner/probe_primary_video_test.go b/internal/scanner/probe_primary_video_test.go index ee9f0371d..177861923 100644 --- a/internal/scanner/probe_primary_video_test.go +++ b/internal/scanner/probe_primary_video_test.go @@ -3,7 +3,6 @@ package scanner import ( "context" "errors" - "os" "path/filepath" "testing" "time" @@ -86,9 +85,7 @@ func TestProbePrimaryVideoTrackRejectsMalformedJSON(t *testing.T) { func TestProbePrimaryVideoTrackHonorsCallerTimeout(t *testing.T) { ffprobe := filepath.Join(t.TempDir(), "ffprobe") - if err := os.WriteFile(ffprobe, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { - t.Fatal(err) - } + writeFakeTool(t, ffprobe, "#!/bin/sh\nexec sleep 30\n") ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() _, err := ProbePrimaryVideoTrack(ctx, ffprobe, "movie.mkv") @@ -101,8 +98,6 @@ func writePrimaryVideoFFprobe(t *testing.T, output string) string { t.Helper() path := filepath.Join(t.TempDir(), "ffprobe") script := "#!/bin/sh\nprintf '%s' '" + output + "'\n" - if err := os.WriteFile(path, []byte(script), 0o755); err != nil { - t.Fatal(err) - } + writeFakeTool(t, path, script) return path } diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index ee41fa969..ea0bf14cc 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/Silo-Server/silo-server/internal/models" @@ -129,9 +130,14 @@ type playbackProbeFileRepository interface { // PlaybackProbeEnsurer repairs missing playback-critical probe metadata on // demand by running a local ffprobe and persisting the result. type PlaybackProbeEnsurer struct { - fileRepo playbackProbeFileRepository + fileRepo playbackProbeFileRepository + // ffprobePath and ffmpegPath are the binaries captured when the ensurer was + // built. They are the fallback only: livePaths, once SetFFmpegPath has been + // called, supersedes them so a changed playback.ffmpeg_path reaches probe + // repair and the copy-safety scan without a server restart. ffprobePath string ffmpegPath string + livePaths atomic.Pointer[probeBinaries] timeout time.Duration // probeFile is the ffprobe entry point; nil means the package's ProbeFile. // Tests substitute it to drive the coalescing behavior deterministically. @@ -194,6 +200,42 @@ func NewPlaybackProbeEnsurer(fileRepo *FileRepository, ffprobePath, ffmpegPath s return e } +// probeBinaries is the pair of executables a probe run needs. They move +// together because both are derived from the single playback.ffmpeg_path +// setting. +type probeBinaries struct { + ffprobePath string + ffmpegPath string +} + +// SetFFmpegPath points probe repair and the copy-safety scan at a different +// FFmpeg install. Wiring it to the config watcher is what lets a changed +// playback.ffmpeg_path take effect without restarting the server. An in-flight +// probe keeps the binary it started with; the next one picks up the new path. +func (e *PlaybackProbeEnsurer) SetFFmpegPath(ffmpegPath string) { + if e == nil { + return + } + ffmpegPath = strings.TrimSpace(ffmpegPath) + e.livePaths.Store(&probeBinaries{ + ffprobePath: FFprobePathFromFFmpeg(ffmpegPath), + ffmpegPath: ffmpegPath, + }) +} + +// binaries returns the executables this probe run should use: the live +// configuration when one has been installed, otherwise the pair captured at +// construction. +func (e *PlaybackProbeEnsurer) binaries() probeBinaries { + if e == nil { + return probeBinaries{} + } + if live := e.livePaths.Load(); live != nil { + return *live + } + return probeBinaries{ffprobePath: e.ffprobePath, ffmpegPath: e.ffmpegPath} +} + // Ensure repairs playback-critical probe metadata and resolves the H.264 // copy-safety verdict. Use it where a play is being prepared — the planner // consumes the verdict to decide whether a video stream-copy is safe. @@ -249,7 +291,7 @@ func (e *PlaybackProbeEnsurer) EnsureCopySafetyCached(ctx context.Context, file // real work for this file: an H.264 video whose verdict is neither cached nor // persisted, on a server that has an ffmpeg to scan with. func (e *PlaybackProbeEnsurer) NeedsCopySafetyScan(file *models.MediaFile) bool { - if e == nil || strings.TrimSpace(e.ffmpegPath) == "" || !needsCopySafetyProbe(file) { + if e == nil || strings.TrimSpace(e.binaries().ffmpegPath) == "" || !needsCopySafetyProbe(file) { return false } _, known := e.knownCopySafetyVerdict(file) @@ -269,10 +311,11 @@ func (e *PlaybackProbeEnsurer) ScanCopySafety(ctx context.Context, file *models. if e == nil || file == nil { return false, false, nil } - if strings.TrimSpace(e.ffmpegPath) == "" { + ffmpegPath := strings.TrimSpace(e.binaries().ffmpegPath) + if ffmpegPath == "" { return false, false, errCopySafetyScanUnavailable } - return e.scanAndPersistCopySafety(ctx, file) + return e.scanAndPersistCopySafety(ctx, file, ffmpegPath) } // KnownCopySafetyVerdict answers the copy-safety question for a file without @@ -325,8 +368,11 @@ func (e *PlaybackProbeEnsurer) ensureProbeRepair(ctx context.Context, file *mode } current := file - if NeedsCriticalProbeRepair(file) && strings.TrimSpace(e.ffprobePath) != "" { - repaired, err := e.ensureCriticalProbe(ctx, file) + // One snapshot per repair: the guard and the ffprobe run must see the same + // binaries, or a SetFFmpegPath between them hands probeFile an empty path. + ffprobePath := strings.TrimSpace(e.binaries().ffprobePath) + if NeedsCriticalProbeRepair(file) && ffprobePath != "" { + repaired, err := e.ensureCriticalProbe(ctx, file, ffprobePath) if err != nil { return file, err } @@ -348,7 +394,7 @@ func (e *PlaybackProbeEnsurer) ensureProbeRepair(ctx context.Context, file *mode // cancellation while waiting. Inside the flight the row is re-read first, so a // caller holding a stale snapshot of an already-repaired file spawns no ffprobe // at all. -func (e *PlaybackProbeEnsurer) ensureCriticalProbe(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { +func (e *PlaybackProbeEnsurer) ensureCriticalProbe(ctx context.Context, file *models.MediaFile, ffprobePath string) (*models.MediaFile, error) { sharedCtx := context.WithoutCancel(ctx) revisionKey := tonemap.RevisionForFile(file).Fingerprint() resultCh := e.probeRepair.DoChan(revisionKey, func() (any, error) { @@ -386,7 +432,7 @@ func (e *PlaybackProbeEnsurer) ensureCriticalProbe(ctx context.Context, file *mo if probeFile == nil { probeFile = ProbeFile } - probe, err := probeFile(probeCtx, e.ffprobePath, current.FilePath) + probe, err := probeFile(probeCtx, ffprobePath, current.FilePath) if err != nil || probe == nil { return nil, err } @@ -415,7 +461,8 @@ func (e *PlaybackProbeEnsurer) ensureCriticalProbe(ctx context.Context, file *mo // media_files row, and only then runs the bitstream scan — so a restart no // longer re-reads the opening seconds of every browsed H.264 file. func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { - if !needsCopySafetyProbe(file) || strings.TrimSpace(e.ffmpegPath) == "" { + ffmpegPath := strings.TrimSpace(e.binaries().ffmpegPath) + if !needsCopySafetyProbe(file) || ffmpegPath == "" { return file, nil } @@ -424,7 +471,7 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model return fileWithMultiplePPS(file, multi), nil } - multi, stale, err := e.scanAndPersistCopySafety(ctx, file) + multi, stale, err := e.scanAndPersistCopySafety(ctx, file, ffmpegPath) if err != nil { // Unknown safety must not fail open to the video-copy path this probe is // intended to guard. Leave MultiplePPS unset and do not cache or persist @@ -489,7 +536,7 @@ func copySafetyFlightKey(file *models.MediaFile) string { // A write refused as stale is neither memoized nor reported as a verdict: the // row has moved to a generation this scan never read, and both the memo and any // downstream notification would be facts about bytes nobody is serving. -func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, bool, error) { +func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile, ffmpegPath string) (bool, bool, error) { fileID := file.ID filePath := file.FilePath fileSize := file.FileSize @@ -501,7 +548,7 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil timeout = 30 * time.Second } scanCtx, cancel := context.WithTimeout(ctx, timeout) - multi, err := DetectMultiplePPSH264(scanCtx, e.ffmpegPath, filePath) + multi, err := DetectMultiplePPSH264(scanCtx, ffmpegPath, filePath) cancel() if err != nil { return copySafetyOutcome{}, err diff --git a/internal/scanner/probe_repair_copy_safety_cached_test.go b/internal/scanner/probe_repair_copy_safety_cached_test.go index 64e36ad29..4d2a55ac3 100644 --- a/internal/scanner/probe_repair_copy_safety_cached_test.go +++ b/internal/scanner/probe_repair_copy_safety_cached_test.go @@ -2,6 +2,7 @@ package scanner import ( "context" + "errors" "path/filepath" "testing" "time" @@ -213,3 +214,45 @@ func TestVideoCopySafetyUnknownIgnoresAudioOnlyFiles(t *testing.T) { t.Fatal("VideoCopySafetyUnknown() = true for an audio-only file, want false") } } + +// SetFFmpegPath is what makes playback.ffmpeg_path take effect without a +// server restart: the ensurer is built with the path captured at boot, and a +// later config change has to reach both the probe and the copy-safety scan. +func TestSetFFmpegPathOverridesBootPaths(t *testing.T) { + ensurer := NewPlaybackProbeEnsurer(nil, "boot-ffprobe", "boot-ffmpeg", time.Second) + + if got := ensurer.binaries(); got.ffmpegPath != "boot-ffmpeg" || got.ffprobePath != "boot-ffprobe" { + t.Fatalf("binaries() before reload = %+v, want the boot pair", got) + } + + ensurer.SetFFmpegPath("/opt/jellyfin-ffmpeg/ffmpeg") + + got := ensurer.binaries() + if got.ffmpegPath != "/opt/jellyfin-ffmpeg/ffmpeg" { + t.Fatalf("binaries().ffmpegPath = %q, want the reloaded path", got.ffmpegPath) + } + if want := filepath.Join("/opt/jellyfin-ffmpeg", "ffprobe"); got.ffprobePath != want { + t.Fatalf("binaries().ffprobePath = %q, want %q derived from the reloaded ffmpeg", got.ffprobePath, want) + } +} + +// A reload that blanks the FFmpeg path disables the copy-safety scan, the same +// as booting without one — the verdict stays unknown instead of being guessed. +func TestSetFFmpegPathEmptyDisablesCopySafetyScan(t *testing.T) { + ffmpegPath, _ := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + ensurer := NewPlaybackProbeEnsurer(nil, "ffprobe", ffmpegPath, time.Second) + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + if !ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = false with a configured ffmpeg, want true") + } + + ensurer.SetFFmpegPath("") + + if ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = true after the path was cleared, want false") + } + if _, _, err := ensurer.ScanCopySafety(context.Background(), file); !errors.Is(err, errCopySafetyScanUnavailable) { + t.Fatalf("ScanCopySafety() error = %v, want errCopySafetyScanUnavailable", err) + } +} diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index 20f2d5e6e..554e13665 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -32,9 +32,7 @@ func fakeFFmpeg(t *testing.T, stdoutPayload string, delay time.Duration) (string sleep = fmt.Sprintf("sleep %.2f\n", delay.Seconds()) } script := fmt.Sprintf("#!/bin/sh\necho run >> %q\n%sprintf '%s'\n", logPath, sleep, stdoutPayload) - if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, script) return ffmpegPath, func() int { return countFFmpegRuns(t, logPath) } } @@ -51,9 +49,7 @@ func fakeFFmpegGated(t *testing.T, stdoutPayload string) (ffmpegPath string, run // The invocation is logged before the gate so the log is the signal that // this process has started, not that it has finished. script := fmt.Sprintf("#!/bin/sh\necho run >> %q\nwhile [ ! -f %q ]; do sleep 0.01; done\nprintf '%s'\n", logPath, releasePath, stdoutPayload) - if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { - t.Fatalf("write gated fake ffmpeg: %v", err) - } + writeFakeTool(t, ffmpegPath, script) runs = func() int { return countFFmpegRuns(t, logPath) } release = func() { if err := os.WriteFile(releasePath, nil, 0o644); err != nil { diff --git a/internal/taskmanager/tasks/cache_metadata_images.go b/internal/taskmanager/tasks/cache_metadata_images.go index 58d37d98e..5bd72eab8 100644 --- a/internal/taskmanager/tasks/cache_metadata_images.go +++ b/internal/taskmanager/tasks/cache_metadata_images.go @@ -4,25 +4,112 @@ import ( "context" "encoding/json" "fmt" + "math" "os" + "runtime" + "runtime/debug" "time" "github.com/google/uuid" "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/taskmanager" ) const ( cacheMetadataImagesIntervalMs = int64(60 * 1000) - // Claim only work that can start immediately. Claiming a large queue page - // stamps one lease on every row up front; with two workers, the unstarted - // tail could expire and be reclaimed before this execution reaches it. - cacheMetadataImagesClaimLimit = 2 - cacheMetadataImagesWorkers = 2 cacheMetadataImagesMaxRuntime = 10 * time.Minute ) +// imageCacheWorkerMemoryBudget is the memory reserved per concurrent image +// job when sizing the pool against a detected memory bound. A job can hold +// the compressed download (capped at 25 MiB in imagecache), the libvips +// working set for the variant ladder, and — today — a full Go-heap decode of +// the original for thumbhash, which for a large provider poster reaches the +// low hundreds of MiB. 512 MiB per worker keeps even a burst of worst-case +// originals from consuming the whole budget, since the baseline server also +// lives inside it. +const imageCacheWorkerMemoryBudget = 512 << 20 + +// imageCacheWorkerCount sizes the image-cache worker pool for the host. Each +// job downloads an original (30s timeout in imagecache) and runs a libvips +// WEBP encode ladder, so the work is a CPU/network mix: 4× the scheduler's +// CPU count keeps cores busy while other workers wait on downloads, and the +// cap keeps a many-core server from monopolizing provider connections. The +// previous fixed pool of 2 measured roughly 60 images/minute on a ~600k-item +// library; 48 workers measured roughly 2,900/minute over a 60-minute window +// (RXWatcher/silo-server@3b377f5c2). +// +// memoryBytes, when positive, is the tightest detected memory bound for this +// process (GOMEMLIMIT, cgroup limit, or system memory) and caps the pool at +// one worker per imageCacheWorkerMemoryBudget so a many-core container with a +// small memory limit cannot be OOM-killed by concurrent decodes. +// +// The floor of 2 is the pool size this task shipped with, and it deliberately +// overrides the per-worker budget below 2×imageCacheWorkerMemoryBudget: a +// sub-1GiB deployment ran 2 workers before this sizing existed, so the memory +// cap never reduces such a host below its long-standing baseline. The budget +// is a sizing heuristic for how far to scale up, not a reservation. +func imageCacheWorkerCount(numCPU int, memoryBytes int64) int { + workers := min(48, 4*max(numCPU, 1)) + if memoryBytes > 0 { + workers = min(workers, max(int(memoryBytes/imageCacheWorkerMemoryBudget), 2)) + } + return workers +} + +// detectImageCacheMemoryBytes returns the tightest memory bound the process +// can see, or 0 when none is detectable (macOS dev boxes, bare Linux without +// cgroups). Every source is consulted and the smallest wins, because none +// implies the others: GOMEMLIMIT can be set looser than a cgroup limit, and +// the effective cgroup limit — own cgroup, ancestors, and root, so a systemd +// MemoryMax= or an inherited pod/slice limit binds, not just a namespaced +// container's root files — can sit above or below host memory. +func detectImageCacheMemoryBytes() int64 { + goLimit := int64(0) + if limit := debug.SetMemoryLimit(-1); limit < math.MaxInt64 { + goLimit = limit + } + hostTotal, _ := nodemetrics.ReadMeminfoTotalBytes("/proc/meminfo") + return tightestMemoryLimit(goLimit, nodemetrics.EffectiveMemoryLimitBytes(), hostTotal) +} + +// tightestMemoryLimit returns the smallest positive limit, or 0 when none is. +func tightestMemoryLimit(limits ...int64) int64 { + tightest := int64(0) + for _, limit := range limits { + if limit > 0 && (tightest == 0 || limit < tightest) { + tightest = limit + } + } + return tightest +} + +var cacheMetadataImagesWorkers = imageCacheWorkerCount(runtime.GOMAXPROCS(0), detectImageCacheMemoryBytes()) + +// cacheMetadataImagesClaimPerWorker sizes the queue page stamped with one +// lease up front. processClaimedJobs dispatches the page through a semaphore, +// so a page larger than the worker count keeps the pool saturated instead of +// waiting on every straggler in a worker-sized batch before the next page can +// be claimed. +// +// The page must drain inside metadata.ImageCacheLeaseDuration (15 minutes) or +// another worker reclaims the unstarted tail and duplicates it. Every job runs +// under metadata.ImageCacheJobTimeout (2 minutes), but that context cannot +// preempt the synchronous decode/encode segment (imageutil.Thumbhash, +// GenerateVariants take no context) — it only stops the job at the next +// context-aware step. That segment operates on inputs capped at 25 MiB +// (imagecache's download limit), so its overshoot is bounded by CPU speed, +// not by the network; 4 jobs per worker budgets the worst chain at +// 4 × (timeout + overshoot), which stays inside the lease with a generous +// overshoot allowance of nearly two minutes per job. +// TestImageCacheWorkerCount asserts the timeout part of this arithmetic +// against the exported constants. +const cacheMetadataImagesClaimPerWorker = 4 + +var cacheMetadataImagesClaimLimit = cacheMetadataImagesClaimPerWorker * cacheMetadataImagesWorkers + type MetadataImageCacheRunner interface { DrainUntilIdle(ctx context.Context, workerID string, claimLimit int, concurrency int, maxRuntime time.Duration, reportProgress metadata.ImageCacheRunProgressReporter) (metadata.ImageCacheRunStats, error) } diff --git a/internal/taskmanager/tasks/cache_metadata_images_test.go b/internal/taskmanager/tasks/cache_metadata_images_test.go index 0814eb10a..d1166b32f 100644 --- a/internal/taskmanager/tasks/cache_metadata_images_test.go +++ b/internal/taskmanager/tasks/cache_metadata_images_test.go @@ -115,11 +115,11 @@ func TestCacheMetadataImagesTaskReportsStats(t *testing.T) { if err := task.Execute(context.Background(), progress); err != nil { t.Fatalf("Execute() error = %v", err) } - if runner.claimLimit != 2 { - t.Fatalf("claimLimit = %d, want one immediately-startable job per worker", runner.claimLimit) + if runner.claimLimit != cacheMetadataImagesClaimLimit { + t.Fatalf("claimLimit = %d, want the shared page size %d", runner.claimLimit, cacheMetadataImagesClaimLimit) } - if runner.concurrency != 2 { - t.Fatalf("concurrency = %d, want 2", runner.concurrency) + if runner.concurrency != cacheMetadataImagesWorkers { + t.Fatalf("concurrency = %d, want the shared worker count %d", runner.concurrency, cacheMetadataImagesWorkers) } if runner.maxRuntime != 10*time.Minute { t.Fatalf("maxRuntime = %s, want 10m", runner.maxRuntime) @@ -161,8 +161,8 @@ func TestBackfillMetadataImagesTaskReportsDiscovery(t *testing.T) { if runner.maxRuntime != 0 { t.Fatalf("maxRuntime = %s, want no deadline for manual backfill", runner.maxRuntime) } - if runner.claimLimit != 2 { - t.Fatalf("claimLimit = %d, want one immediately-startable job per worker", runner.claimLimit) + if runner.claimLimit != cacheMetadataImagesClaimLimit { + t.Fatalf("claimLimit = %d, want the shared page size %d", runner.claimLimit, cacheMetadataImagesClaimLimit) } if len(runner.workerIDs) != 1 || !strings.Contains(runner.workerIDs[0], ":backfill:") { t.Fatalf("backfill worker IDs = %#v, want one execution-scoped backfill owner", runner.workerIDs) @@ -329,3 +329,69 @@ func TestBackfillMetadataImagesTaskProgressDoesNotFallWhenDiscoveryWidensTheRun( t.Fatalf("recovered report = %g, want 75", progress.percents[3]) } } + +func TestImageCacheWorkerCount(t *testing.T) { + const gib = int64(1) << 30 + cases := []struct { + numCPU int + memory int64 + want int + }{ + {numCPU: 0, memory: 0, want: 4}, // defensive floor; GOMAXPROCS never reports < 1 + {numCPU: 1, memory: 0, want: 4}, // small household box: modest but no longer crippled + {numCPU: 4, memory: 0, want: 16}, + {numCPU: 12, memory: 0, want: 48}, + {numCPU: 16, memory: 0, want: 48}, // cap: more cores must not monopolize providers + {numCPU: 64, memory: 0, want: 48}, + // A many-core container with a small memory limit is bounded by + // memory, never below the original pool of 2 — a sub-1GiB deployment + // already ran 2 workers before this change, so 2 is the floor, not 1. + {numCPU: 16, memory: 256 << 20, want: 2}, + {numCPU: 16, memory: 512 << 20, want: 2}, + {numCPU: 16, memory: 1 * gib, want: 2}, + {numCPU: 16, memory: 2 * gib, want: 4}, + {numCPU: 16, memory: 8 * gib, want: 16}, + {numCPU: 16, memory: 64 * gib, want: 48}, + // Plenty of memory but few cores: CPU stays the binding cap. + {numCPU: 2, memory: 64 * gib, want: 8}, + } + for _, tc := range cases { + if got := imageCacheWorkerCount(tc.numCPU, tc.memory); got != tc.want { + t.Errorf("imageCacheWorkerCount(%d, %d) = %d, want %d", tc.numCPU, tc.memory, got, tc.want) + } + } + // A claimed page is stamped with one lease up front, so it must fully + // drain before the lease expires or another worker reclaims the tail. + // The job timeout cannot preempt the synchronous decode/encode segment, + // so the timeout-based drain must leave real headroom under the lease + // for that bounded overshoot — at least one extra timeout per job. + drain := time.Duration(cacheMetadataImagesClaimPerWorker) * metadata.ImageCacheJobTimeout + overshootBudget := time.Duration(cacheMetadataImagesClaimPerWorker) * metadata.ImageCacheJobTimeout / 2 + if drain+overshootBudget > metadata.ImageCacheLeaseDuration { + t.Errorf("page drain %s plus overshoot budget %s must stay within the %s claim lease", drain, overshootBudget, metadata.ImageCacheLeaseDuration) + } + if cacheMetadataImagesClaimLimit != cacheMetadataImagesClaimPerWorker*cacheMetadataImagesWorkers { + t.Errorf("claim limit = %d, want %d per worker (%d)", cacheMetadataImagesClaimLimit, cacheMetadataImagesClaimPerWorker, cacheMetadataImagesClaimPerWorker*cacheMetadataImagesWorkers) + } +} + +// Conflicting memory bounds resolve to the smallest positive one: GOMEMLIMIT +// can be set looser than the cgroup limit and the cgroup limit can sit above +// host memory, so no single source can be preferred outright. +func TestTightestMemoryLimit(t *testing.T) { + const gib = int64(1) << 30 + cases := []struct { + limits []int64 + want int64 + }{ + {limits: []int64{4 * gib, 2 * gib, 8 * gib}, want: 2 * gib}, // smallest wins + {limits: []int64{0, 2 * gib, 0}, want: 2 * gib}, // zeros mean "no bound", not zero bytes + {limits: []int64{0, 0, 0}, want: 0}, // nothing detectable + {limits: []int64{6 * gib}, want: 6 * gib}, + } + for _, tc := range cases { + if got := tightestMemoryLimit(tc.limits...); got != tc.want { + t.Errorf("tightestMemoryLimit(%v) = %d, want %d", tc.limits, got, tc.want) + } + } +} diff --git a/internal/taskmanager/tasks/catalog_search_index.go b/internal/taskmanager/tasks/catalog_search_index.go index fc89a2b25..7e05c6293 100644 --- a/internal/taskmanager/tasks/catalog_search_index.go +++ b/internal/taskmanager/tasks/catalog_search_index.go @@ -25,7 +25,7 @@ func NewSyncCatalogSearchIndexTask(worker CatalogSearchIndexWorker) *SyncCatalog func (t *SyncCatalogSearchIndexTask) Key() string { return "sync_catalog_search_index" } func (t *SyncCatalogSearchIndexTask) Name() string { return "Sync Catalog Search Index" } func (t *SyncCatalogSearchIndexTask) Description() string { - return "Drains catalog search outbox events into the configured Meilisearch index." + return "Builds or upgrades the Meilisearch catalog index when needed, then syncs pending catalog changes." } func (t *SyncCatalogSearchIndexTask) Category() taskmanager.TaskCategory { return taskmanager.TaskCategoryLibrary @@ -52,7 +52,11 @@ func (t *SyncCatalogSearchIndexTask) Execute(ctx context.Context, progress taskm } stats, err := t.worker.SyncOutbox(ctx, progress) if err != nil { - progress.Report(100, fmt.Sprintf("Catalog search sync failed after %d events", stats.Events)) + if stats.RebuildAttempted { + progress.Report(100, fmt.Sprintf("Catalog search rebuild failed after %d documents", stats.DocumentCount)) + } else { + progress.Report(100, fmt.Sprintf("Catalog search sync failed after %d events", stats.Events)) + } return err } return nil diff --git a/internal/taskmanager/tasks/catalog_search_index_test.go b/internal/taskmanager/tasks/catalog_search_index_test.go new file mode 100644 index 000000000..b80730e61 --- /dev/null +++ b/internal/taskmanager/tasks/catalog_search_index_test.go @@ -0,0 +1,79 @@ +package tasks + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +type catalogSearchIndexWorkerStub struct { + shouldSync bool + syncStats catalog.CatalogSearchIndexSyncStats + syncErr error +} + +func (w catalogSearchIndexWorkerStub) ShouldSyncRun(context.Context) (bool, error) { + return w.shouldSync, nil +} + +func (w catalogSearchIndexWorkerStub) SyncOutbox(context.Context, catalog.SearchIndexProgressReporter) (catalog.CatalogSearchIndexSyncStats, error) { + return w.syncStats, w.syncErr +} + +type catalogSearchProgressStub struct { + message string +} + +func (p *catalogSearchProgressStub) Report(_ float64, message string) { p.message = message } +func (*catalogSearchProgressStub) SetResultData(json.RawMessage) {} + +func (catalogSearchIndexWorkerStub) Rebuild(context.Context, catalog.SearchIndexProgressReporter) (catalog.CatalogSearchIndexRebuildStats, error) { + return catalog.CatalogSearchIndexRebuildStats{}, nil +} + +func TestSyncCatalogSearchIndexTaskRunsAtStartupAndRetries(t *testing.T) { + task := NewSyncCatalogSearchIndexTask(catalogSearchIndexWorkerStub{shouldSync: true}) + triggers := task.DefaultTriggers() + if len(triggers) != 2 { + t.Fatalf("DefaultTriggers() length = %d, want 2", len(triggers)) + } + if triggers[0].Type != taskmanager.TriggerTypeStartup { + t.Fatalf("first trigger = %q, want startup", triggers[0].Type) + } + if triggers[1].Type != taskmanager.TriggerTypeInterval || triggers[1].IntervalMs != 60*1000 { + t.Fatalf("retry trigger = %#v, want 60s interval", triggers[1]) + } + shouldRun, err := task.ShouldRun(t.Context()) + if err != nil || !shouldRun { + t.Fatalf("ShouldRun() = %t, %v, want true, nil", shouldRun, err) + } +} + +func TestRebuildCatalogSearchIndexTaskRemainsManualOnly(t *testing.T) { + task := NewRebuildCatalogSearchIndexTask(catalogSearchIndexWorkerStub{}) + if triggers := task.DefaultTriggers(); len(triggers) != 0 { + t.Fatalf("DefaultTriggers() = %#v, want manual-only", triggers) + } +} + +func TestSyncCatalogSearchIndexTaskReportsAutomaticRebuildFailure(t *testing.T) { + worker := catalogSearchIndexWorkerStub{ + syncStats: catalog.CatalogSearchIndexSyncStats{ + RebuildAttempted: true, + DocumentCount: 42, + }, + syncErr: errors.New("indexing failed"), + } + progress := &catalogSearchProgressStub{} + err := NewSyncCatalogSearchIndexTask(worker).Execute(t.Context(), progress) + if !errors.Is(err, worker.syncErr) { + t.Fatalf("Execute() error = %v, want %v", err, worker.syncErr) + } + if progress.message != "Catalog search rebuild failed after 42 documents" { + t.Fatalf("progress message = %q", progress.message) + } +} diff --git a/internal/tonemap/preflight.go b/internal/tonemap/preflight.go index 9543269b7..0dcf76882 100644 --- a/internal/tonemap/preflight.go +++ b/internal/tonemap/preflight.go @@ -441,7 +441,8 @@ func sourceConversionPreflightArgs(request SourcePreflightRequest, position floa if device == "" { device = defaultDRIRenderDevice } - args = append(args, "-init_hw_device", qsvVAAPIInitDevice(device), "-init_hw_device", "qsv=qs@va", "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va") + args = append(args, QSVInitDeviceArgs(device)...) + args = append(args, "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va") if !request.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } @@ -449,7 +450,8 @@ func sourceConversionPreflightArgs(request SourcePreflightRequest, position floa if device == "" { device = defaultDRIRenderDevice } - args = append(args, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va") + args = append(args, VAAPIInitDeviceArgs("va", device)...) + args = append(args, "-filter_hw_device", "va") if !request.SoftwareVideoDecode { args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi") } diff --git a/internal/tonemap/probe.go b/internal/tonemap/probe.go index f4b426ff0..5196ef397 100644 --- a/internal/tonemap/probe.go +++ b/internal/tonemap/probe.go @@ -5,11 +5,14 @@ import ( "context" "encoding/base64" "errors" + "log/slog" "os" "os/exec" "slices" + "strconv" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -19,8 +22,14 @@ const ( probeCommandTimeout = 5 * time.Second probeNegativeTTL = 15 * time.Second probeTimeoutSlack = time.Second - probeEndpointSlack = 20 * time.Second - probeRequestSlack = 5 * time.Second + // probeEndpointSlack covers what a capability endpoint spends around the + // tone-map matrix itself: one bounded hardware detection walk (30s in + // playback.hwAccelWalkTimeout), the transformation registry's three 3s + // commands, and response overhead. It cannot be derived from those + // constants — playback imports this package, not the other way round — so + // it is raised whenever either budget grows. + probeEndpointSlack = 45 * time.Second + probeRequestSlack = 5 * time.Second ) // One deterministic 64x64 Main 10 HEVC frame. Keeping the compressed fixture @@ -46,6 +55,12 @@ var probeCache = struct { sync.Mutex entries map[string]probeCacheEntry group singleflight.Group + // generation counts invalidations and is part of every cache and + // singleflight key. It is what makes InvalidateProbeCache supersede a probe + // already in flight instead of merely clearing the map in front of it: the + // flight stores its inventory under the generation it started in, and the + // next caller asks a different key and therefore runs a fresh probe. + generation uint64 }{entries: make(map[string]probeCacheEntry)} // Probe returns the cached, smoke-tested tone-map capabilities for an FFmpeg @@ -57,8 +72,8 @@ func Probe(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevice stri // probeCached coalesces identical probes without allowing one caller's // cancellation to abort the shared work needed by other playback requests. func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevice string, run CommandRunner, now func() time.Time) (Capabilities, error) { - key := probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice) probeCache.Lock() + key := probeCacheKey(probeCache.generation, ffmpegPath, hardwareBackend, hardwareDevice) if cached, ok := probeCache.entries[key]; ok && probeCacheEntryCurrent(cached, now()) { result := append(Capabilities(nil), cached.capabilities...) probeCache.Unlock() @@ -66,6 +81,13 @@ func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevic } probeCache.Unlock() + // Claimed here, on the calling goroutine, not inside the function below. + // DoChan schedules that function and returns without waiting for it, so a + // caller whose context is already done returns while the probe has not + // reached its first line — and this probe runs real smoke encodes on the + // GPU. Anything that needs the encoder to itself has to see them; see + // ProbesInFlight. + probesInFlight.Add(1) resultCh := probeCache.group.DoChan(key, func() (any, error) { probeCache.Lock() cached, ok := probeCache.entries[key] @@ -90,8 +112,16 @@ func probeCached(ctx context.Context, ffmpegPath, hardwareBackend, hardwareDevic }) select { case <-ctx.Done(): + // The flight outlives this caller by design — its probe context is + // rooted at Background so a canceled request cannot kill work another + // request is waiting on — so the claim goes with the flight, not with us. + go func() { + <-resultCh + probesInFlight.Add(-1) + }() return nil, ctx.Err() case result := <-resultCh: + probesInFlight.Add(-1) if result.Err != nil { return nil, result.Err } @@ -131,7 +161,7 @@ func probeWithRunner( // probeCacheKey binds reusable capabilities to the resolved FFmpeg binary and // the driver facts for every configured hardware device. -func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { +func probeCacheKey(generation uint64, ffmpegPath, hardwareBackend, hardwareDevice string) string { binaryIdentity := strings.TrimSpace(ffmpegPath) if _, cacheKey, cacheable := ffmpegBinaryCacheKey(binaryIdentity); cacheable { binaryIdentity = cacheKey @@ -147,7 +177,61 @@ func probeCacheKey(ffmpegPath, hardwareBackend, hardwareDevice string) string { driverIdentities = append(driverIdentities, driverFingerprint(backend, configuredDevice)) } } - return strings.Join([]string{binaryIdentity, backend, device, strings.Join(driverIdentities, ",")}, "\x00") + return strings.Join([]string{ + strconv.FormatUint(generation, 10), + binaryIdentity, backend, device, strings.Join(driverIdentities, ","), + }, "\x00") +} + +// InvalidateProbeCache drops every cached tone-map inventory so the next probe +// re-runs its listings and single-frame conversions. +// +// A non-empty inventory is cached permanently (see probeCacheEntryCurrent), +// which is right for playback and wrong for an operator who just upgraded a +// driver or an FFmpeg build in place: the binary's identity key only changes +// when the file does, and a driver has no key at all. This is the seam an +// operator-triggered re-probe uses to force the matrix to run again. +// +// A probe already in flight is neither canceled nor discarded — canceling +// shared work would fail the unrelated playback request waiting on it — but it +// is superseded: bumping the generation moves every cache and singleflight key, +// so the in-flight probe stores its inventory where nothing will read it and +// the next caller runs a genuinely cold probe rather than joining the old +// flight. Without that, a re-probe racing a background capability fetch would +// republish the very inventory it was asked to discard. +func InvalidateProbeCache() { + probeCache.Lock() + defer probeCache.Unlock() + probeCache.generation++ + probeCache.entries = make(map[string]probeCacheEntry) +} + +// probesInFlight counts tone-map probes this process has claimed the encoder +// for, including ones whose caller has already given up on them. +var probesInFlight atomic.Int64 + +// ProbesInFlight reports how many tone-map probes this process has claimed the +// encoder for. +// +// The matrix behind one of these is real FFmpeg smoke encodes, and a probe +// outlives its caller by design: the singleflight task runs on a background +// context so a canceled request cannot kill work another request is waiting on. +// A component that released its own claim on the GPU when its call returned — +// the transcode node's capability build does exactly that — can therefore leave +// encodes running with nothing accounting for them. Anything that needs the +// encoder exclusively must add this to whatever else it counts as busy, or it +// will start a second matrix beside the first and publish the collision as a +// capability failure. +// +// It counts claims rather than processes and errs high, for the same reason its +// hardware-probe counterpart does: an overcount costs a retry, an undercount +// costs a false verdict. +func ProbesInFlight() int { + count := probesInFlight.Load() + if count < 0 { + return 0 + } + return int(count) } // probeCacheEntryCurrent reports whether a complete result or unexpired @@ -209,6 +293,32 @@ func ProbeEndpointTimeout(hardwareBackend, hardwareDevice string) time.Duration return ProbeTotalTimeout(backend, hardwareDevice) + probeEndpointSlack } +// ProbeRequestSlack is the transport and response margin a remote caller adds +// on top of a node's endpoint budget. Exported so playback can compose the same +// request budget without duplicating the number. +const ProbeRequestSlack = probeRequestSlack + +// MaxProbedDevices is the device count the ceiling on an advertised probe +// budget is derived from. +// +// The matrix grows with the device set, which has no hard limit — an operator +// can list as many render devices as the host has. This is the largest set a +// caller will keep waiting for, chosen well above any real GPU node so that a +// legitimate configuration never meets it and the ceiling only ever bounds a +// worker advertising something absurd. +const MaxProbedDevices = 16 + +// MaxProbeRequestTimeout is the largest budget a node may advertise and be +// believed, derived from the same formula the node itself uses so the two +// cannot drift apart. +func MaxProbeRequestTimeout() time.Duration { + devices := make([]string, 0, MaxProbedDevices) + for i := range MaxProbedDevices { + devices = append(devices, defaultDRIRenderDevice+strconv.Itoa(i)) + } + return ProbeRequestTimeout(BackendQSV, strings.Join(devices, ",")) +} + // ProbeRequestTimeout gives a remote caller additional transport and response // margin beyond the server-side endpoint budget. func ProbeRequestTimeout(hardwareBackend, hardwareDevice string) time.Duration { @@ -296,9 +406,29 @@ func probeDevices(value, backend string) []string { } return []string{defaultDRIRenderDevice} } + if len(devices) > MaxProbedDevices { + // Past the cap the budget every caller allows stops covering the matrix, + // so probing further would guarantee the request is canceled rather + // than finished. Truncating is the honest failure: the devices that are + // probed get real verdicts, and the omission is logged rather than + // silently folded into a shorter answer. + noteProbeDevicesTruncated(len(devices)) + devices = devices[:MaxProbedDevices] + } return devices } +// probeDevicesTruncatedLogged latches the truncation warning to one line per +// process: a device list is a standing configuration, not an event. +var probeDevicesTruncatedLogged sync.Once + +func noteProbeDevicesTruncated(configured int) { + probeDevicesTruncatedLogged.Do(func() { + slog.Warn("tone-map probe covers only the first configured devices; the rest are not verified", + "component", "tonemap", "configured", configured, "probed", MaxProbedDevices) + }) +} + // intersectSourceKinds preserves the left-hand ordering while retaining source // kinds supported by both sets. func intersectSourceKinds(left, right []SourceKind) []SourceKind { @@ -402,15 +532,15 @@ func hardwareSmokeArgs(fixturePath, backend, hardwareDevice string, kind SourceK base := []string{ffmpegHideBannerArg, ffmpegLogLevelArg, ffmpegErrorLogLevel} switch backend { case BackendQSV: + base = append(base, QSVInitDeviceArgs(device)...) base = append(base, - "-init_hw_device", qsvVAAPIInitDevice(device), - "-init_hw_device", "qsv=qs@va", "-init_hw_device", "opencl=ocl@va", "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI, ) case BackendVAAPI: - base = append(base, "-init_hw_device", "vaapi=va:"+device, "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI) + base = append(base, VAAPIInitDeviceArgs("va", device)...) + base = append(base, "-filter_hw_device", "va", "-hwaccel", BackendVAAPI, "-hwaccel_output_format", BackendVAAPI) case BackendNVENC: cudaDevice := device if cudaDevice == "" { diff --git a/internal/tonemap/probe_invalidate_test.go b/internal/tonemap/probe_invalidate_test.go new file mode 100644 index 000000000..d7b3b2e90 --- /dev/null +++ b/internal/tonemap/probe_invalidate_test.go @@ -0,0 +1,113 @@ +package tonemap + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +// A non-empty inventory never expires, which is the blind spot +// InvalidateProbeCache closes: a driver replaced underneath a running node +// changes the answer without changing the binary's identity key. The observable +// contract is that the probe commands run again. +func TestInvalidateProbeCacheForcesAnotherProbe(t *testing.T) { + resetProbeCache(t) + now := time.Unix(100, 0) + calls := 0 + runner := func(_ context.Context, _ string, args ...string) ([]byte, error) { + calls++ + if len(args) > 0 && args[len(args)-1] == "-filters" { + return []byte(" .S. zscale V->V\n .S. tonemapx V->V\n .S. sidedata V->V\n"), nil + } + if len(args) > 0 && args[len(args)-1] == "-encoders" { + return []byte("libx264"), nil + } + return nil, nil + } + clock := func() time.Time { return now } + + probe := func(stage string) int { + capabilities, err := probeCached(context.Background(), "/ffmpeg-invalidate", BackendSoftware, "", runner, clock) + if err != nil { + t.Fatalf("%s probe error = %v", stage, err) + } + if len(capabilities) != 1 { + t.Fatalf("%s probe capabilities = %#v, want one software entry", stage, capabilities) + } + return calls + } + + first := probe("first") + if first == 0 { + t.Fatal("first probe ran no commands") + } + if cached := probe("cached"); cached != first { + t.Fatalf("cached probe ran %d commands, want the cached inventory reused", cached-first) + } + + InvalidateProbeCache() + + if reprobed := probe("re-probed"); reprobed != first*2 { + t.Fatalf("probe after invalidation ran %d commands total, want %d", reprobed, first*2) + } +} + +// Clearing the map is not enough: a probe already in flight completes and +// stores its inventory, so without a generation in the key the caller that +// invalidated would join that flight and be handed the very result it asked to +// discard — a re-probe reporting "nothing changed" about hardware it never +// re-examined. The generation moves the key instead, so the in-flight probe +// writes where nothing will read and the next caller runs a cold one. +func TestInvalidateProbeCacheSupersedesAnInFlightProbe(t *testing.T) { + resetProbeCache(t) + now := time.Unix(100, 0) + clock := func() time.Time { return now } + + // started closes once the first probe is inside its runner; blocked holds it + // there until the test has invalidated, so the race is decided rather than + // slept on. + started := make(chan struct{}) + blocked := make(chan struct{}) + var runs atomic.Int32 + runner := func(_ context.Context, _ string, args ...string) ([]byte, error) { + if runs.Add(1) == 1 { + close(started) + <-blocked + } + if len(args) > 0 && args[len(args)-1] == "-filters" { + return []byte(" .S. zscale V->V\n .S. tonemapx V->V\n .S. sidedata V->V\n"), nil + } + if len(args) > 0 && args[len(args)-1] == "-encoders" { + return []byte("libx264"), nil + } + return nil, nil + } + + var wg sync.WaitGroup + wg.Go(func() { + if _, err := probeCached(context.Background(), "/ffmpeg-inflight", BackendSoftware, "", runner, clock); err != nil { + t.Errorf("in-flight probe error = %v", err) + } + }) + + <-started + InvalidateProbeCache() + close(blocked) + + capabilities, err := probeCached(context.Background(), "/ffmpeg-inflight", BackendSoftware, "", runner, clock) + if err != nil { + t.Fatalf("post-invalidation probe error = %v", err) + } + if len(capabilities) != 1 { + t.Fatalf("post-invalidation capabilities = %#v, want one software entry", capabilities) + } + wg.Wait() + + // One command from the blocked flight is enough to prove the second probe + // did not simply wait on it: a joined caller would have run none of its own. + if got := runs.Load(); got < 2 { + t.Fatalf("runner invocations = %d, want the post-invalidation probe to run its own commands", got) + } +} diff --git a/internal/tonemap/probe_test.go b/internal/tonemap/probe_test.go index 81845d7f0..ca677f131 100644 --- a/internal/tonemap/probe_test.go +++ b/internal/tonemap/probe_test.go @@ -5,7 +5,11 @@ import ( "errors" "os" "path/filepath" + "runtime" + "slices" + "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -94,15 +98,21 @@ func TestProbeTotalTimeoutCoversBoundedCommandMatrix(t *testing.T) { } func TestProbeEndpointTimeoutCoversDetectionAndProbeBudgets(t *testing.T) { - if got, want := ProbeEndpointTimeout(BackendQSV, "/dev/dri/renderD128"), 81*time.Second; got != want { + if got, want := ProbeEndpointTimeout(BackendQSV, "/dev/dri/renderD128"), 106*time.Second; got != want { t.Fatalf("ProbeEndpointTimeout() = %s, want %s", got, want) } - if got, want := ProbeEndpointTimeout("auto", "/dev/dri/renderD128,/dev/dri/renderD129"), 106*time.Second; got != want { + if got, want := ProbeEndpointTimeout("auto", "/dev/dri/renderD128,/dev/dri/renderD129"), 131*time.Second; got != want { t.Fatalf("ProbeEndpointTimeout(auto) = %s, want %s", got, want) } - if got, want := ProbeRequestTimeout(BackendQSV, "/dev/dri/renderD128"), 86*time.Second; got != want { + if got, want := ProbeRequestTimeout(BackendQSV, "/dev/dri/renderD128"), 111*time.Second; got != want { t.Fatalf("ProbeRequestTimeout() = %s, want %s", got, want) } + // The slack has to outlast a full hardware detection walk plus the + // transformation registry probe, or a node answers 503 while its own + // detection is still running. + if probeEndpointSlack < 30*time.Second+3*3*time.Second { + t.Fatalf("probeEndpointSlack = %s, too small for detection and registry probes", probeEndpointSlack) + } } // TestProbeEmptyCapabilitiesExpire verifies failed discovery is retried after a short interval. @@ -396,10 +406,101 @@ func TestProbeCallerCancellationDoesNotCancelSharedProbe(t *testing.T) { } } -// resetProbeCache clears shared probe state between tests. +// resetProbeCache clears shared probe state between tests. It delegates to the +// exported invalidation so tests exercise the same seam the operator-facing +// re-probe action uses. func resetProbeCache(t *testing.T) { t.Helper() - probeCache.Lock() - probeCache.entries = make(map[string]probeCacheEntry) - probeCache.Unlock() + InvalidateProbeCache() +} + +// A tone-map probe outlives its caller by design, so a component that released +// its own claim on the GPU when its call returned can leave smoke encodes +// running with nothing accounting for them. The count is what lets the transcode +// node's re-probe gate see that. +func TestProbesInFlightCountsADetachedProbe(t *testing.T) { + awaitNoProbesInFlight(t) + + started := make(chan struct{}) + release := make(chan struct{}) + // The probe runs several commands; only the first needs to announce itself. + var announce, released sync.Once + t.Cleanup(func() { released.Do(func() { close(release) }) }) + + // The probe has to be running before the caller gives up, or the flight + // finishes on the canceled context and there is nothing detached to count. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := probeCached(ctx, "ffmpeg", BackendQSV, "/dev/dri/renderD128", + func(context.Context, string, ...string) ([]byte, error) { + announce.Do(func() { close(started) }) + <-release + return nil, errors.New("probe abandoned") + }, time.Now) + done <- err + }() + + <-started + cancel() + if err := <-done; err == nil { + t.Fatal("an abandoned probe reported success") + } + + // Checked the instant the caller returned, which is when its own claim on + // the encoder goes away while the smoke encode keeps running. + if got := ProbesInFlight(); got < 1 { + t.Fatalf("ProbesInFlight() = %d the moment the caller returned, want at least 1", got) + } + released.Do(func() { close(release) }) + awaitNoProbesInFlight(t) +} + +// awaitNoProbesInFlight waits for every claim on the encoder to be released, +// including ones detached from a caller that has already returned. Waiting on +// the counter rather than on a delay keeps this independent of machine load. +func awaitNoProbesInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if got := ProbesInFlight(); got == 0 { + return + } else if time.Now().After(deadline) { + t.Fatalf("ProbesInFlight() = %d, want every detached probe released", got) + } + runtime.Gosched() + } +} + +// The advertised budget every caller clamps to assumes a largest device set. If +// the worker probed past it, it would advertise a budget those callers then cut +// below what it actually needs, and cold capability requests would be canceled +// short forever. The probe set is capped so the two ends agree. +func TestProbeDevicesCapsTheConfiguredSet(t *testing.T) { + configured := make([]string, 0, MaxProbedDevices+4) + for i := range MaxProbedDevices + 4 { + configured = append(configured, defaultDRIRenderDevice+strconv.Itoa(i)) + } + + got := probeDevices(strings.Join(configured, ","), BackendQSV) + if len(got) != MaxProbedDevices { + t.Fatalf("probed %d devices, want the %d cap", len(got), MaxProbedDevices) + } + if !slices.Equal(got, configured[:MaxProbedDevices]) { + t.Fatalf("probed %v, want the first %d configured", got, MaxProbedDevices) + } + + // The budget a node advertises for that capped set is therefore never above + // what its callers allow — which is the property the cap exists for. + if advertised, ceiling := ProbeRequestTimeout(BackendQSV, strings.Join(configured, ",")), + MaxProbeRequestTimeout(); advertised > ceiling { + t.Fatalf("advertised %v exceeds the %v callers allow", advertised, ceiling) + } + + // A set inside the cap is untouched. + small := configured[:3] + if got := probeDevices(strings.Join(small, ","), BackendQSV); !slices.Equal(got, small) { + t.Fatalf("probed %v, want the configured %v unchanged", got, small) + } } diff --git a/internal/tonemap/tonemap.go b/internal/tonemap/tonemap.go index 22128d106..949f5e860 100644 --- a/internal/tonemap/tonemap.go +++ b/internal/tonemap/tonemap.go @@ -705,6 +705,25 @@ func qsvVAAPIInitDevice(device string) string { return "vaapi=va:" + device + ",driver=iHD,kernel_driver=i915,vendor_id=0x8086" } +// initHWDeviceFlag is FFmpeg's hardware-device declaration flag, shared by +// every init chain built here. +const initHWDeviceFlag = "-init_hw_device" + +// QSVInitDeviceArgs declares the Intel VAAPI display and derives the QSV +// device from it. Every QSV command line in the server — transcode, encoder +// warmup, capability probes, tone-map smoke tests, chapter thumbnails — must +// initialize hardware through this chain, so a driver constraint is fixed in +// one place. +func QSVInitDeviceArgs(device string) []string { + return []string{initHWDeviceFlag, qsvVAAPIInitDevice(device), initHWDeviceFlag, "qsv=qs@va"} +} + +// VAAPIInitDeviceArgs declares one VAAPI device under the alias the caller's +// filter graph and encoder reference. +func VAAPIInitDeviceArgs(alias, device string) []string { + return []string{initHWDeviceFlag, "vaapi=" + alias + ":" + device} +} + // HDRMetadataRemovalFilter removes side data that would otherwise incorrectly // label the converted SDR frames as HDR or Dolby Vision. func HDRMetadataRemovalFilter() string { diff --git a/internal/transcodenode/capability_client.go b/internal/transcodenode/capability_client.go index 6b7aff155..e7ed1049f 100644 --- a/internal/transcodenode/capability_client.go +++ b/internal/transcodenode/capability_client.go @@ -19,9 +19,26 @@ const maxHWCapabilitiesResponseBytes = 1 << 20 // cloned so its transport, timeout, and cookie jar remain available without // mutating its redirect policy. func FetchHWCapabilities(ctx context.Context, baseClient *http.Client, nodeURL, jwtSecret string) (playback.HWAccelInfo, int, error) { + info, _, status, err := FetchHWCapabilitiesPayload(ctx, baseClient, nodeURL, jwtSecret) + return info, status, err +} + +// FetchHWCapabilitiesPayload is FetchHWCapabilities with the node's own +// response bytes returned alongside the decoded report. +// +// A caller that *persists* the report must use this form. Re-marshaling the +// decoded struct instead drops every field this build does not know about, and +// during a rolling upgrade a node is routinely newer than the API server that +// reads it. The truncated payload would then be stored under the node's own +// hash, so after the API is upgraded the sweep still sees the hashes agree and +// never refetches: the durable inventory stays missing fields the new code +// reads, until some unrelated capability change or a manual re-probe moves the +// hash. The bytes are already bounded and parsed by the time they are returned, +// which is what makes storing them verbatim safe. +func FetchHWCapabilitiesPayload(ctx context.Context, baseClient *http.Client, nodeURL, jwtSecret string) (playback.HWAccelInfo, []byte, int, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(nodeURL, "/")+"/hw-capabilities", nil) if err != nil { - return playback.HWAccelInfo{}, 0, logredact.SanitizeURLError(err) + return playback.HWAccelInfo{}, nil, 0, logredact.SanitizeURLError(err) } request.Header.Set("Authorization", "Bearer "+jwtSecret) @@ -34,26 +51,26 @@ func FetchHWCapabilities(ctx context.Context, baseClient *http.Client, nodeURL, } response, err := client.Do(request) if err != nil { - return playback.HWAccelInfo{}, 0, logredact.SanitizeURLError(err) + return playback.HWAccelInfo{}, nil, 0, logredact.SanitizeURLError(err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { // Drain the (small) error body so the transport can reuse the // connection instead of tearing it down on every failed probe. _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) - return playback.HWAccelInfo{}, response.StatusCode, nil + return playback.HWAccelInfo{}, nil, response.StatusCode, nil } data, err := io.ReadAll(io.LimitReader(response.Body, maxHWCapabilitiesResponseBytes+1)) if err != nil { - return playback.HWAccelInfo{}, response.StatusCode, err + return playback.HWAccelInfo{}, nil, response.StatusCode, err } if len(data) > maxHWCapabilitiesResponseBytes { - return playback.HWAccelInfo{}, response.StatusCode, fmt.Errorf("node capability response exceeds %d bytes", maxHWCapabilitiesResponseBytes) + return playback.HWAccelInfo{}, nil, response.StatusCode, fmt.Errorf("node capability response exceeds %d bytes", maxHWCapabilitiesResponseBytes) } var info playback.HWAccelInfo if err := json.Unmarshal(data, &info); err != nil { - return playback.HWAccelInfo{}, response.StatusCode, err + return playback.HWAccelInfo{}, nil, response.StatusCode, err } - return info, response.StatusCode, nil + return info, data, response.StatusCode, nil } diff --git a/internal/transcodenode/capability_snapshot_test.go b/internal/transcodenode/capability_snapshot_test.go new file mode 100644 index 000000000..278c14285 --- /dev/null +++ b/internal/transcodenode/capability_snapshot_test.go @@ -0,0 +1,139 @@ +package transcodenode + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// newCapabilityTestServer builds a node whose configured ffmpeg is a script +// that records every invocation, so a test can assert nothing probed. +func newCapabilityTestServer(t *testing.T) (*Server, string) { + t.Helper() + dir := t.TempDir() + logPath := filepath.Join(dir, "ffmpeg-invocations.log") + ffmpegPath := filepath.Join(dir, "ffmpeg") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + logPath + "\nexit 1\n" + if err := os.WriteFile(ffmpegPath, []byte(script), 0o700); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + + watcher := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = testSecret + cfg.Playback.TranscodeDir = t.TempDir() + cfg.Playback.FFmpegPath = ffmpegPath + watcher.SetConfigForTest(cfg) + return &Server{watcher: watcher, sessions: make(map[string]*playback.TranscodeSession)}, logPath +} + +func decodeHealth(t *testing.T, server *Server) HealthResponse { + t.Helper() + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var health HealthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &health); err != nil { + t.Fatalf("decode health: %v", err) + } + return health +} + +// Health is polled every 30 seconds per node and is the liveness signal the +// pools act on. It must publish the last snapshot's hash and nothing more: if +// it probed, a slow or wedged ffmpeg would make a live node look dead. +func TestHealthPublishesStoredCapabilityHashWithoutProbing(t *testing.T) { + server, ffmpegLog := newCapabilityTestServer(t) + + if got := decodeHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q before any snapshot, want empty", got) + } + + server.storeCapabilityHash("sha256:abc123") + + if got := decodeHealth(t, server).CapabilitiesHash; got != "sha256:abc123" { + t.Fatalf("capabilities_hash = %q, want the stored snapshot hash", got) + } + if _, err := os.Stat(ffmpegLog); !os.IsNotExist(err) { + contents, _ := os.ReadFile(ffmpegLog) + t.Fatalf("health ran ffmpeg probes:\n%s", contents) + } +} + +// The capability endpoint and the background snapshot must agree, so a served +// report carries its hash and health starts advertising it immediately — +// otherwise the API would refetch a report it already has. +func TestHWCapabilitiesPublishesCapabilityHash(t *testing.T) { + server := newTestServer(t) + + request := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+testSecret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + if recorder.Code == http.StatusServiceUnavailable { + t.Skip("this host's ffmpeg cannot answer a capability probe") + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(recorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash == "" { + t.Fatal("served capability report carries no capability_hash") + } + // The hash must describe this payload, not some earlier one. + served := info + served.CapabilityHash = "" + if want := playback.ComputeCapabilityHash(served); want != info.CapabilityHash { + t.Fatalf("capability_hash = %s, want %s for the served payload", info.CapabilityHash, want) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != info.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the just-served %q", got, info.CapabilityHash) + } +} + +// The first snapshot waits on encoder warmup so it measures a primed encoder, +// and a canceled node must not leave that wait running. +func TestStartCapabilitySnapshotsWaitsForReadyChannel(t *testing.T) { + server, ffmpegLog := newCapabilityTestServer(t) + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + + server.StartCapabilitySnapshots(ctx, ready) + cancel() + <-ctx.Done() + + if _, err := os.Stat(ffmpegLog); !os.IsNotExist(err) { + contents, _ := os.ReadFile(ffmpegLog) + t.Fatalf("snapshot probed before warmup completed:\n%s", contents) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != "" { + t.Fatalf("capabilities_hash = %q, want empty while the snapshot is gated", got) + } +} + +// A failed probe is not evidence the hardware changed, so the previously +// published hash must survive it. +func TestRefreshCapabilitySnapshotKeepsHashOnProbeFailure(t *testing.T) { + server, _ := newCapabilityTestServer(t) + server.storeCapabilityHash("sha256:previous") + + server.refreshCapabilitySnapshot(context.Background()) + + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous hash kept after a failed probe", got) + } +} diff --git a/internal/transcodenode/gpugate.go b/internal/transcodenode/gpugate.go new file mode 100644 index 000000000..371f05cb1 --- /dev/null +++ b/internal/transcodenode/gpugate.go @@ -0,0 +1,114 @@ +package transcodenode + +import "sync" + +// gpuGate keeps an operator-triggered capability re-probe and the node's own +// GPU work off the encoder at the same time. +// +// Every hardware probe ends in a real smoke encode, which opens an encoder +// session. A card at its concurrent-session cap fails that encode with an error +// nothing can tell apart from a missing device or a broken driver, so a probe +// that races a transcode publishes a hardware regression for a GPU that is +// fine — and the API persists it, latches it, and routes the node to software +// until a clean report arrives. +// +// A point-in-time "are there active jobs" check cannot prevent that: a node +// idle at the check accepts a start milliseconds later, while the probe still +// has minutes to run. What is needed is one exclusion both sides consult, held +// from before the probe begins until after it ends. +// +// The gate is deliberately asymmetric. Work never waits: it is admitted or +// refused immediately, because a viewer pressing play must not queue behind a +// multi-minute probe. The re-probe never waits either: it refuses with 409 and +// tells the operator to retry when the node is idle, because blocking would +// hold an admin HTTP connection open for the length of a stream. +type gpuGate struct { + mu sync.Mutex + // workers counts GPU work that has been admitted and has not finished. It + // is separate from Server.activeJobs because that counter is incremented + // only once ffmpeg is already running: the window this gate exists to close + // is precisely the one between admitting work and it becoming visible + // there. + workers int + // reprobing is set for the whole capability rebuild, including the cache + // invalidation that precedes it. + reprobing bool +} + +// beginWork admits one unit of GPU work, or reports false while a re-probe +// holds the encoder. A caller that is admitted must call endWork exactly once. +func (g *gpuGate) beginWork() bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.reprobing { + return false + } + g.workers++ + return true +} + +// holdWork registers GPU work that is already running, and so cannot be +// refused. +// +// The gate admits work at its start and the node counts it in activeJobs once +// ffmpeg is up, which between them cover a session from admission to teardown — +// except for teardown itself. TranscodeSession.Close waits for ffmpeg to exit, +// so the encoder holds its GPU session for the whole call, while every teardown +// path drops activeJobs first so a stop is reflected immediately. That leaves a +// live encoder counted by neither, and a re-probe landing in the gap sees an +// idle node and smoke-encodes beside it — publishing exactly the false hardware +// failure this gate exists to prevent. Refusing here is not an option: a stop +// must always proceed. Counting it is. +func (g *gpuGate) holdWork() { + g.mu.Lock() + g.workers++ + g.mu.Unlock() +} + +// endWork releases one unit of admitted GPU work. +func (g *gpuGate) endWork() { + g.mu.Lock() + if g.workers > 0 { + g.workers-- + } + g.mu.Unlock() +} + +// beginReprobe claims the encoder exclusively, or reports the work in progress +// that stopped it. +// +// otherWork counts everything the gate does not track itself: the node's own +// running sessions, and the probes still running for callers that have gone +// away. It is a function rather than a value because it is read here, under the +// lock that grants the claim. Sampling it at the call site leaves a window — the +// request can be descheduled between reading zero and acquiring the lock, and a +// capability build that starts and is abandoned in that window leaves its +// background probe running while the count the gate sees still says idle. +// +// The detached probes are the piece that is not this node's own bookkeeping. A +// hardware probe runs on a background context so an abandoned caller cannot +// kill work another request is waiting on, which means the capability build +// releases its gate claim while ffmpeg may still be encoding. Without counting +// those, a re-probe claims an encoder that is not free and its smoke matrix +// races the one already running — publishing the false hardware verdict this +// gate exists to prevent, by the same mechanism, one layer down. +func (g *gpuGate) beginReprobe(otherWork func() int) (busy int, ok bool) { + g.mu.Lock() + defer g.mu.Unlock() + busy = g.workers + if otherWork != nil { + busy += otherWork() + } + if g.reprobing || busy > 0 { + return busy, false + } + g.reprobing = true + return 0, true +} + +// endReprobe releases the exclusive claim. +func (g *gpuGate) endReprobe() { + g.mu.Lock() + g.reprobing = false + g.mu.Unlock() +} diff --git a/internal/transcodenode/gpugate_test.go b/internal/transcodenode/gpugate_test.go new file mode 100644 index 000000000..b31e14a9e --- /dev/null +++ b/internal/transcodenode/gpugate_test.go @@ -0,0 +1,175 @@ +package transcodenode + +import "testing" + +// The bug this gate exists for: a node idle at a point-in-time check accepts a +// transcode milliseconds later, and the re-probe's smoke encode — minutes long +// — then races a live encoder session and publishes working hardware as failed. +// Admitted work has to keep the re-probe out even before it is visible as an +// active job. +func TestGPUGateRefusesReprobeWhileWorkIsAdmitted(t *testing.T) { + var gate gpuGate + + if !gate.beginWork() { + t.Fatal("beginWork on an idle gate was refused") + } + // activeJobs is still 0 here: the counter only moves once ffmpeg is running, + // which is exactly the window a point-in-time check missed. + if busy, ok := gate.beginReprobe(otherWork(0)); ok { + t.Fatal("re-probe admitted while a transcode was starting") + } else if busy != 1 { + t.Fatalf("busy = %d, want the admitted work counted", busy) + } + + gate.endWork() + if _, ok := gate.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused after the work finished") + } +} + +// The node's own running-session count is read under the same lock, so "no +// admitted work" and "no active jobs" cannot be true at two different instants. +func TestGPUGateRefusesReprobeWhileJobsAreActive(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(otherWork(2)) + if ok { + t.Fatal("re-probe admitted on a node running transcodes") + } + if busy != 2 { + t.Fatalf("busy = %d, want 2", busy) + } +} + +// A re-probe holds the encoder for the whole rebuild, so work arriving mid-probe +// is refused rather than queued: a viewer pressing play must not wait minutes +// for a probe, and the API retries elsewhere. +func TestGPUGateRefusesWorkWhileReprobing(t *testing.T) { + var gate gpuGate + + if _, ok := gate.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused on an idle gate") + } + if gate.beginWork() { + t.Fatal("GPU work admitted while a re-probe held the encoder") + } + if _, ok := gate.beginReprobe(otherWork(0)); ok { + t.Fatal("a second concurrent re-probe was admitted") + } + + gate.endReprobe() + if !gate.beginWork() { + t.Fatal("GPU work refused after the re-probe released the encoder") + } + gate.endWork() +} + +// endWork must not drive the counter negative, or one unbalanced release would +// let a re-probe run beside real transcodes forever. +func TestGPUGateEndWorkDoesNotUnderflow(t *testing.T) { + var gate gpuGate + + gate.endWork() + gate.endWork() + if !gate.beginWork() { + t.Fatal("beginWork refused after unbalanced releases") + } + if _, ok := gate.beginReprobe(otherWork(0)); ok { + t.Fatal("re-probe admitted while one unit of work was outstanding") + } +} + +// Teardown is GPU work too, and it cannot be refused: a stop must always +// proceed. TranscodeSession.Close waits for ffmpeg to exit, so the encoder holds +// its GPU session for the whole call while activeJobs has already dropped — +// counted by neither unless the gate holds it. +func TestGPUGateHoldWorkIsNeverRefusedAndKeepsReprobesOut(t *testing.T) { + var gate gpuGate + + gate.holdWork() + if busy, ok := gate.beginReprobe(otherWork(0)); ok { + t.Fatal("re-probe admitted while a session was still closing") + } else if busy != 1 { + t.Fatalf("busy = %d, want the closing session counted", busy) + } + + gate.endWork() + if _, ok := gate.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused after the teardown finished") + } + + // A re-probe already holding the encoder must not turn a stop into a hang or + // a refusal; the teardown is registered regardless. + gate.holdWork() + gate.endWork() +} + +// A hardware probe runs on a background context so an abandoned caller cannot +// kill work another request is waiting on. The capability build therefore +// releases its gate claim while ffmpeg may still be encoding, and a re-probe +// that counted only this node's own bookkeeping would claim an encoder that is +// not free — its smoke matrix racing the one already running, publishing the +// false hardware verdict this gate exists to prevent. +func TestGPUGateRefusesReprobeWhileADetachedProbeRuns(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(otherWork(1)) + if ok { + t.Fatal("re-probe admitted while a detached smoke encode was still running") + } + if busy != 1 { + t.Fatalf("busy = %d, want the detached probe counted", busy) + } + + if _, ok := gate.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused once no probe was in flight") + } +} + +// Tone-map probes detach from their caller exactly as hardware probes do, and +// they run their own FFmpeg smoke encodes, so a re-probe that counted only the +// hardware ones would start a second matrix beside a running first. +func TestGPUGateCountsEveryDetachedProbeSource(t *testing.T) { + var gate gpuGate + + busy, ok := gate.beginReprobe(otherWork(2)) + if ok { + t.Fatal("re-probe admitted while detached smoke encodes were still running") + } + if busy != 2 { + t.Fatalf("busy = %d, want both detached probe sources counted", busy) + } +} + +// otherWork builds the callback beginReprobe reads under its own lock, for +// tests that want a fixed count. +func otherWork(count int) func() int { return func() int { return count } } + +// The count has to be read under the lock that grants the claim, not sampled +// before it. A request descheduled between reading zero and acquiring the lock +// would otherwise claim an encoder that a capability build has since started +// and abandoned, leaving that build's background probe running. +func TestGPUGateReadsOtherWorkUnderTheClaimLock(t *testing.T) { + var gate gpuGate + + // Whatever this reports at the moment of the claim is what decides it: a + // probe that starts between the caller's own check and the lock is seen. + appeared := false + busy, ok := gate.beginReprobe(func() int { + appeared = true + return 1 + }) + if !appeared { + t.Fatal("beginReprobe did not consult the count while holding the lock") + } + if ok { + t.Fatal("re-probe admitted despite work appearing at claim time") + } + if busy != 1 { + t.Fatalf("busy = %d, want the work counted at claim time", busy) + } + + if _, ok := gate.beginReprobe(nil); !ok { + t.Fatal("re-probe refused with nothing else to count") + } +} diff --git a/internal/transcodenode/metrics_test.go b/internal/transcodenode/metrics_test.go new file mode 100644 index 000000000..2d2c59fc8 --- /dev/null +++ b/internal/transcodenode/metrics_test.go @@ -0,0 +1,191 @@ +package transcodenode + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/nodemetrics" +) + +// newFakeSampler answers with a fixed reading so the handlers under test are +// exercised without a Linux host beneath them. +func newFakeSampler() *nodemetrics.Sampler { + video, render := 63, 12 + return nodemetrics.NewFixedSamplerForTest(nodemetrics.Snapshot{ + Available: true, + SampledAt: time.Now(), + System: &nodemetrics.SystemStats{ + CPUPct: 41, Load1: 3.2, Cores: 16, + MemUsedMB: 9011, MemTotalMB: 32768, + Disks: []nodemetrics.DiskStats{{Path: "/transcode", Role: nodemetrics.ScratchDiskRole, Scratch: true, UsedGB: 210, TotalGB: 500}}, + NetRxBps: 1200000, NetTxBps: 98000000, + }, + GPU: []nodemetrics.GPUStats{{ + Device: "/dev/dri/renderD128", Vendor: "intel", Sessions: 2, + VideoBusyPct: &video, RenderBusyPct: &render, Source: nodemetrics.SourceFdinfo, + }}, + }) +} + +// Health is what the cluster routes on. Reading the sampler's published +// snapshot — rather than measuring on the request — is what keeps a wedged +// mount or a hung GPU query from turning into a health timeout, so this asserts +// the handler answers promptly and completely. +func TestHealthIncludesResourceSampleWithoutBlocking(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + answered <- recorder + }() + + var recorder *httptest.ResponseRecorder + select { + case recorder = <-answered: + case <-time.After(5 * time.Second): + t.Fatal("health handler blocked") + } + + var body struct { + Status string `json:"status"` + System *struct { + CPUPct int `json:"cpu_pct"` + } `json:"system"` + GPU []struct { + Device string `json:"device"` + Sessions int `json:"sessions"` + } `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + if body.Status != "ok" { + t.Fatalf("status = %q", body.Status) + } + if body.System == nil || body.System.CPUPct != 41 { + t.Fatalf("system = %+v, want the sampled cpu percentage", body.System) + } + if len(body.GPU) != 1 || body.GPU[0].Device != "/dev/dri/renderD128" || body.GPU[0].Sessions != 2 { + t.Fatalf("gpu = %+v", body.GPU) + } +} + +// A node with no sampler — a non-Linux host, or one built before sampling — +// must serve exactly the health response it always did. +func TestHealthOmitsResourceFieldsWithoutASampler(t *testing.T) { + server := newTestServer(t) + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + for _, key := range []string{"system", "gpu"} { + if _, ok := body[key]; ok { + t.Fatalf("%s emitted without a sampler: %s", key, recorder.Body) + } + } + if body["status"] != "ok" { + t.Fatalf("status = %v, want the unchanged response", body["status"]) + } +} + +func TestStatusIncludesResourceSample(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + recorder := httptest.NewRecorder() + server.handleStatus(recorder, httptest.NewRequest(http.MethodGet, "/status", nil)) + + var body struct { + System *json.RawMessage `json:"system"` + GPU *json.RawMessage `json:"gpu"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("status body: %v (%s)", err, recorder.Body) + } + if body.System == nil || body.GPU == nil { + t.Fatalf("status omitted the resource sample: %s", recorder.Body) + } +} + +// Operators who scrape must get the same numbers the UI shows, without a +// credential — the same posture the API listener's own /metrics has. +func TestMetricsEndpointIsMountedAndUnauthenticated(t *testing.T) { + server := newTestServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("GET /metrics = %d, want 200", recorder.Code) + } + if recorder.Body.Len() == 0 { + t.Fatal("GET /metrics returned an empty body") + } +} + +// /api/v1/health takes no bearer token, so anyone who can reach the node can +// read it. A disk entry's path is deployment layout — the transcode scratch +// volume, and on an API host every library root — which is exactly what the +// admin-authenticated resources endpoint exists to gate and what /metrics +// already withholds by labeling series with a role. The fill and the role must +// still be there, or the API's health sweep loses the scratch reading that +// admission control depends on. +func TestHealthOmitsFilesystemPaths(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + if body := recorder.Body.String(); strings.Contains(body, "/transcode") { + t.Fatalf("health body discloses a filesystem path: %s", body) + } + var body struct { + System *struct { + Disks []struct { + Path string `json:"path"` + Role string `json:"role"` + Scratch bool `json:"scratch"` + UsedGB float64 `json:"used_gb"` + } `json:"disks"` + } `json:"system"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("health body: %v (%s)", err, recorder.Body) + } + if body.System == nil || len(body.System.Disks) != 1 { + t.Fatalf("system = %+v, want one disk entry", body.System) + } + disk := body.System.Disks[0] + if disk.Path != "" { + t.Fatalf("disk path = %q, want it withheld", disk.Path) + } + if disk.Role != nodemetrics.ScratchDiskRole || !disk.Scratch || disk.UsedGB != 210 { + t.Fatalf("disk = %+v, want the scratch role and its fill kept", disk) + } +} + +// Redacting for /health must not reach the sampler's own snapshot: /status is +// bearer-authed and the admin resources endpoint is admin-authed, and both are +// meant to show operators where a mount actually is. +func TestStatusKeepsFilesystemPaths(t *testing.T) { + server := newTestServer(t) + server.metrics = newFakeSampler() + + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + + if got := server.metrics.Snapshot().System.Disks[0].Path; got != "/transcode" { + t.Fatalf("sampler snapshot path = %q after a health response, want it intact", got) + } +} diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go new file mode 100644 index 000000000..6ed3fbeb2 --- /dev/null +++ b/internal/transcodenode/reprobe.go @@ -0,0 +1,113 @@ +package transcodenode + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +// reprobeCapabilitiesResponse is what an operator-triggered re-probe reports +// back. It is deliberately tiny: the caller that wants the whole inventory +// fetches /hw-capabilities, and after this call that endpoint answers from the +// freshly warmed caches. +type reprobeCapabilitiesResponse struct { + // Resolved is the backend the node would now use. + Resolved string `json:"resolved"` + // CapabilityHash identifies the snapshot this re-probe published, so the + // caller can tell a re-probe that changed something from one that confirmed + // the previous answer. + CapabilityHash string `json:"capability_hash"` +} + +// handleReprobeCapabilities discards this node's cached probe verdicts and +// rebuilds the capability snapshot against live hardware. +// +// Both probe caches keep a *successful* verdict for the process lifetime, which +// is the right default — a GPU that encoded a frame does not stop being able to +// between two playback requests, and re-verifying per request would put ffmpeg +// execs on the playback path. The blind spot that leaves is hardware that has +// stopped working underneath a running node: a driver replaced, a device taken +// out of the container, an ffmpeg swapped in place. No cache key sees any of +// that, so a verdict that was true keeps being served until the node restarts. +// The opposite direction needs no help — a failed GPU probe carries a 15-second +// negative TTL and is retried on its own, so a repaired driver is picked up by +// the next capability snapshot. What this route adds there is the tone-map +// matrix, which caches any non-empty inventory permanently and so can stay +// software-only on a host whose GPU was broken at start. +// +// The rebuild reuses the ordinary snapshot assembly and budget, so a re-probe +// can never cost more than a cold capability fetch already may, and a rebuild +// that does not finish keeps the previously published hash: an incomplete probe +// is not evidence the hardware changed, and republishing a degraded snapshot +// would announce a hardware change that did not happen. +// +// It is refused on a node that is transcoding, for the same reason: every +// hardware probe ends in a real smoke encode that opens an encoder session, and +// a card at its concurrent session cap fails that encode with an error nothing +// can tell apart from a missing device or a broken driver. The verdict would +// then be published as verified:false, and the server would persist a hardware +// regression for a GPU that is fine and is at that moment encoding. Waiting for +// the node to drain costs an operator a retry; a false regression costs them a +// hardware investigation and, through the tone-map inventory in the same +// snapshot, degraded routing until the next clean probe. +func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Request) { + // Claimed for the whole rebuild, not sampled once: a node idle at a + // point-in-time check can accept a transcode milliseconds later and still + // collide with the smoke encode minutes into the probe. The gate is the + // same exclusion the transcode-start path consults, so from here until the + // deferred release no new GPU work is admitted. + busy, ok := s.gpu.beginReprobe(func() int { + return int(s.activeJobs.Load()) + playback.HWProbesInFlight() + tonemap.ProbesInFlight() + }) + if !ok { + slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", + "component", "transcodenode", "active_jobs", busy) + http.Error(w, fmt.Sprintf( + "node is running %d transcode job(s); a re-probe smoke-encodes on the GPU and a busy encoder would report working hardware as failed. Retry when the node is idle.", + busy), http.StatusConflict) + return + } + defer s.gpu.endReprobe() + + // Held across the invalidation and the rebuild together: discarding the + // verdicts and recomputing them has to be one step, or the scheduled + // snapshot could start its own cold matrix in between and run ffmpeg on the + // same GPU at the same time. + s.capabilityBuildMu.Lock() + defer s.capabilityBuildMu.Unlock() + + playback.InvalidateHWProbeCache() + tonemap.InvalidateProbeCache() + // The resource sampler retires nvidia-smi after repeated failure, and a + // driver that was broken at start is exactly what a re-probe is called for. + // Without this the node re-verifies its encoders here and still reports no + // GPU utilization until the breaker's own retry interval comes round. + s.metrics.RetrySources() + + // buildCapabilitySnapshotLocked owns the probe deadline, so a re-probe can + // never cost more than a cold capability fetch already may. + info, err := s.buildCapabilitySnapshotLocked(r.Context()) + if err != nil { + slog.WarnContext(r.Context(), "transcode node capability re-probe incomplete", + "component", "transcodenode", "error", err) + http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) + return + } + previous := s.storedCapabilityHash() + // A re-probed report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) + slog.InfoContext(r.Context(), "transcode node capabilities re-probed", "component", "transcodenode", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(reprobeCapabilitiesResponse{ + Resolved: info.Resolved, + CapabilityHash: info.CapabilityHash, + }); err != nil { + slog.WarnContext(r.Context(), "encode transcode node re-probe result", "component", "transcodenode", "error", err) + } +} diff --git a/internal/transcodenode/reprobe_test.go b/internal/transcodenode/reprobe_test.go new file mode 100644 index 000000000..fe72c1b65 --- /dev/null +++ b/internal/transcodenode/reprobe_test.go @@ -0,0 +1,397 @@ +package transcodenode + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +func postReprobe(t *testing.T, server *Server) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil) + request.Header.Set("Authorization", "Bearer "+testSecret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + return recorder +} + +// A re-probe must recompute and publish, so health starts advertising the new +// hash immediately — the whole point of the action is that the API stops seeing +// a stale answer without waiting for the 15-minute snapshot tick. +func TestReprobeCapabilitiesRecomputesAndStoresHash(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:stale") + + recorder := postReprobe(t, server) + if recorder.Code == http.StatusServiceUnavailable { + t.Skip("this host's ffmpeg cannot answer a capability probe") + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + + var result reprobeCapabilitiesResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatalf("decode re-probe result: %v", err) + } + if result.CapabilityHash == "" { + t.Fatal("re-probe reported no capability_hash") + } + if result.CapabilityHash == "sha256:stale" { + t.Fatal("re-probe echoed the stale hash instead of a recomputed one") + } + if result.Resolved == "" { + t.Fatal("re-probe reported no resolved backend") + } + if got := server.storedCapabilityHash(); got != result.CapabilityHash { + t.Fatalf("stored hash = %q, want the re-probed %q", got, result.CapabilityHash) + } + if got := decodeHealth(t, server).CapabilitiesHash; got != result.CapabilityHash { + t.Fatalf("health capabilities_hash = %q, want the re-probed %q", got, result.CapabilityHash) + } + + // The reported hash must describe the report the capability endpoint would + // now serve, or the API would refetch and store something else. + capabilityRequest := httptest.NewRequest(http.MethodGet, "/hw-capabilities", nil) + capabilityRequest.Header.Set("Authorization", "Bearer "+testSecret) + capabilityRecorder := httptest.NewRecorder() + server.Handler().ServeHTTP(capabilityRecorder, capabilityRequest) + if capabilityRecorder.Code != http.StatusOK { + t.Fatalf("capability status = %d after a re-probe", capabilityRecorder.Code) + } + var info playback.HWAccelInfo + if err := json.Unmarshal(capabilityRecorder.Body.Bytes(), &info); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if info.CapabilityHash != result.CapabilityHash { + t.Fatalf("served capability_hash = %q, want the re-probed %q", info.CapabilityHash, result.CapabilityHash) + } +} + +// An incomplete probe is not evidence the hardware changed, so a degraded +// re-probe must answer 503 and leave the previously published hash alone — +// publishing a partial report would announce a hardware change that did not +// happen and make the API store it. +func TestReprobeCapabilitiesKeepsHashOnProbeFailure(t *testing.T) { + server, _ := newCapabilityTestServer(t) + server.storeCapabilityHash("sha256:previous") + + if got := postReprobe(t, server).Code; got != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 for a probe that cannot complete", got) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous hash kept", got) + } +} + +// Every hardware probe ends in a real smoke encode that opens an encoder +// session. On a card at its concurrent session cap that encode fails with an +// error indistinguishable from a missing device, and the verdict would be +// published as verified:false — a hardware regression the server then persists +// and warns on, for a GPU that is fine and is at that moment encoding. So a busy +// node refuses, and keeps the report it has. +func TestReprobeCapabilitiesRefusesWhileTranscoding(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + server.activeJobs.Store(2) + t.Cleanup(func() { server.activeJobs.Store(0) }) + + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while the node is transcoding", recorder.Code) + } + if body := recorder.Body.String(); !strings.Contains(body, "idle") { + t.Fatalf("body = %q, want it to tell the operator when to retry", body) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// The route is bearer-authed like the rest of the admin group: it executes +// ffmpeg, so an unauthenticated caller could otherwise make a node do work. +func TestReprobeCapabilitiesRequiresBearer(t *testing.T) { + server := newTestServer(t) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/admin/reprobe-capabilities", nil)) + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 without a bearer token", recorder.Code) + } +} + +// The active-job count only moves once ffmpeg is already running, so checking it +// alone leaves a window: a node idle at the check accepts a transcode while the +// probe still has minutes to go, and the smoke encode races the live encoder +// after all. Work that has been admitted but is not yet an active job has to +// refuse the re-probe too. +func TestReprobeCapabilitiesRefusesWhileWorkIsStarting(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + if !server.gpu.beginWork() { + t.Fatal("beginWork on an idle node was refused") + } + t.Cleanup(server.gpu.endWork) + + // activeJobs is deliberately zero: this is exactly the state the old + // point-in-time check read as idle. + if got := server.activeJobs.Load(); got != 0 { + t.Fatalf("active jobs = %d, want the pre-registration window", got) + } + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while a transcode is starting", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// The other direction: while a re-probe holds the encoder, new GPU work is +// refused rather than allowed to collide with the smoke encode. It is refused, +// not queued — a viewer pressing play must not wait out a multi-minute probe, +// and the API retries on another node. +func TestTranscodeStartRefusedWhileReprobing(t *testing.T) { + server := newTestServer(t) + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused on an idle node") + } + t.Cleanup(server.gpu.endReprobe) + + if server.gpu.beginWork() { + t.Fatal("GPU work admitted while a re-probe held the encoder") + } +} + +// A hardware thumbnail extraction reserves a render device and runs ffmpeg on +// it, but never touches activeJobs — so before it consulted the gate it left +// the node looking idle and a re-probe could smoke-encode beside it. +func TestReprobeCapabilitiesRefusesWhileExtractingAThumbnail(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + if !server.gpu.beginWork() { + t.Fatal("beginWork on an idle node was refused") + } + t.Cleanup(server.gpu.endWork) + + if recorder := postReprobe(t, server); recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while a GPU extraction holds the encoder", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// The re-probe deliberately does not join a capability build already in flight +// — bumping the invalidation generation is what makes it honest — so without a +// lock the scheduled snapshot's ffmpeg matrix and the operator's would run at +// once on the same GPU, which is the collision the 409 exists to prevent. +// +// Ordering is asserted from receipts rather than a timeout: the builder reports +// when it has been admitted, and records on the far side of the lock whether +// this test had already released it. A sleep here could only ever say "it had +// not finished yet", which is also true when it never started. +func TestCapabilityBuildsAreSerialized(t *testing.T) { + server := newTestServer(t) + + admitted := make(chan struct{}, 1) + server.capabilityBuildAdmitted = func() { admitted <- struct{}{} } + + var released, acquiredAfterRelease atomic.Bool + server.capabilityBuildMu.Lock() + + building := make(chan struct{}) + go func() { + defer close(building) + // Any builder: the scheduled snapshot takes the same lock the endpoint + // and the re-probe do. + server.refreshCapabilitySnapshot(context.Background()) + acquiredAfterRelease.Store(released.Load()) + }() + + select { + case <-admitted: + case <-time.After(30 * time.Second): + t.Fatal("the capability build was never admitted") + } + + released.Store(true) + server.capabilityBuildMu.Unlock() + select { + case <-building: + case <-time.After(30 * time.Second): + t.Fatal("the capability build never ran after the lock was released") + } + if !acquiredAfterRelease.Load() { + t.Fatal("a capability build ran while another held the build lock") + } +} + +// /admin/force-reload deliberately tears down every live session so a config +// change cannot leave a running ffmpeg on stale settings. The control plane's +// own housekeeping must not do that: the API nudges a node after its +// acceleration overrides change, and the documented contract is that sessions +// already transcoding keep the backend they started with. +func TestReloadConfigKeepsActiveSessions(t *testing.T) { + server := newTestServer(t) + server.sessions["live-1"] = &playback.TranscodeSession{} + server.activeJobs.Store(1) + + request := httptest.NewRequest(http.MethodPost, "/admin/reload-config", nil) + request.Header.Set("Authorization", "Bearer "+testSecret) + recorder := httptest.NewRecorder() + server.Handler().ServeHTTP(recorder, request) + + // This fixture has no database, so the reload itself cannot succeed. What + // is under test is the route's blast radius, not its happy path: either + // outcome must leave the live session running. + if recorder.Code != http.StatusNoContent && recorder.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + server.mu.RLock() + _, alive := server.sessions["live-1"] + server.mu.RUnlock() + if !alive { + t.Fatal("a configuration reload tore down a live playback session") + } + if got := server.activeJobs.Load(); got != 1 { + t.Fatalf("active jobs = %d, want the session still counted", got) + } +} + +// An ordinary snapshot runs ffmpeg on the GPU whenever the probe caches are +// cold, so it registers as GPU work and a manual re-probe cannot claim an +// apparently idle encoder beside it. +func TestCapabilitySnapshotRegistersAsGPUWork(t *testing.T) { + server := newTestServer(t) + + // The builder reports the moment it holds the work slot, which is the state + // under test — polling the gate on a timer could observe it before or after. + admitted := make(chan struct{}, 1) + server.capabilityBuildAdmitted = func() { admitted <- struct{}{} } + + server.capabilityBuildMu.Lock() + building := make(chan struct{}) + go func() { + defer close(building) + server.refreshCapabilitySnapshot(context.Background()) + }() + + select { + case <-admitted: + case <-time.After(30 * time.Second): + t.Fatal("the capability snapshot was never admitted as GPU work") + } + if _, ok := server.gpu.beginReprobe(otherWork(0)); ok { + server.gpu.endReprobe() + t.Fatal("a re-probe was admitted while a capability snapshot held the encoder") + } + + server.capabilityBuildMu.Unlock() + select { + case <-building: + case <-time.After(30 * time.Second): + t.Fatal("the capability snapshot never completed") + } +} + +// The other direction: a snapshot refuses rather than running its matrix beside +// a re-probe's, and the previously published hash stands. +func TestCapabilitySnapshotRefusedWhileReprobing(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused on an idle node") + } + t.Cleanup(server.gpu.endReprobe) + + if _, err := server.buildCapabilitySnapshot(context.Background()); !errors.Is(err, ErrCapabilityBuildBusy) { + t.Fatalf("err = %v, want ErrCapabilityBuildBusy", err) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } +} + +// Every teardown path drops activeJobs before closing the session, so a stop is +// reflected immediately — but Close waits for ffmpeg to exit, so the encoder +// keeps its GPU session for the whole call. Without the gate holding it, that +// live encoder is counted by neither activeJobs nor the gate, and a re-probe +// landing in the gap smoke-encodes beside it and publishes the false hardware +// failure the gate exists to prevent. +func TestReprobeCapabilitiesRefusedWhileASessionIsStillClosing(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + server.activeJobs.Store(1) + + var ( + jobsDuringClose int32 + reprobeAdmitted bool + busyDuringClose int + ) + // Observed from inside the teardown, which is the only instant that matters. + err := server.retireGPUSession(func() error { + jobsDuringClose = server.activeJobs.Load() + busyDuringClose, reprobeAdmitted = server.gpu.beginReprobe(otherWork(int(jobsDuringClose))) + if reprobeAdmitted { + server.gpu.endReprobe() + } + return nil + }) + if err != nil { + t.Fatalf("retireGPUSession: %v", err) + } + + if jobsDuringClose != 0 { + t.Fatalf("active jobs = %d during Close, want the 0 that made this a gap", jobsDuringClose) + } + if reprobeAdmitted { + t.Fatal("re-probe admitted while a session was still closing its encoder") + } + if busyDuringClose != 1 { + t.Fatalf("busy = %d, want the closing session counted as GPU work", busyDuringClose) + } + + // The hold is released with the teardown, so an idle node re-probes again. + if _, ok := server.gpu.beginReprobe(otherWork(int(server.activeJobs.Load()))); !ok { + t.Fatal("re-probe refused after the teardown completed") + } + server.gpu.endReprobe() +} + +// Warmup is a real smoke encode, and the listener opens while it may still be +// running: an admin re-probe arriving in a node's first seconds would otherwise +// see an idle gate and run its matrix beside it, publishing a false hardware +// failure on a session-limited GPU. +func TestReprobeCapabilitiesRefusedWhileEncoderWarmupRuns(t *testing.T) { + server := newTestServer(t) + server.storeCapabilityHash("sha256:previous") + + // The gate is what warmup holds; observed here rather than by starting a + // real ffmpeg, which the test host has no hardware for. + server.gpu.holdWork() + if got := server.activeJobs.Load(); got != 0 { + t.Fatalf("active jobs = %d, want the warmup window where nothing is registered", got) + } + + recorder := postReprobe(t, server) + if recorder.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 while the encoder is warming", recorder.Code) + } + if got := server.storedCapabilityHash(); got != "sha256:previous" { + t.Fatalf("stored hash = %q, want the previous report untouched", got) + } + + server.gpu.endWork() + if _, ok := server.gpu.beginReprobe(otherWork(0)); !ok { + t.Fatal("re-probe refused after warmup finished") + } + server.gpu.endReprobe() +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 37276bcb3..dd15427b6 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -17,12 +17,14 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/sync/singleflight" "github.com/Silo-Server/silo-server/internal/chapterthumbs" "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" @@ -118,6 +120,22 @@ func ValidateAudioRecipeAttestation(req TranscodeStartRequest, response Transcod type HealthResponse struct { Status string `json:"status"` ActiveJobs int32 `json:"active_jobs"` + // CapabilitiesHash identifies this node's last computed hardware capability + // snapshot. It is read from the stored snapshot only — health must stay a + // cheap liveness answer, so it never triggers a probe — and is empty until + // the first background snapshot completes. + CapabilitiesHash string `json:"capabilities_hash,omitempty"` + // System and GPU are this node's last resource sample. Like the hash above + // they are read from a snapshot the sampler already published, never + // measured on the request: health is what the cluster routes on, so it must + // answer at the same speed whether or not a mount is hung or a GPU query is + // wedged. Both are omitted on a host that cannot be sampled. + // + // This route takes no credential, so the sample is path-free: disk entries + // carry their role and their fill, never where they are mounted. See + // nodemetrics.Snapshot.RedactPaths. + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } // sessionIdleTTL is how long a job may go without a manifest or segment @@ -195,6 +213,49 @@ type Server struct { // recipeStore is the control-plane recipe store consulted when a forwarded // token carries no recipe (the jellycompat node hop). Nil disables that path. recipeStore recipeStore + + // capabilityHash is the last computed capability snapshot's hash, published + // by /health without probing. Nil until the first snapshot or capability + // request completes. + capabilityHash atomic.Pointer[string] + + // metrics samples host and GPU resources in the background. Nil until + // StartMetricsSampler runs, which leaves health exactly as it was before. + metrics *nodemetrics.Sampler + + // gpu keeps a capability re-probe's smoke encode and real transcodes off + // the encoder at the same time; see gpuGate. + gpu gpuGate + + // capabilityBuildAdmitted, when set, is called once a capability build has + // claimed the GPU work slot and is about to wait for the build lock. It is + // the seam a test uses to observe that ordering rather than sleeping on it; + // production leaves it nil. + capabilityBuildAdmitted func() + + // capabilityBuildMu serializes capability assemblies with each other. + // + // The gpuGate covers transcodes and downloads, not other capability + // builders, and the probe caches no longer coalesce a re-probe with work + // already in flight — bumping the invalidation generation is what makes the + // re-probe honest. Those two together would otherwise let the scheduled + // snapshot (or an authenticated /hw-capabilities request) run its smoke + // matrix beside the operator's, which on session-limited hardware is + // exactly the collision that publishes a false regression. + capabilityBuildMu sync.Mutex +} + +// storedCapabilityHash returns the last published capability hash, or empty +// when none has been computed yet. +func (s *Server) storedCapabilityHash() string { + if hash := s.capabilityHash.Load(); hash != nil { + return *hash + } + return "" +} + +func (s *Server) storeCapabilityHash(hash string) { + s.capabilityHash.Store(&hash) } func (s *Server) resolveToneMapRecipe(ctx context.Context, opts *playback.TranscodeOpts) error { @@ -311,7 +372,7 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv } // StartOrphanSweeper runs the age-guarded orphan-transcode sweep immediately and -// then hourly until ctx is cancelled. It never blocks (a slow network-filesystem +// then hourly until ctx is canceled. It never blocks (a slow network-filesystem // delete runs in its own goroutine), so it is safe to call before the node binds // its listener. This is the node's only filesystem-level reclaimer of dirs left // behind by a session that was dropped without its output dir being removed — the @@ -333,21 +394,37 @@ func (s *Server) StartOrphanSweeper(ctx context.Context) { } // StartHardwareEncoderWarmup primes the configured hardware encoder behind -// node startup. It is best effort and never delays the health listener. -func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) { +// node startup. It is best effort and never delays the health listener. The +// returned channel closes once warmup has settled (including when there was +// nothing to warm), so work that wants a primed encoder — the first capability +// snapshot — can wait for it instead of racing it. +func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} { + done := make(chan struct{}) if s == nil || s.watcher == nil || ctx == nil { - return + close(done) + return done } cfg := s.watcher.Config() if cfg == nil { - return + close(done) + return done } playbackCfg := cfg.Playback + // Claimed here, before the goroutine exists. Warmup is a real smoke encode + // and the listener opens while it may still be running, so an admin re-probe + // arriving in a node's first seconds must not see an idle gate — and a claim + // taken inside the goroutine leaves exactly that window, since the scheduler + // makes no promise about when it runs. Held rather than requested: warmup is + // already committed by the time it could be refused. + s.gpu.holdWork() go func() { + defer close(done) + defer s.gpu.endWork() if err := playback.WarmHardwareEncoder(ctx, playbackCfg.FFmpegPath, playbackCfg.HWAccel, playbackCfg.HWDevice); err != nil { slog.DebugContext(ctx, "transcode node hardware encoder warmup failed", "component", "transcodenode", "error", err) } }() + return done } // activeSessionIDs snapshots the ids of currently registered jobs so the orphan @@ -490,8 +567,7 @@ func (s *Server) reapSession(sessionID string, session *playback.TranscodeSessio delete(s.lastAccess, sessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - if err := session.Close(); err != nil { + if err := s.closeSessionOffGPU(session); err != nil { slog.Error("close idle transcode session", "component", "transcodenode", "error", err, "session", sessionID, "playback_session_id", sessionID) } if s.tracker != nil { @@ -501,6 +577,30 @@ func (s *Server) reapSession(sessionID string, session *playback.TranscodeSessio "session", sessionID, "playback_session_id", sessionID, "idle_ms", time.Since(last).Milliseconds()) } +// closeSessionOffGPU retires one session: it drops the node's job count and +// closes the encoder, with the GPU gate holding the session as work for the +// whole teardown. +// +// The two counters have to overlap here, the same way the gate and activeJobs +// overlap when a session starts. Close waits for ffmpeg to exit, so the encoder +// keeps its GPU session for the length of the call, while activeJobs has to drop +// first for a stop to be reflected immediately. Without the hold, that live +// encoder is counted by nothing and a re-probe admitted in the gap smoke-encodes +// beside it. +func (s *Server) closeSessionOffGPU(session *playback.TranscodeSession) error { + return s.retireGPUSession(session.Close) +} + +// retireGPUSession drops the node's job count and runs one session's teardown +// with the gate holding it as work throughout. The hold discipline is the same +// whatever the teardown is, which is why it is separate from what it closes. +func (s *Server) retireGPUSession(close func() error) error { + s.gpu.holdWork() + defer s.gpu.endWork() + s.activeJobs.Add(-1) + return close() +} + // recipeStore reads a remote transcode's reconstruction recipe written by central // at transcode start, keyed by the transport id this node serves the job under. // It is the reconstruct source for every flow whose request cannot carry a @@ -545,6 +645,10 @@ func (s *Server) Handler() http.Handler { s.startIdleReaper() r := chi.NewRouter() r.Get("/api/v1/health", s.handleHealth) + // Unauthenticated, matching the API listener's own /metrics posture: a + // scrape target that needs a credential is a scrape target that goes + // unmonitored, and the exposure is host resource counters, not media. + r.Method(http.MethodGet, "/metrics", promhttp.Handler()) r.Group(func(r chi.Router) { r.Use(s.requireBearer) @@ -559,6 +663,8 @@ func (s *Server) Handler() http.Handler { r.Get("/transcode/{session_id}/master.m3u8", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/master.m3u8", s.handleManifest)) r.Get("/transcode/{session_id}/segment/{name}", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/segment/{name}", s.handleSegment)) r.Post("/admin/force-reload", s.handleForceReload) + r.Post("/admin/reload-config", s.handleReloadConfig) + r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) }) return r @@ -607,6 +713,16 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { return } } + // Claimed only once this request is committed to producing something: a + // prepared download encodes on the GPU like any transcode, and the tone-map + // recipe resolution just below runs ffmpeg probes on it too. Serving an + // artifact that already exists touches neither, so a re-probe must not + // refuse it. + if !s.gpu.beginWork() { + http.Error(w, "node is re-probing its hardware; retry shortly", http.StatusServiceUnavailable) + return + } + defer s.gpu.endWork() if req.ToneMapRequested() { if err := s.resolveToneMapRecipe(r.Context(), &opts); err != nil { writeToneMapRecipeError(w, err) @@ -731,7 +847,7 @@ func resolveToneMapRecipe(ctx context.Context, opts *playback.TranscodeOpts) err if ctx == nil { ctx = context.Background() } - resolveCtx, cancel := context.WithTimeout(ctx, tonemap.ProbeEndpointTimeout(opts.HWAccel, opts.HWDevice)) + resolveCtx, cancel := context.WithTimeout(ctx, playback.CapabilityEndpointTimeout(opts.HWAccel, opts.HWDevice)) defer cancel() if err := resolveCtx.Err(); err != nil { return err @@ -879,15 +995,80 @@ func (s *Server) trackDownloadPrepare(ctx context.Context, info nodesessions.Ses } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + snapshot := s.metrics.Snapshot().RedactPaths() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(HealthResponse{ - Status: "ok", - ActiveJobs: s.activeJobs.Load(), + Status: "ok", + ActiveJobs: s.activeJobs.Load(), + CapabilitiesHash: s.storedCapabilityHash(), + System: snapshot.System, + GPU: snapshot.GPU, }) } -// handleHWCapabilities reports live smoke-tested node capabilities. -func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { +// StartMetricsSampler begins background resource sampling until ctx is +// canceled, and publishes the readings on /health, /status and /metrics. +// +// The scratch dir is the only mount a transcode node samples: it is the volume +// that silently kills transcodes when it fills, and it is the one path the node +// is guaranteed to be able to see. Media roots are the API host's to report, +// because path visibility varies per node and a node cannot tell "root I cannot +// see" from "root that does not exist". +func (s *Server) StartMetricsSampler(ctx context.Context) { + if s == nil || ctx == nil { + return + } + s.metrics = nodemetrics.NewSampler(nodemetrics.Options{ + // Fixed for this process: a node writes every session under the + // directory it resolved at startup, so a later config edit does not move + // the volume this one is filling. + ScratchDir: func() string { return s.transcodeDir }, + DeviceSessions: playback.HWDeviceLoadSnapshot, + DeviceIdentities: playback.SamplerDeviceIdentities, + }) + s.metrics.Start(ctx) +} + +// buildCapabilitySnapshot runs the node's full capability detection: hardware +// walk, tone-map probe, and transformation registry, hashed into one identity. +// It is the single assembly used by both the capability endpoint and the +// background snapshot, so the hash a health response advertises always +// describes the payload the endpoint would serve. +// +// The probes behind it are individually cached, so repeating this is cheap once +// the first pass has run. +// ErrCapabilityBuildBusy reports that a capability snapshot was not attempted +// because a re-probe holds the encoder. The previously published hash stands. +var ErrCapabilityBuildBusy = errors.New("capability build refused while the node is re-probing") + +func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { + // A snapshot runs ffmpeg on the GPU whenever the probe caches are cold, so + // it registers as GPU work. That is what stops a manual re-probe from + // claiming an apparently idle encoder and running its own smoke matrix + // beside this one. + // + // It deliberately does *not* refuse while transcodes are running. A node + // under sustained load would then never refresh its inventory, and its + // advertised hash would go stale indefinitely — a worse failure than the + // cold-start contention this would avoid, which a positive probe result + // caches away after the first success and which the next snapshot corrects. + if !s.gpu.beginWork() { + return playback.HWAccelInfo{}, ErrCapabilityBuildBusy + } + defer s.gpu.endWork() + if s.capabilityBuildAdmitted != nil { + s.capabilityBuildAdmitted() + } + s.capabilityBuildMu.Lock() + defer s.capabilityBuildMu.Unlock() + return s.buildCapabilitySnapshotLocked(ctx) +} + +// buildCapabilitySnapshotLocked is buildCapabilitySnapshot's body. Callers must +// hold capabilityBuildMu; the re-probe takes it itself so its cache +// invalidation and its rebuild are one step no other builder can interleave +// with. +func (s *Server) buildCapabilitySnapshotLocked(ctx context.Context) (playback.HWAccelInfo, error) { ffmpegPath := "" configuredHWAccel := playback.HWAccelNone hwDevice := "" @@ -896,29 +1077,101 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { configuredHWAccel = cfg.Playback.HWAccel hwDevice = cfg.Playback.HWDevice } - resolveCtx, cancel := context.WithTimeout(r.Context(), toneMapCapabilityResolveTimeout(configuredHWAccel, hwDevice)) + resolveCtx, cancel := context.WithTimeout(ctx, toneMapCapabilityResolveTimeout(configuredHWAccel, hwDevice)) defer cancel() - hwAccel := playback.ResolveHWAccelWithFFmpegContext(resolveCtx, configuredHWAccel, ffmpegPath) - info := playback.DetectHWAccelWithFFmpegContext(resolveCtx, ffmpegPath) - info.ProbeRequestTimeoutMillis = tonemap.ProbeRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() - capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), hwAccel, hwDevice) + // One detection walk answers both questions: Resolved honors the configured + // backend's pass-through contract, and DetectedBackends explains it. + // + // A walk that ran out of budget is refused rather than published: it marks + // unprobed backends Verified=false, which is byte-identical to a real + // hardware failure, so hashing it would make the API persist a capability + // regression for hardware that is fine. + info, err := playback.DetectHWAccelWithFFmpegContextResult(resolveCtx, configuredHWAccel, ffmpegPath, hwDevice) if err != nil { - http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) - return + return playback.HWAccelInfo{}, err + } + info.ProbeRequestTimeoutMillis = playback.CapabilityRequestTimeout(configuredHWAccel, hwDevice).Milliseconds() + capabilities, err := tonemap.Probe(resolveCtx, playback.ResolveFFmpegPath(ffmpegPath), info.Resolved, hwDevice) + if err != nil { + return playback.HWAccelInfo{}, err } info.ToneMapCapabilities = capabilities registry, err := playback.ProbeTransformationRegistryWithToneMapV3Result(resolveCtx, ffmpegPath, info.ToneMapCapabilities) + if err != nil { + return playback.HWAccelInfo{}, err + } + info.Transformations = registry.Advertised() + info.CapabilityHash = playback.ComputeCapabilityHash(info) + return info, nil +} + +// handleHWCapabilities reports live smoke-tested node capabilities. +func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { + info, err := s.buildCapabilitySnapshot(r.Context()) if err != nil { http.Error(w, "capability probe unavailable", http.StatusServiceUnavailable) return } - info.Transformations = registry.Advertised() + // A served report is as authoritative as a scheduled snapshot, so health + // starts advertising this hash immediately rather than at the next tick. + s.storeCapabilityHash(info.CapabilityHash) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(info) } +// capabilitySnapshotInterval is how often the node recomputes its capability +// snapshot. The probes behind it are cached, so this mostly re-reads sysfs and +// re-hashes; it exists to notice hardware or ffmpeg changing underneath a +// long-running node without waiting for a restart. +const capabilitySnapshotInterval = 15 * time.Minute + +// StartCapabilitySnapshots keeps the capability hash published by /health +// current, in the background, until ctx is canceled. ready gates the first +// snapshot so it observes a primed encoder rather than racing warmup; a nil +// channel means snapshot immediately. +func (s *Server) StartCapabilitySnapshots(ctx context.Context, ready <-chan struct{}) { + if s == nil || ctx == nil { + return + } + go func() { + if ready != nil { + select { + case <-ctx.Done(): + return + case <-ready: + } + } + s.refreshCapabilitySnapshot(ctx) + ticker := time.NewTicker(capabilitySnapshotInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.refreshCapabilitySnapshot(ctx) + } + } + }() +} + +func (s *Server) refreshCapabilitySnapshot(ctx context.Context) { + info, err := s.buildCapabilitySnapshot(ctx) + if err != nil { + // Keep the previous hash: a failed probe is not evidence the hardware + // changed, and clearing it would look like a downgrade to the API. + slog.DebugContext(ctx, "transcode node capability snapshot failed", "component", "transcodenode", "error", err) + return + } + if previous := s.storedCapabilityHash(); previous != "" && previous != info.CapabilityHash { + slog.InfoContext(ctx, "transcode node capabilities changed", "component", "transcodenode", + "previous_hash", previous, "hash", info.CapabilityHash, "resolved", info.Resolved) + } + s.storeCapabilityHash(info.CapabilityHash) +} + func toneMapCapabilityResolveTimeout(hardwareBackend, hardwareDevice string) time.Duration { - return tonemap.ProbeEndpointTimeout(hardwareBackend, hardwareDevice) + return playback.CapabilityEndpointTimeout(hardwareBackend, hardwareDevice) } func (s *Server) handleChapterThumbnailExtract(w http.ResponseWriter, r *http.Request) { @@ -940,6 +1193,16 @@ func (s *Server) handleChapterThumbnailExtract(w http.ResponseWriter, r *http.Re writeChapterThumbnailError(w, http.StatusServiceUnavailable, "node_unavailable", "node not configured") return } + // A QSV or VAAPI extraction reserves a render device and runs ffmpeg on it, + // so it takes the same exclusion a transcode does. Without this the route + // leaves the gate looking idle — it never touches activeJobs either — and a + // re-probe could smoke-encode beside a live extraction. + if !s.gpu.beginWork() { + writeChapterThumbnailError(w, http.StatusServiceUnavailable, "node_unavailable", + "node is re-probing its hardware; retry shortly") + return + } + defer s.gpu.endWork() frame, reason, err := chapterthumbs.ExtractFrame(r.Context(), chapterthumbs.FrameExtractOptions{ InputPath: req.InputPath, SeekSeconds: req.SeekSeconds, @@ -1009,6 +1272,15 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { http.Error(w, "node not configured", http.StatusServiceUnavailable) return } + // Held from here until the handler returns, by which point activeJobs + // covers the session. Without the overlap a re-probe could start in the gap + // between admitting this transcode and it becoming visible as an active + // job, and its smoke encode would then race a live encoder session. + if !s.gpu.beginWork() { + http.Error(w, "node is re-probing its hardware; retry shortly", http.StatusServiceUnavailable) + return + } + defer s.gpu.endWork() outputDir := s.sessionOutputDir(req.SessionID) opts := playback.TranscodeOpts{ @@ -1090,8 +1362,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { delete(s.sessions, req.SessionID) delete(s.lastAccess, req.SessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - _ = old.Close() + _ = s.closeSessionOffGPU(old) // Move the old segment directory aside and delete it in the // background: removing a long session's segments can take seconds // on slow disks, and the playback start that triggered this switch @@ -1339,6 +1610,15 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe if cfg == nil { return nil, nil } + // A reconstruct spawns ffmpeg on the GPU exactly as a fresh start does, so + // it takes the same exclusion against a running capability re-probe. Held + // until this call returns, by which point activeJobs covers the session. + if !s.gpu.beginWork() { + slog.InfoContext(r.Context(), "transcode node reconstruct deferred while re-probing hardware", + "component", "transcodenode", "session", sessionID) + return nil, nil + } + defer s.gpu.endWork() outputDir := s.sessionOutputDir(sessionID) opts := card.TranscodeOpts(outputDir, cfg.Playback.FFmpegPath, s.ffmpegSink) opts.SessionID = sessionID @@ -1524,9 +1804,8 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { delete(s.sessions, sessionID) delete(s.lastAccess, sessionID) s.mu.Unlock() - s.activeJobs.Add(-1) - if err := session.Close(); err != nil { + if err := s.closeSessionOffGPU(session); err != nil { slog.ErrorContext(r.Context(), "close transcode session", "component", "transcodenode", "error", err, "session", sessionID, "playback_session_id", sessionID) } @@ -1735,6 +2014,29 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, segPath) } +// handleReloadConfig re-reads this node's configuration and nothing else. +// +// It exists because /admin/force-reload is destructive: it tears down every +// live playback session so a configuration change cannot leave a running ffmpeg +// on stale settings. That is the right answer when an operator asks for it +// explicitly, and the wrong one for the control plane's own housekeeping — the +// API nudges a node after its acceleration overrides change, and a policy edit +// that says it applies to new transcodes must not interrupt the ones already +// playing. Sessions keep the settings they started with, which is exactly what +// the override documentation promises. +func (s *Server) handleReloadConfig(w http.ResponseWriter, r *http.Request) { + // The same lock the destructive route takes, so a start cannot be admitted + // against a config this reload is in the middle of replacing. + s.reloadMu.Lock() + defer s.reloadMu.Unlock() + if err := s.watcher.ForceReload(r.Context()); err != nil { + http.Error(w, "reload failed: "+err.Error(), http.StatusInternalServerError) + return + } + slog.InfoContext(r.Context(), "transcode node configuration reloaded", "component", "transcodenode") + w.WriteHeader(http.StatusNoContent) +} + func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { s.reloadMu.Lock() defer s.reloadMu.Unlock() @@ -1765,9 +2067,8 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { delete(s.sessions, victim.id) delete(s.lastAccess, victim.id) s.mu.Unlock() - s.activeJobs.Add(-1) - victim.session.Close() + _ = s.closeSessionOffGPU(victim.session) if err := os.RemoveAll(s.sessionOutputDir(victim.id)); err != nil { slog.WarnContext(r.Context(), "remove transcode session directory during reload", "component", "transcodenode", "session", victim.id, "error", err) } @@ -1802,15 +2103,20 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { } s.mu.RUnlock() + snapshot := s.metrics.Snapshot() w.Header().Set("Content-Type", "application/json") type statusResponse struct { - Status string `json:"status"` - ActiveJobs int32 `json:"active_jobs"` - Sessions []string `json:"sessions"` + Status string `json:"status"` + ActiveJobs int32 `json:"active_jobs"` + Sessions []string `json:"sessions"` + System *nodemetrics.SystemStats `json:"system,omitempty"` + GPU []nodemetrics.GPUStats `json:"gpu,omitempty"` } json.NewEncoder(w).Encode(statusResponse{ Status: "ok", ActiveJobs: s.activeJobs.Load(), Sessions: sessionIDs, + System: snapshot.System, + GPU: snapshot.GPU, }) } diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go index 553c44972..915b55370 100644 --- a/internal/transcodenode/server_test.go +++ b/internal/transcodenode/server_test.go @@ -380,10 +380,15 @@ func TestHandleHWCapabilitiesReturnsServiceUnavailableWhenDeadlineExpires(t *tes } func TestToneMapCapabilityResolveTimeoutCoversConfiguredProbeBudget(t *testing.T) { + // The endpoint budget is both halves — the hardware walk and the tone-map + // matrix — because the endpoint runs both. got := toneMapCapabilityResolveTimeout(tonemap.BackendQSV, "/dev/dri/renderD128") - if want := tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { + if want := playback.CapabilityEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got != want { t.Fatalf("capability resolve timeout = %v, want endpoint budget %v", got, want) } + if tone := tonemap.ProbeEndpointTimeout(tonemap.BackendQSV, "/dev/dri/renderD128"); got <= tone { + t.Fatalf("capability resolve timeout = %v, want more than the tone-map half alone (%v)", got, tone) + } } func TestHandleHWCapabilitiesAdvertisesEffectiveProbeRequestTimeout(t *testing.T) { @@ -405,7 +410,7 @@ func TestHandleHWCapabilitiesAdvertisesEffectiveProbeRequestTimeout(t *testing.T if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil { t.Fatal(err) } - want := float64(tonemap.ProbeRequestTimeout(playback.HWAccelNone, "").Milliseconds()) + want := float64(playback.CapabilityRequestTimeout(playback.HWAccelNone, "").Milliseconds()) if got := response["probe_request_timeout_ms"]; got != want { t.Fatalf("probe request timeout = %v, want %.0fms", got, want) } diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt index 125f4d408..82b94fb1f 100644 --- a/internal/transcodenode/testdata/media_routes.txt +++ b/internal/transcodenode/testdata/media_routes.txt @@ -1,5 +1,7 @@ # fixture 1 POST /admin/force-reload non-media +POST /admin/reload-config non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media @@ -7,6 +9,7 @@ GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media @@ -14,6 +17,8 @@ GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false true GET /transcode/{session_id}/segment/{name} media playback internal_relay false true # fixture 2 POST /admin/force-reload non-media +POST /admin/reload-config non-media +POST /admin/reprobe-capabilities non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media @@ -21,6 +26,7 @@ GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media +GET /metrics non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media diff --git a/internal/worker/cleanup.go b/internal/worker/cleanup.go index 626ca7669..0a9942e53 100644 --- a/internal/worker/cleanup.go +++ b/internal/worker/cleanup.go @@ -173,6 +173,9 @@ func (c *SessionCleaner) CleanStale(ctx context.Context) (int, error) { } } + // Hub and cache bus are separate audiences (connected admin clients vs + // cross-node cache invalidation), so both publish when either is wired — + // see the same pattern in the reconciler. if totalDeleted > 0 && c.EventsHub != nil { if err := c.EventsHub.PublishJSON( ctx, @@ -183,7 +186,8 @@ func (c *SessionCleaner) CleanStale(ctx context.Context) (int, error) { ); err != nil { return int(totalDeleted), fmt.Errorf("publishing playback cleanup invalidation: %w", err) } - } else if c.EventBus != nil && totalDeleted > 0 { + } + if totalDeleted > 0 && c.EventBus != nil { if err := c.EventBus.Publish(ctx, cache.ChannelPlayback, cache.Event{ Type: cache.EventPlaybackSessionsChanged, Payload: "cleanup", diff --git a/internal/worker/reconciler.go b/internal/worker/reconciler.go index 0a57a210d..9dc0f2685 100644 --- a/internal/worker/reconciler.go +++ b/internal/worker/reconciler.go @@ -230,6 +230,12 @@ func (r *Reconciler) ReconcileNodeSessions(ctx context.Context, reportingNode st if err := tx.Commit(ctx); err != nil { return fmt.Errorf("committing transaction: %w", err) } + // The two publishes serve different consumers and are not alternatives: + // the events hub pushes "sessions.replaced" to connected admin clients, + // while the cache bus invalidates the playback-derived admin aggregates + // (stats, activity, leaderboards, timeseries) across nodes. Gating the bus + // behind the hub's absence left those caches TTL-only in the normal + // configuration, where both are wired. if changed && r.EventsHub != nil { if err := r.EventsHub.PublishJSON( ctx, @@ -240,7 +246,8 @@ func (r *Reconciler) ReconcileNodeSessions(ctx context.Context, reportingNode st ); err != nil { log.Printf("reconciler: failed to publish session event for node %s: %v", reportingNode, err) } - } else if changed && r.EventBus != nil { + } + if changed && r.EventBus != nil { if err := r.EventBus.Publish(ctx, cache.ChannelPlayback, cache.Event{ Type: cache.EventPlaybackSessionsChanged, Payload: reportingNode, diff --git a/migrations/postgres_search_exact_indexes_test.go b/migrations/postgres_search_exact_indexes_test.go index ee339f353..609ec06aa 100644 --- a/migrations/postgres_search_exact_indexes_test.go +++ b/migrations/postgres_search_exact_indexes_test.go @@ -19,6 +19,7 @@ func TestPostgresSearchExactTitleIndexesAreConcurrentAndRetrySafe(t *testing.T) "ADD COLUMN IF NOT EXISTS search_overview_vector tsvector", "CREATE OR REPLACE FUNCTION public.set_episode_catalog_entry_search_fields()", "CREATE TRIGGER trg_episode_catalog_entries_search_fields", + "still_thumbhash, overview, created_at OR DELETE", "UPDATE public.episode_catalog_entries ece", "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_media_items_title_normalized_exact", "ON public.media_items (title_normalized text_pattern_ops, content_id)", @@ -37,4 +38,8 @@ func TestPostgresSearchExactTitleIndexesAreConcurrentAndRetrySafe(t *testing.T) t.Fatalf("migration missing %q:\n%s", required, sql) } } + downMarker := strings.Index(sql, "-- +goose Down") + if downMarker < 0 || !strings.Contains(sql[downMarker:], "still_thumbhash, created_at OR DELETE") { + t.Fatalf("migration down path does not restore the original episode refresh trigger:\n%s", sql) + } } diff --git a/migrations/sql/20260826225109_node_gpu_capabilities.sql b/migrations/sql/20260826225109_node_gpu_capabilities.sql new file mode 100644 index 000000000..7e6bdbc7d --- /dev/null +++ b/migrations/sql/20260826225109_node_gpu_capabilities.sql @@ -0,0 +1,34 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capabilities jsonb, + ADD COLUMN capabilities_hash text, + ADD COLUMN capabilities_refreshed_at timestamptz; + +COMMENT ON COLUMN stream_nodes.capabilities IS + 'Last capability report fetched from this node (the /hw-capabilities ' + 'payload: resolved backend, render devices with PCI address and GPU uuid, ' + 'probed backends, transformations, tone-map executors). NULL until the ' + 'node has advertised a capability hash and the fetch succeeded. Durable ' + 'so GPU inventory survives an API restart and so a node that goes ' + 'unhealthy still reports what hardware it had.'; + +COMMENT ON COLUMN stream_nodes.capabilities_hash IS + 'Identity of the stored capabilities payload, as computed by the node. ' + 'The health sweep refetches only when the node reports a hash different ' + 'from this one, so an unchanged node costs one health request. NULL until ' + 'the first successful fetch.'; + +COMMENT ON COLUMN stream_nodes.capabilities_refreshed_at IS + 'When the stored capabilities were last fetched and persisted. This is the ' + 'age of the inventory, not of the last health check; a node checked every ' + '30s may keep capabilities from hours ago because nothing changed.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capabilities, + DROP COLUMN IF EXISTS capabilities_hash, + DROP COLUMN IF EXISTS capabilities_refreshed_at; +-- +goose StatementEnd diff --git a/migrations/sql/20260827004808_node_last_stats.sql b/migrations/sql/20260827004808_node_last_stats.sql new file mode 100644 index 000000000..c46251ac4 --- /dev/null +++ b/migrations/sql/20260827004808_node_last_stats.sql @@ -0,0 +1,23 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN last_stats jsonb; + +COMMENT ON COLUMN stream_nodes.last_stats IS + 'Most recent host resource sample reported by this node in its health ' + 'response: {"system": {cpu_pct, load1, cores, mem_used_mb, mem_total_mb, ' + 'disks, net_rx_bps, net_tx_bps}, "gpu": [{device, vendor, sessions, ' + 'video_busy_pct, render_busy_pct, total_busy_pct, vram_used_mb, ' + 'vram_total_mb, source}]}. Written by the same 30s health update that ' + 'writes active_jobs, so it is exactly as old as last_health_check. ' + 'Current sample only — this is not a history table; operators who want ' + 'trends scrape the node /metrics endpoint. NULL when the node reported no ' + 'sample: a build predating resource sampling, a non-Linux host, or a node ' + 'that failed its health check.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS last_stats; +-- +goose StatementEnd diff --git a/migrations/sql/20260827013713_admin_dashboard_layouts.sql b/migrations/sql/20260827013713_admin_dashboard_layouts.sql new file mode 100644 index 000000000..18b7f35ee --- /dev/null +++ b/migrations/sql/20260827013713_admin_dashboard_layouts.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- Per-admin-account admin dashboard widget layout. The server treats the blob +-- as opaque; the web client validates widget ids and spans when it loads it. +CREATE TABLE IF NOT EXISTS admin_dashboard_layouts ( + user_id integer PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + layout jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE IF EXISTS admin_dashboard_layouts; diff --git a/migrations/sql/20260827015256_dashboard_activity_indexes.sql b/migrations/sql/20260827015256_dashboard_activity_indexes.sql new file mode 100644 index 000000000..26af3fa01 --- /dev/null +++ b/migrations/sql/20260827015256_dashboard_activity_indexes.sql @@ -0,0 +1,50 @@ +-- +goose NO TRANSACTION +-- +goose Up +-- The admin dashboard's activity aggregates scan by start/watch time over a +-- rolling window. Both tables are only indexed on their end-of-play timestamps +-- today (idx_playback_history_admin_ended, idx_user_watch_history_*), which a +-- "started in the last N hours" or "watched in the last N days" filter cannot +-- use. +-- +-- Both tables grow with every play on a busy deployment, so the indexes are +-- built CONCURRENTLY and this migration runs outside a transaction: a plain +-- CREATE INDEX would hold a write lock for the length of the build. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_index i ON i.indexrelid = c.oid + WHERE n.nspname = 'public' + AND c.relname = 'idx_playback_history_admin_started' + AND NOT i.indisvalid + ) THEN + DROP INDEX public.idx_playback_history_admin_started; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_index i ON i.indexrelid = c.oid + WHERE n.nspname = 'public' + AND c.relname = 'idx_user_watch_history_watched_at' + AND NOT i.indisvalid + ) THEN + DROP INDEX public.idx_user_watch_history_watched_at; + END IF; +END; +$$; +-- +goose StatementEnd + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_playback_history_admin_started + ON public.playback_history_admin USING btree (started_at DESC); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_user_watch_history_watched_at + ON public.user_watch_history USING btree (watched_at DESC); + +-- +goose Down +DROP INDEX CONCURRENTLY IF EXISTS public.idx_user_watch_history_watched_at; + +DROP INDEX CONCURRENTLY IF EXISTS public.idx_playback_history_admin_started; diff --git a/migrations/sql/20260827021132_dashboard_metric_samples.sql b/migrations/sql/20260827021132_dashboard_metric_samples.sql new file mode 100644 index 000000000..dcc27f734 --- /dev/null +++ b/migrations/sql/20260827021132_dashboard_metric_samples.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- Minute-resolution samples for the admin dashboard's concurrent-stream and +-- egress charts. Neither series can be reconstructed after the fact: live +-- sessions leave no per-minute trace and node egress is a rolling average, so +-- the sampler writes them as they happen. +-- +-- One row per minute per source. 'shared' is the cluster-wide snapshot (any +-- replica may write it; the primary key makes the first writer win), while +-- 'proc:' rows carry the viewer egress served by one API process, +-- which stream_nodes does not cover. +CREATE TABLE IF NOT EXISTS dashboard_metric_samples ( + bucket timestamptz NOT NULL, + source text NOT NULL, + streams_total integer NOT NULL DEFAULT 0, + streams_direct integer NOT NULL DEFAULT 0, + streams_remux integer NOT NULL DEFAULT 0, + streams_transcode integer NOT NULL DEFAULT 0, + egress_kbps bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (bucket, source) +); + +-- +goose Down +DROP TABLE IF EXISTS dashboard_metric_samples; diff --git a/migrations/sql/20260827025521_node_hw_overrides.sql b/migrations/sql/20260827025521_node_hw_overrides.sql new file mode 100644 index 000000000..6e6c23f8b --- /dev/null +++ b/migrations/sql/20260827025521_node_hw_overrides.sql @@ -0,0 +1,32 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN hw_accel_override text + CHECK (hw_accel_override IN ('auto', 'qsv', 'vaapi', 'nvenc', 'none')), + ADD COLUMN hw_device_override text; + +COMMENT ON COLUMN stream_nodes.hw_accel_override IS + 'Per-node hardware acceleration backend, overriding the cluster-wide ' + 'playback.hw_accel setting for this node only. NULL means inherit the ' + 'cluster value — the normal case for a homogeneous deployment. Set it ' + 'when one node''s hardware differs from the rest (a CPU-only box in a QSV ' + 'cluster sets ''none''). The node reads its own row (matched on url ' + 'against its NODE_URL) on every config reload, so this is the value it ' + 'probes with and falls back to; the API dispatches remote transcodes with ' + 'it too, in preference to the cluster value. A change applies without a ' + 'restart, except to the boot-time encoder warmup and to sessions already ' + 'transcoding.'; + +COMMENT ON COLUMN stream_nodes.hw_device_override IS + 'Per-node hardware device (render node path or index) for the backend ' + 'above, overriding the cluster-wide playback.hw_device for this node ' + 'only. NULL means inherit the cluster value. Applies on the same terms as ' + 'hw_accel_override.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS hw_accel_override, + DROP COLUMN IF EXISTS hw_device_override; +-- +goose StatementEnd diff --git a/migrations/sql/20260827125333_node_capability_drift.sql b/migrations/sql/20260827125333_node_capability_drift.sql new file mode 100644 index 000000000..dbd90fe21 --- /dev/null +++ b/migrations/sql/20260827125333_node_capability_drift.sql @@ -0,0 +1,25 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capability_drift text; + +COMMENT ON COLUMN stream_nodes.capability_drift IS + 'Human-readable note describing how this node''s hardware got worse, set ' + 'when a capability refetch shows a previously verified backend now failing ' + 'its probe or a render device that has disappeared. It stays set until a ' + 'refetch produces a report whose probes all pass: a refetch that merely ' + 'loses nothing further leaves it alone, because a comparison against an ' + 'already-degraded report always finds nothing and would report a still- ' + 'broken node as repaired. NULL therefore means no standing regression. It ' + 'exists because that regression is otherwise only a log line: a driver that ' + 'stopped working turns a node from a GPU transcoder into a silent CPU one, ' + 'and the node stays healthy throughout. Written only by the node health ' + 'sweep, in the same statement as capabilities and capabilities_hash, so it ' + 'always describes the report stored beside it. Not a routing input.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capability_drift; +-- +goose StatementEnd diff --git a/migrations/sql/20260827192932_node_capability_drift_baseline.sql b/migrations/sql/20260827192932_node_capability_drift_baseline.sql new file mode 100644 index 000000000..75903aa48 --- /dev/null +++ b/migrations/sql/20260827192932_node_capability_drift_baseline.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN capability_drift_baseline jsonb; + +COMMENT ON COLUMN stream_nodes.capability_drift_baseline IS + 'Machine-readable record of what capability_drift is standing for: ' + '{"backends": ["qsv"], "devices": [["GPU-uuid", "0000:03:00.0", ' + '"/dev/dri/renderD128"]]} — the backends that must verify again and the ' + 'device alias sets one of whose members must reappear before the note is ' + 'cleared. Each device is stored as every stable name it answered to, so a ' + 'card that comes back renumbered, or whose nvidia-smi uuid is missing on ' + 'the pass that finds it, still matches. NULL exactly when capability_drift ' + 'is NULL. It exists because recovery cannot be derived from the stored ' + 'report alone: once a degraded report is stored, every later comparison is ' + 'degraded-to-degraded and finds nothing lost, and mere growth in the ' + 'inventory is not recovery either — adding an unrelated GPU to a node that ' + 'lost one is not the lost one returning. Written by the node health sweep ' + 'in the same statement as capabilities and capability_drift, so it always ' + 'describes the note beside it. Not a routing input.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS capability_drift_baseline; +-- +goose StatementEnd diff --git a/migrations/sql/20260828180230_node_public_url.sql b/migrations/sql/20260828180230_node_public_url.sql new file mode 100644 index 000000000..ff1a052fd --- /dev/null +++ b/migrations/sql/20260828180230_node_public_url.sql @@ -0,0 +1,23 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE stream_nodes + ADD COLUMN public_url text; + +COMMENT ON COLUMN stream_nodes.public_url IS + 'Base URL streaming clients are given for this node, when it differs from ' + 'url. url is the backend address: what the API server dials for health ' + 'checks, capability fetches, and dispatch, and what a proxy dials to reach ' + 'a transcode node — on a private network that should be a private address, ' + 'which keeps co-located proxy/transcode traffic on the LAN instead of ' + 'hairpinning through a public load balancer. public_url is only ever used ' + 'to build the stream and download URLs handed to clients, so it is only ' + 'meaningful on proxy nodes: clients never talk to transcode nodes. NULL ' + 'means clients use url, which is every deployment registered before the ' + 'column existed and every deployment with one flat network.'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE stream_nodes + DROP COLUMN IF EXISTS public_url; +-- +goose StatementEnd diff --git a/migrations/sql/20260828222432_dashboard_download_egress.sql b/migrations/sql/20260828222432_dashboard_download_egress.sql new file mode 100644 index 000000000..121e63ac2 --- /dev/null +++ b/migrations/sql/20260828222432_dashboard_download_egress.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- Split the sampled egress series: egress_kbps stays the total viewer egress a +-- source served (unchanged semantics, so existing charts stay truthful), and +-- download_egress_kbps carries the subset served by file-transfer routes +-- (offline downloads, direct downloads, ebook/ABS file fetches) — telemetry +-- ClassTransfer traffic, as opposed to streaming playback. +-- +-- Rows written before this migration report 0 downloads, which reads as "the +-- split was not measured yet", never as inflated playback: playback is derived +-- as egress_kbps - download_egress_kbps. +-- +-- The table is minute-resolution and pruned to a 31-day window (a few hundred +-- thousand rows at most), so a plain in-transaction ALTER is safe. +ALTER TABLE dashboard_metric_samples + ADD COLUMN IF NOT EXISTS download_egress_kbps bigint NOT NULL DEFAULT 0; + +-- +goose Down +ALTER TABLE dashboard_metric_samples + DROP COLUMN IF EXISTS download_egress_kbps; diff --git a/migrations/sql/20260828235638_fix_reopen_image_ladder_ambiguity.sql b/migrations/sql/20260828235638_fix_reopen_image_ladder_ambiguity.sql new file mode 100644 index 000000000..f1a7d9059 --- /dev/null +++ b/migrations/sql/20260828235638_fix_reopen_image_ladder_ambiguity.sql @@ -0,0 +1,187 @@ +-- +goose Up +-- reopen_image_ladder_backfill_v2 declared a plpgsql variable named +-- image_type while its manifest probe filters on the column +-- artwork_revision_gc_candidates.image_type. The bare right-hand reference in +-- "manifest.image_type = image_type" matches both, and plpgsql's default +-- variable_conflict=error raises 42702 the first time the probe runs — which +-- is only once backfilled_version has reached 2, so every local cached-path +-- publication on a v2-complete deployment fails outright instead of reopening +-- the fence. Renaming the variable removes the collision; behavior is +-- otherwise identical. +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION public.reopen_image_ladder_backfill_v2() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + new_row jsonb := to_jsonb(NEW); + old_row jsonb; + arg_index integer := 0; + path_column text; + manifest_image_type text; + rung_pattern text; + cached_path text; + previous_path text; + state_version integer; + changed_cached_path boolean := false; +BEGIN + IF TG_OP = 'UPDATE' THEN + old_row := to_jsonb(OLD); + END IF; + + -- Find an affected local cached-path publication before taking the shared + -- singleton lock. The lock is still taken even when the state is below v2: + -- that closes the ordering with a concurrent final confirmation. + WHILE arg_index < TG_NARGS LOOP + path_column := TG_ARGV[arg_index]; + cached_path := new_row ->> path_column; + previous_path := CASE WHEN old_row IS NULL THEN NULL ELSE old_row ->> path_column END; + IF COALESCE(BTRIM(cached_path), '') <> '' + AND cached_path NOT LIKE '%://%' + AND (TG_OP = 'INSERT' OR cached_path IS DISTINCT FROM previous_path) THEN + changed_cached_path := true; + EXIT; + END IF; + arg_index := arg_index + 3; + END LOOP; + + IF NOT changed_cached_path THEN + RETURN NEW; + END IF; + + SELECT backfilled_version + INTO state_version + FROM public.image_ladder_backfill_state + WHERE id = 1 + FOR UPDATE; + + IF state_version < 2 THEN + RETURN NEW; + END IF; + + arg_index := 0; + WHILE arg_index < TG_NARGS LOOP + path_column := TG_ARGV[arg_index]; + manifest_image_type := TG_ARGV[arg_index + 1]; + rung_pattern := TG_ARGV[arg_index + 2]; + cached_path := new_row ->> path_column; + previous_path := CASE WHEN old_row IS NULL THEN NULL ELSE old_row ->> path_column END; + + IF COALESCE(BTRIM(cached_path), '') <> '' + AND cached_path NOT LIKE '%://%' + AND (TG_OP = 'INSERT' OR cached_path IS DISTINCT FROM previous_path) + AND NOT EXISTS ( + SELECT 1 + FROM public.artwork_revision_gc_candidates manifest + WHERE manifest.original_path = cached_path + AND manifest.image_type = manifest_image_type + AND EXISTS ( + SELECT 1 + FROM unnest(manifest.object_keys) object_key + WHERE object_key LIKE rung_pattern + ) + ) THEN + UPDATE public.image_ladder_backfill_state + SET backfilled_version = LEAST(backfilled_version, 1), + last_attempt_at = NULL, + updated_at = NOW() + WHERE id = 1; + RETURN NEW; + END IF; + + arg_index := arg_index + 3; + END LOOP; + + RETURN NEW; +END; +$$; +-- +goose StatementEnd + +-- +goose Down +-- Restores the previous definition verbatim, including the ambiguous +-- image_type variable this migration exists to remove. +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION public.reopen_image_ladder_backfill_v2() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + new_row jsonb := to_jsonb(NEW); + old_row jsonb; + arg_index integer := 0; + path_column text; + image_type text; + rung_pattern text; + cached_path text; + previous_path text; + state_version integer; + changed_cached_path boolean := false; +BEGIN + IF TG_OP = 'UPDATE' THEN + old_row := to_jsonb(OLD); + END IF; + + WHILE arg_index < TG_NARGS LOOP + path_column := TG_ARGV[arg_index]; + cached_path := new_row ->> path_column; + previous_path := CASE WHEN old_row IS NULL THEN NULL ELSE old_row ->> path_column END; + IF COALESCE(BTRIM(cached_path), '') <> '' + AND cached_path NOT LIKE '%://%' + AND (TG_OP = 'INSERT' OR cached_path IS DISTINCT FROM previous_path) THEN + changed_cached_path := true; + EXIT; + END IF; + arg_index := arg_index + 3; + END LOOP; + + IF NOT changed_cached_path THEN + RETURN NEW; + END IF; + + SELECT backfilled_version + INTO state_version + FROM public.image_ladder_backfill_state + WHERE id = 1 + FOR UPDATE; + + IF state_version < 2 THEN + RETURN NEW; + END IF; + + arg_index := 0; + WHILE arg_index < TG_NARGS LOOP + path_column := TG_ARGV[arg_index]; + image_type := TG_ARGV[arg_index + 1]; + rung_pattern := TG_ARGV[arg_index + 2]; + cached_path := new_row ->> path_column; + previous_path := CASE WHEN old_row IS NULL THEN NULL ELSE old_row ->> path_column END; + + IF COALESCE(BTRIM(cached_path), '') <> '' + AND cached_path NOT LIKE '%://%' + AND (TG_OP = 'INSERT' OR cached_path IS DISTINCT FROM previous_path) + AND NOT EXISTS ( + SELECT 1 + FROM public.artwork_revision_gc_candidates manifest + WHERE manifest.original_path = cached_path + AND manifest.image_type = image_type + AND EXISTS ( + SELECT 1 + FROM unnest(manifest.object_keys) object_key + WHERE object_key LIKE rung_pattern + ) + ) THEN + UPDATE public.image_ladder_backfill_state + SET backfilled_version = LEAST(backfilled_version, 1), + last_attempt_at = NULL, + updated_at = NOW() + WHERE id = 1; + RETURN NEW; + END IF; + + arg_index := arg_index + 3; + END LOOP; + + RETURN NEW; +END; +$$; +-- +goose StatementEnd diff --git a/migrations/sql/20260829025159_optimize_postgres_search_exact_titles.sql b/migrations/sql/20260829025159_optimize_postgres_search_exact_titles.sql index 3f4e87e33..c538ee346 100644 --- a/migrations/sql/20260829025159_optimize_postgres_search_exact_titles.sql +++ b/migrations/sql/20260829025159_optimize_postgres_search_exact_titles.sql @@ -43,6 +43,16 @@ CREATE TRIGGER trg_episode_catalog_entries_search_fields BEFORE INSERT OR UPDATE OF episode_id, title ON public.episode_catalog_entries FOR EACH ROW EXECUTE FUNCTION public.set_episode_catalog_entry_search_fields(); +-- Migration 142 refreshes episode_catalog_entries when searchable episode +-- metadata changes. Include overview in that existing trigger so editing an +-- episode description rewrites the stored overview vector as well; without +-- this, overview search stays stale until another episode field changes. +DROP TRIGGER IF EXISTS trg_episode_catalog_entries_episodes ON public.episodes; +CREATE TRIGGER trg_episode_catalog_entries_episodes +AFTER INSERT OR UPDATE OF content_id, series_id, title, episode_number, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, overview, created_at OR DELETE +ON public.episodes +FOR EACH ROW EXECUTE FUNCTION public.episode_catalog_entries_episodes_trigger(); + UPDATE public.episode_catalog_entries ece SET search_title_normalized = public.normalize_search_text(ece.title), @@ -110,6 +120,14 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episode_catalog_entries_search_episo ON public.episode_catalog_entries (episode_id, media_folder_id); -- +goose Down +-- Restore migration 142's original refresh set when the stored overview search +-- document is removed. +DROP TRIGGER IF EXISTS trg_episode_catalog_entries_episodes ON public.episodes; +CREATE TRIGGER trg_episode_catalog_entries_episodes +AFTER INSERT OR UPDATE OF content_id, series_id, title, episode_number, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, created_at OR DELETE +ON public.episodes +FOR EACH ROW EXECUTE FUNCTION public.episode_catalog_entries_episodes_trigger(); + DROP INDEX CONCURRENTLY IF EXISTS public.idx_episode_catalog_entries_search_episode; DROP INDEX CONCURRENTLY IF EXISTS public.idx_episode_catalog_entries_search_overview; DROP INDEX CONCURRENTLY IF EXISTS public.idx_episode_catalog_entries_search_title; diff --git a/web/src/App.tsx b/web/src/App.tsx index 1a9025a5b..ab4f11737 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -8,7 +8,9 @@ import { type ReactNode, } from "react"; import { - BrowserRouter, + createBrowserRouter, + createRoutesFromElements, + Outlet, Routes, Route, Navigate, @@ -17,11 +19,16 @@ import { useParams, useSearchParams, } from "react-router"; +// The DOM build of the provider is the same component with `ReactDOM.flushSync` +// wired in, which is what lets a view-transition navigation apply its state +// inside `startViewTransition`. Links across the app opt into view transitions. +import { RouterProvider } from "react-router/dom"; import { QueryClientProvider, useQueryClient } from "@tanstack/react-query"; import { queryClient } from "@/lib/query-client"; import { AuthProvider, useAuth } from "@/hooks/useAuth"; import { useCurrentProfile } from "@/hooks/useCurrentProfile"; import { useIsActingAdmin } from "@/hooks/useIsActingAdmin"; +import { useNavigationDirection } from "@/hooks/useNavigationDirection"; import { ThemeProvider } from "@/hooks/useTheme"; import { DateTimeFormatProvider, useDateTimeFormat } from "@/hooks/useDateTimeFormat"; import { CustomThemeProvider } from "@/contexts/CustomThemeProvider"; @@ -32,7 +39,6 @@ import ImpersonationBanner from "@/components/ImpersonationBanner"; import { loadStoredImpersonationAdminSession } from "@/lib/impersonationSession"; import { Toaster } from "@/components/ui/sonner"; import { RealtimeEventsProvider } from "@/components/RealtimeEventsProvider"; -import NavigationTransitionProvider from "@/components/NavigationTransitionProvider"; import { useEventChannel } from "@/components/realtimeEventsContext"; import { useSettingValuesRealtime } from "@/hooks/queries/settingValues"; import Layout from "@/components/Layout"; @@ -58,20 +64,30 @@ import { buildQueryCatalogHref, buildUserCollectionCatalogHref, } from "@/pages/catalogSearchParams"; +import { buildLegacyAutoscanRedirectTarget } from "@/pages/autoscanSearchParams"; import { buildLegacyWebhookSyncRedirectTarget } from "@/lib/webhookSync"; import { toast } from "sonner"; import { prewarmCodecDetection } from "@/player/hooks/useCodecDetection"; +import { prefetchRouteChunks, type RouteChunkImport } from "@/lib/routeChunkPrefetch"; + +// Hot routes keep their import factory in a named binding so the idle warm-up +// below can pull the chunk before the user navigates. See HOT_ROUTE_CHUNKS. +const importLibraryPage = () => import("@/pages/LibraryPage"); +const importItemDetail = () => import("@/pages/ItemDetail/index"); +const importPersonDetail = () => import("@/pages/PersonDetail"); +const importCollections = () => import("@/pages/Collections"); +const importRecommendations = () => import("@/pages/Recommendations"); const AdminLayout = lazy(() => import("@/components/AdminLayout")); const OAuthComplete = lazy(() => import("@/pages/OAuthComplete")); const ActivateDevice = lazy(() => import("@/pages/ActivateDevice")); const SetupWizard = lazy(() => import("@/pages/SetupWizard")); const Profiles = lazy(() => import("@/pages/Profiles")); -const LibraryPage = lazy(() => import("@/pages/LibraryPage")); -const ItemDetail = lazy(() => import("@/pages/ItemDetail/index")); +const LibraryPage = lazy(importLibraryPage); +const ItemDetail = lazy(importItemDetail); const EbookReader = lazy(() => import("@/pages/EbookReader")); -const PersonDetail = lazy(() => import("@/pages/PersonDetail")); -const Collections = lazy(() => import("@/pages/Collections")); +const PersonDetail = lazy(importPersonDetail); +const Collections = lazy(importCollections); const CollectionEditor = lazy(() => import("@/pages/CollectionEditor")); const Notifications = lazy(() => import("@/pages/Notifications")); const DeviceSettings = lazy(() => import("@/pages/settings/DeviceSettings")); @@ -86,7 +102,6 @@ const AdminDiagnostics = lazy(() => import("@/pages/AdminDiagnostics")); const AdminAccessGroups = lazy(() => import("@/pages/AdminAccessGroups")); const AdminUsers = lazy(() => import("@/pages/AdminUsers")); const AdminRequests = lazy(() => import("@/pages/AdminRequests")); -const AdminAutoscan = lazy(() => import("@/pages/AdminAutoscan")); const AdminDevices = lazy(() => import("@/pages/AdminDevices")); const AdminLibraries = lazy(() => import("@/pages/AdminLibraries")); const AdminSettingsLayout = lazy(() => import("@/pages/admin-settings/AdminSettingsLayout")); @@ -106,7 +121,7 @@ const AdminPlugins = lazy(() => import("@/pages/AdminPlugins")); const AdminHistoryImport = lazy(() => import("@/pages/AdminHistoryImport")); const AdminRecommendations = lazy(() => import("@/pages/AdminRecommendations")); const AdminPolicyLayout = lazy(() => import("@/pages/admin-policy/AdminPolicyLayout")); -const Recommendations = lazy(() => import("@/pages/Recommendations")); +const Recommendations = lazy(importRecommendations); const RecommendationsSection = lazy(() => import("@/pages/RecommendationsSection")); const Calendar = lazy(() => import("@/pages/Calendar")); const Signup = lazy(() => import("@/pages/Signup")); @@ -134,11 +149,31 @@ const WatchTogetherRoomPage = lazy(() => import("@/pages/WatchTogetherRoomPage") const WatchRoute = lazy(() => import("@/pages/WatchRoute")); const ProfileCustomizeHome = lazy(() => import("@/pages/ProfileCustomizeHome")); -/** Scrolls to top on pathname change (custom replacement for ScrollRestoration which requires data router). */ +/** + * Routes a browsing session reaches within the first few interactions. Home + * links straight into item details, the sidebar into libraries, and item pages + * into people and recommendations, so paying their chunk cost while the app is + * idle is cheaper than paying it inside a navigation. + */ +const HOT_ROUTE_CHUNKS: readonly RouteChunkImport[] = [ + importItemDetail, + importLibraryPage, + importPersonDetail, + importRecommendations, + importCollections, +]; + +/** + * Scrolls to top on pathname change. Kept in place of react-router's + * ``, which the data router would now allow: that one + * restores the previous offset on back/forward, while every page here expects + * to open at the top. + */ function useScrollRestoration() { const { pathname } = useLocation(); - // Run before paint so route transitions never capture one frame at the - // previous page's scroll offset and then visibly jump to the top. + // Layout effect, not effect: after paint the browser has already shown one + // frame of the new route at the old route's scroll offset, which reads as + // a jump to the top rather than an arrival at it. useLayoutEffect(() => { window.scrollTo(0, 0); }, [pathname]); @@ -169,6 +204,17 @@ function ScrollRestorationManager() { return null; } +/** + * Tracks history provenance and the direction the page is moving in. A leaf + * rather than a call inside `AppShell`: the hook reads the location, and + * subscribing `AppShell` to it would re-render the whole provider stack on + * every navigation. + */ +function NavigationDirectionManager() { + useNavigationDirection(); + return null; +} + function RouteLoading() { return (
@@ -365,6 +411,16 @@ function LegacyWebhookSyncRedirect() { return ; } +/** + * `/admin/autoscan` → the Autoscan tab on Libraries. The old query has to be + * translated, not dropped: the panel's own view moved from `tab` to `view` + * because `tab` now names the Libraries tab hosting it. + */ +function LegacyAutoscanRedirect() { + const { search } = useLocation(); + return ; +} + function LegacyPersonalCatalogRedirect({ source, }: { @@ -468,7 +524,8 @@ function AppRoutes() { } /> } /> } /> - } /> + {/* Autoscan is a tab on Libraries now; keep old links working. */} + } /> } /> } /> } /> @@ -480,7 +537,7 @@ function AppRoutes() { } /> } /> } /> - } /> + } /> } /> } /> } /> @@ -673,43 +730,90 @@ function PlaybackCapabilityPrewarmer() { return null; } -export default function App() { +/** Warms the hot route chunks once the first screen has settled. */ +function RouteChunkPrewarmer() { + const { user } = useAuth(); + const isAuthenticated = Boolean(user); + useEffect(() => { + // Nothing behind these routes is reachable while signed out, and the login + // screen is exactly where bandwidth should stay free for the first paint. + if (!isAuthenticated) return; + return prefetchRouteChunks(HOT_ROUTE_CHUNKS); + }, [isAuthenticated]); + return null; +} + +/** + * Everything that used to sit directly inside ``: providers, + * app-wide chrome, and the routed page tree behind one Suspense boundary. + * + * `RouterProvider` takes no children, so this is the data router's single root + * layout route. Nothing was hoisted above the router: ErrorBoundary, + * RealtimeEventsProvider and WatchPlaybackProvider all read the location or + * navigate, and the providers that need nothing from the router sit above those + * in the chain — so splitting the stack would reorder providers for no gain. + * The element below is created once at module scope, so React skips + * re-rendering this subtree on navigation exactly as it did when the tree hung + * off ``. + */ +function AppShell() { return ( - - - - - - - - - - - - - - - - - - - }> - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + }> + + + + + + + + + + + + + + + ); } + +/** + * A data router is what makes navigation blockable (`useBlocker`, used by + * `UnsavedChangesGuard` to protect staged settings edits) — that is the whole + * reason for `createBrowserRouter` here. The route tree itself stays + * declarative below the root: the splat child hands off to ``, whose + * descendant routes match from `/` because a splat contributes nothing to the + * pathname base. + */ +const appRoutes = createRoutesFromElements( + }> + } /> + , +); + +export default function App() { + // Per-App-instance rather than a module-scope singleton: a router captures + // the current history the moment it is built, and tests render App more than + // once against different entries. + const [router] = useState(() => createBrowserRouter(appRoutes)); + + return ; +} diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 89246d17c..855e53f99 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -2464,14 +2464,25 @@ export interface AdminStats { total_show_files?: number; active_streams: number; total_storage_bytes: number; - watch_provider_activity: WatchProviderActivity; + /** + * One entry per watch provider, ordered by `provider`. Covers every provider + * registered on the server — including ones a plugin contributes at runtime, + * which appear with zeros — plus any provider that still has stored activity + * after its plugin was removed (`registered: false`). + */ + watch_providers: WatchProviderStats[]; } -export interface WatchProviderActivity { - trakt_connected_profiles: number; - trakt_enabled_profiles: number; - trakt_export_enabled: number; - trakt_scrobble_enabled: number; +export interface WatchProviderStats { + provider: string; + display_name: string; + registered: boolean; + scrobbling: boolean; + exporting: boolean; + connected_profiles: number; + enabled_profiles: number; + export_enabled_profiles: number; + scrobble_enabled_profiles: number; last_sync_completed_at?: string; sync_runs_24h: number; sync_errors_24h: number; @@ -3793,11 +3804,174 @@ export interface UpdatePluginSettingsRequest { } // Stream Nodes + +/** One render device in a node's stored capability report. */ +export interface NodeRenderDevice { + path: string; + /** sysfs PCI slot (e.g. "0000:03:00.0"); absent for a non-PCI device. */ + pci_address?: string; + /** NVIDIA's permanent GPU identity; only present with nvidia-smi installed. */ + gpu_uuid?: string; + description: string; +} + +/** One hardware backend with candidate devices, plus its probe outcome. */ +export interface NodeDetectedBackend { + backend: string; + /** A real single-frame encode passed, not just an FFmpeg build-flag listing. */ + verified: boolean; + devices?: string[]; + /** The candidate that passed. Empty for NVENC, which selects via CUDA. */ + device?: string; + /** Why verification failed, attributed per device when several were tried. */ + reason?: string; + /** + * No probe was attempted: none of the backend's candidate devices is + * accessible to this node (e.g. a proxy reading the cluster-wide hw_device). + * Not a driver failure; reason lists the skipped devices. + */ + skipped?: boolean; +} + +/** + * A node's stored hardware capability report — the body its /hw-capabilities + * endpoint served. The payload also carries the node's transformation and + * tone-map advertisements, which no admin surface reads yet. + */ +export interface NodeCapabilities { + /** Backend that would actually be used: nvenc, qsv, vaapi, or none. */ + resolved?: string; + render_devices?: string[] | null; + render_device_details?: NodeRenderDevice[] | null; + intel_detected?: boolean; + detected_backends?: NodeDetectedBackend[]; + /** Kernel boot identity (Linux only); scopes pci_address to one boot. */ + boot_id?: string; + /** + * Every GPU nvidia-smi reports, sorted. Independent of + * render_device_details: an NVIDIA container often has /dev/nvidia* and the + * toolkit but no /dev/dri, so this is the only identity such a host has. + */ + nvidia_gpu_uuids?: string[]; + /** "sha256:" over the report's hardware identity and capabilities. */ + capability_hash?: string; + source?: string; + node_url?: string; +} + +/** One sampled mount inside a resource sample. */ +export interface HostDiskStats { + /** + * Where the mount is. Present only on credentialed surfaces — a node's + * `/status` and `GET /admin/system/resources`. A node's `/health` takes no + * credential and therefore omits it, so anything rendered from `last_stats` + * must fall back to `role`. + */ + path?: string; + /** + * What the mount is for: `scratch` for the transcode working directory, + * `library-N` positionally for each media root. Assigned server-side when the + * sample is built, so it names the same mount on every surface. + */ + role?: string; + /** Capacity in GiB. `used_gb` counts filesystem-reserved blocks, as `df` does. */ + used_gb?: number; + /** + * Capacity usable by the node process — used plus still-available. Blocks a + * filesystem reserves for root are in neither, so this reads lower than the + * device's nameplate size, and `used_gb`/`total_gb` is `df`'s Use%. + */ + total_gb?: number; + /** Real numbers carried over from an earlier pass because the probe has not returned. */ + stale?: boolean; + /** Never measured on this host: `used_gb`/`total_gb` are meaningless. */ + unavailable?: boolean; + /** + * The node's transcode working directory — the one mount whose filling up + * breaks transcoding rather than browsing. Set on at most one entry per + * sample; a media root sharing that volume is deduplicated onto it. Absent on + * every other mount, and on a node predating the flag. + */ + scratch?: boolean; +} + +/** + * A host's CPU/memory/disk/network sample. Every field is optional: sampling is + * Linux-only, individual probes degrade independently, and a server predating + * resource sampling sends none of this. + */ +export interface HostSystemStats { + /** + * Aggregate busy percentage across all cores over the sampling interval, 0-100. + * Under a cgroup it is that container's own usage against its own quota. + */ + cpu_pct?: number; + /** 1-minute load average; unlike cpu_pct it also counts tasks blocked on storage. */ + load1?: number; + /** CPUs this host may use: the cgroup quota where one is set, otherwise the kernel's count. */ + cores?: number; + mem_used_mb?: number; + mem_total_mb?: number; + /** Scratch dir first, then media roots; deduplicated by filesystem. */ + disks?: HostDiskStats[] | null; + /** Aggregate throughput in *bits* per second, loopback excluded. */ + net_rx_bps?: number; + net_tx_bps?: number; +} + +/** One GPU's sample. */ +export interface HostGPUStats { + /** Render node path (/dev/dri/renderD128), a PCI address, or "cuda:N". */ + device?: string; + vendor?: string; + /** Workloads this host has pinned to the device, from the playback balancer. */ + sessions?: number; + /** Engine busy percentages over the sampling interval. */ + video_busy_pct?: number; + render_busy_pct?: number; + /** Whole-GPU utilization including other tenants. Absent is not zero. */ + total_busy_pct?: number | null; + vram_used_mb?: number | null; + vram_total_mb?: number | null; + /** "fdinfo", "nvidia-smi", "fdinfo+nvidia-smi", or "unavailable". */ + source?: string; +} + +/** + * A node's most recent resource sample, written by the same health check that + * writes `active_jobs` — so it is exactly as old as `last_health_check` and + * never fresher. Absent on a node that reports none, and on every server + * predating resource sampling. + */ +export interface NodeLastStats { + system?: HostSystemStats | null; + gpu?: HostGPUStats[] | null; +} + +/** + * The API host's own sample (GET /admin/system/resources) — the counterpart to + * a node's `last_stats`. `available` is false on a host that cannot be sampled + * (non-Linux, no sampler, or before the first sample lands), in which case the + * rest is absent. + */ +export interface SystemResources { + available?: boolean; + sampled_at?: string; + system?: HostSystemStats | null; + gpu?: HostGPUStats[] | null; +} + export interface StreamNode { id: number; name: string; type: string; url: string; + /** + * Client-facing base URL when it differs from `url`. `url` is the backend + * address the server and nodes dial; this is what stream and download URLs + * are built on for proxy nodes. Absent or null means clients use `url`. + */ + public_url?: string | null; enabled: boolean; healthy: boolean; active_jobs: number; @@ -3807,12 +3981,50 @@ export interface StreamNode { egress_kbps: number; last_health_check: string | null; created_at: string; + // Capability fields are owned by the background health sweep and are absent + // until one report has been stored — and on every server predating them. + capabilities?: NodeCapabilities | null; + capabilities_hash?: string; + /** + * The hash the node named on its last health check, present only once a check + * has happened. It differs from `capabilities_hash` while a refetch is + * outstanding or failing, which is the one case a fresh `last_health_check` + * cannot rule out; it is present and empty when the node answers with no hash + * at all, as a build predating capability reports does. Absent means nothing + * has asked yet, which says nothing about the stored report. + */ + advertised_capabilities_hash?: string; + /** When `capabilities` was fetched: the age of the inventory, not the health check. */ + capabilities_refreshed_at?: string; + /** Stable per-GPU identities; two nodes sharing one share hardware. */ + physical_gpu_keys?: string[]; + /** The node's resource sample from the last health check. */ + last_stats?: NodeLastStats | null; + /** + * This node's own acceleration policy. Absent or null is the normal case: + * the node inherits the cluster-wide playback.hw_accel / playback.hw_device + * settings. A value here is what the node resolves against from its next + * config reload, and what remote transcodes to it are dispatched with. + */ + hw_accel_override?: string | null; + /** Comma-separated render device paths pinned to this node; null inherits. */ + hw_device_override?: string | null; + /** + * Human-readable note describing how this node's hardware got worse at the + * last capability refetch: a backend that used to pass its probe and now + * fails, or a render device that is gone. Absent means the last refetch found + * no regression — it is not a latched incident, and a repaired node loses the + * note on its next refetch. Nothing routes on it. + */ + capability_drift?: string | null; } export interface CreateNodeRequest { name: string; type: string; url: string; + // Client-facing base URL, proxy nodes only; empty means clients use `url`. + public_url?: string; group?: string; max_jobs?: number; max_bandwidth_kbps?: number; @@ -3821,11 +4033,18 @@ export interface CreateNodeRequest { export interface UpdateNodeRequest { name?: string; url?: string; + // An omitted public_url leaves the stored value alone; an explicit null (or + // an empty string) sends clients back to `url`. + public_url?: string | null; enabled?: boolean; // Empty string clears the group; 0 clears the caps (unlimited). group?: string; max_jobs?: number; max_bandwidth_kbps?: number; + // An omitted override leaves the stored value alone; an explicit null (or an + // empty string) restores inheritance of the cluster-wide playback setting. + hw_accel_override?: string | null; + hw_device_override?: string | null; } export interface CheckNodeResponse { @@ -3834,6 +4053,27 @@ export interface CheckNodeResponse { egress_kbps: number; } +/** + * Answer to POST /admin/nodes/{id}/reprobe. The call is always 200: a node that + * refused or could not be reached is reported as `status: "error"` here rather + * than as an HTTP status, matching the per-node check route. + */ +export interface ReprobeNodeResult { + node_id: number; + node_name: string; + status: "ok" | "error"; + error?: string; + /** Backend the node picked after re-probing: nvenc, qsv, vaapi, or none. */ + resolved?: string; + /** Hash of the snapshot the node published; compare against `capabilities_hash`. */ + capability_hash?: string; + /** + * This server also refetched and stored the node's new inventory before + * answering. False means the stored row catches up on a later health sweep. + */ + capabilities_refreshed: boolean; +} + // User-facing library (simplified, no admin fields) export interface UserLibrary { id: number; @@ -4358,6 +4598,12 @@ export interface RateLimitConfig { active?: boolean; /** Backend the running limiter uses; may differ from `backend` until restart. */ active_backend?: string; + /** + * Whether the Redis backend can be selected at all (GET responses only). + * Sentinel and REDIS_URL deployments have no stored `redis.url`, so only the + * server can answer this. + */ + redis_available?: boolean; } export interface RateLimitUpdateResponse { @@ -4382,13 +4628,176 @@ export interface AdminSettingsUpdateResponse { restart_required_keys?: string[]; } +// Admin dashboard layout (per admin account, server-persisted). +// +// The server stores the document verbatim and validates only its size and that +// it is a JSON object: widget ids, column spans and row heights are the web +// client's vocabulary. `layout` is therefore typed as `unknown` on the read +// side so callers must sanitize it before use — a layout written by a newer or +// older build can name widgets this one does not have, or omit `rows`, which +// predates two-axis resizing. +export interface AdminDashboardLayoutEntry { + id: string; + span: number; + rows: number; +} + +export interface AdminDashboardLayoutDocument { + version: number; + entries: AdminDashboardLayoutEntry[]; +} + +export interface AdminDashboardLayoutResponse { + layout: unknown; + updated_at: string | null; +} + +// One backing service on the admin health strip. `configured: false` means the +// deployment runs without it — a supported single-node shape for Redis — and +// `ok` is then absent rather than false, so "not present" and "present but +// broken" stay distinguishable. +export interface AdminHealthComponent { + configured: boolean; + ok?: boolean; + latency_ms?: number; +} + +export interface AdminServerHealth { + postgres: AdminHealthComponent; + redis: AdminHealthComponent; + errors_24h: number; + warnings_24h: number; +} + export interface AdminServerStatus { started_at: string; restart_required: boolean; restart_required_at?: string; restart_required_reason?: string; + /** + * Every distinct reason marked since boot ("setting:" for settings + * saves), so pending restarts can be scoped per subsystem. The singular + * field only remembers the last save. + */ + restart_required_reasons?: string[]; + /** Increments on every restart-required save; re-arms the dismissed banner. */ + restart_mark_count?: number; restart_requested: boolean; restart_requested_at?: string; + /** Absent on servers predating the dashboard health summary. */ + health?: AdminServerHealth; +} + +// GET /admin/stats/playback-activity. `buckets` carries only hours that saw a +// session, so the client zero-fills the window before charting it. +export interface AdminPlaybackActivityBucket { + hour: string; + direct: number; + remux: number; + transcode: number; +} + +// Time-to-first-frame and failed-start counts are deliberately absent: nothing +// records playback start events yet. See docs/admin-api.md. +export interface AdminPlaybackReliability { + sessions_started: number; + transcode_starts: number; + finalized_sessions: number; + completed_sessions: number; + completion_rate: number; + unique_profiles: number; +} + +// `bucket_seconds` is 3600 up to a two-day window and 86400 beyond it; the +// client zero-fills the window on that grid. `hour` on a bucket is its start +// instant at either width. +export interface AdminPlaybackActivity { + hours: number; + bucket_seconds: number; + // The window on the server's clock; the chart anchors its bucket grid on + // `to` so client clock skew cannot misplace the boundary buckets. Optional + // because responses predating the fields lack them. + from?: string; + to?: string; + buckets: AdminPlaybackActivityBucket[]; + reliability: AdminPlaybackReliability; + profiles_active_24h: number; +} + +// GET /admin/stats/top-activity. Episodes are rolled up to their series, so a +// title's media_item_id is a series content id for TV. +export interface AdminTopTitle { + media_item_id: string; + title: string; + media_type: string; + plays: number; + total_seconds: number; +} + +export interface AdminTopProfile { + user_id: number; + username: string; + profile_id: string; + profile_name: string; + plays: number; + total_seconds: number; +} + +export interface AdminTopActivity { + days: number; + limit: number; + titles: AdminTopTitle[]; + profiles: AdminTopProfile[]; +} + +// GET /admin/stats/timeseries. One point per sampled minute; minutes the +// sampler missed are absent rather than zero, so charts draw them as gaps. +export interface AdminTimeseriesPoint { + t: string; + streams: number; + direct: number; + remux: number; + transcode: number; + egress_kbps: number; + // File-transfer subset of `egress_kbps` (offline/direct downloads, ebook and + // ABS file fetches, API-served only). Always <= egress_kbps; playback egress + // is the difference. Optional because responses predating the split lack it, + // and 0 on samples written before the split — "not measured", not "no + // downloads". + download_egress_kbps?: number; +} + +// `oldest_sample_at` is null until the sampler has written anything, which is +// what the "collecting data" chart state keys off. `resolution_seconds` is the +// bucket the server aggregated into, which widens with the requested window — +// read it rather than assuming the sampler's minute. +export interface AdminTimeseries { + resolution_seconds: number; + from: string; + to: string; + oldest_sample_at: string | null; + points: AdminTimeseriesPoint[]; +} + +// GET /admin/stats/downloads. Headline numbers and top_users count active +// managed device entries (media somebody keeps offline); the 24h counters also +// include one-shot web downloads. All zeros with an empty top_users on a +// deployment where nobody downloads — the widget's empty state, not an error. +export interface AdminDownloadsUser { + user_id: number; + username: string; + downloads: number; + total_bytes: number; +} + +export interface AdminDownloadsStats { + users_with_downloads: number; + active_downloads: number; + total_bytes: number; + downloads_started_24h: number; + downloads_completed_24h: number; + limit: number; + top_users: AdminDownloadsUser[]; } // IP visibility diff --git a/web/src/app.css b/web/src/app.css index c6658e6ea..776641d73 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -113,6 +113,12 @@ /* ── Hero Backdrop Defaults ─────────────────────────────────── */ --hero-backdrop-brightness: 0.75; --hero-backdrop-saturate: 0.9; + /* Detail heroes tint their artwork with a flat `--background` wash instead + of a `filter`, so a viewport resize does not re-rasterize a filtered + layer every frame. This is the detail-page equivalent of + `--hero-backdrop-brightness`: 25% over a dark background lands close to + brightness(0.75). Themes tune it here rather than losing the knob. */ + --hero-backdrop-scrim: 25%; --hero-text-shadow: 0 18px 50px rgb(0 0 0 / 45%); } @@ -181,46 +187,6 @@ opacity: 0; } } -@keyframes route-forward-out { - from { - opacity: 1; - transform: translateX(0); - } - to { - opacity: 0; - transform: translateX(-8px); - } -} -@keyframes route-forward-in { - from { - opacity: 0; - transform: translateX(12px); - } - to { - opacity: 1; - transform: translateX(0); - } -} -@keyframes route-back-out { - from { - opacity: 1; - transform: translateX(0); - } - to { - opacity: 0; - transform: translateX(8px); - } -} -@keyframes route-back-in { - from { - opacity: 0; - transform: translateX(-12px); - } - to { - opacity: 1; - transform: translateX(0); - } -} @keyframes ken-burns-a { 0% { transform: scale(1) translate(0, 0); @@ -276,6 +242,30 @@ } } +/* Directional page motion for nested navigation. `--nav-slide-shift` is the + distance the OUTGOING page travels; the incoming page always enters from the + opposite side, so one pair of keyframes covers both directions. */ +@keyframes nav-slide-out { + from { + opacity: 1; + transform: translateX(0); + } + to { + opacity: 0; + transform: translateX(var(--nav-slide-shift)); + } +} +@keyframes nav-slide-in { + from { + opacity: 0; + transform: translateX(calc(-1 * var(--nav-slide-shift))); + } + to { + opacity: 1; + transform: translateX(0); + } +} + /* ================================================================ VIEW TRANSITIONS — Page navigation animations ================================================================ @@ -284,32 +274,68 @@ and animates between them. ================================================================ */ -/* Route motion uses the viewport-sized root snapshot. Naming the variable- - height scrolling
makes the browser interpolate its width, height, - and screen position with layout work on every frame—especially visible - when a scrolled season returns to a shorter series page. Desktop item - entry/exit still bypasses snapshots so its live sidebar motion is preserved. */ -::view-transition-old(root) { - animation: 180ms cubic-bezier(0.4, 0, 1, 1) both route-forward-out; - mix-blend-mode: normal; +/* The sidebar collapse (`.sidebar-surface`) is a live CSS transition running + across the same navigation. The root snapshot would freeze the old 260px + sidebar and cross-fade it over that motion, so inside the sidebar shell the + root group is held still and `main-content` carries the whole transition; the + rail, the impersonation banner, and the mobile header keep animating in the + live document. + + Scoped to `html[data-app-shell]`, which `Layout` sets while it is mounted, + because `main-content` is named in `Layout` and nowhere else. Un-naming the + root outright would leave the routes that render outside it — /watch, + /reader/*, /admin/*, /login — with no captured element at all, so their + navigations would have nothing to animate rather than one thing too many. */ +html[data-app-shell]::view-transition-old(root), +html[data-app-shell]::view-transition-new(root) { + animation: none; } -::view-transition-new(root) { - animation: 260ms cubic-bezier(0.22, 1, 0.36, 1) both route-forward-in; - mix-blend-mode: normal; +/* main-content's own box moves with the rail (`lg:ml-[260px]` to `lg:ml-16`), + so the captured group has to travel on the rail's clock rather than the UA's + 250ms default, or the two edges separate near the end of the collapse. */ +::view-transition-group(main-content) { + animation-duration: var(--duration-sidebar-collapse); + animation-timing-function: var(--ease-sidebar-collapse); } -html[data-navigation-direction="back"]::view-transition-old(root) { - animation-name: route-back-out; +::view-transition-old(main-content) { + animation: 200ms cubic-bezier(0, 0, 0.2, 1) both fade-out; } -html[data-navigation-direction="back"]::view-transition-new(root) { - animation-name: route-back-in; +::view-transition-new(main-content) { + animation: 300ms cubic-bezier(0, 0, 0.2, 1) both slide-up; +} + +/* Nested detail navigation moves sideways instead, on the sidebar's own tokens, + so following a breadcrumb up reads as the reverse of the push that made it. + Navigations whose direction is unknowable — a POP whose entry predates this + page load, an external pushState — leave the attribute off and fall through + to the neutral rules above. Both halves share one duration and easing: the + pseudo-elements default to `mix-blend-mode: plus-lighter`, which only + cross-fades cleanly while the two opacities sum to 1. */ +html[data-navigation-direction="forward"] { + --nav-slide-shift: -24px; +} +html[data-navigation-direction="back"] { + --nav-slide-shift: 24px; +} +html[data-navigation-direction]::view-transition-old(main-content) { + animation: var(--duration-sidebar-collapse) var(--ease-sidebar-collapse) both nav-slide-out; +} +html[data-navigation-direction]::view-transition-new(main-content) { + animation: var(--duration-sidebar-collapse) var(--ease-sidebar-collapse) both nav-slide-in; } @media (prefers-reduced-motion: reduce) { - ::view-transition-old(root), - ::view-transition-new(root) { + /* The directional selectors carry their own specificity — (0,1,2) against the + bare pseudo-elements' (0,0,1) — so they have to be suppressed by name here + or they would keep animating for someone who asked for no motion. */ + ::view-transition-old(main-content), + ::view-transition-new(main-content), + ::view-transition-group(main-content), + html[data-navigation-direction]::view-transition-old(main-content), + html[data-navigation-direction]::view-transition-new(main-content) { animation: none; } :root { @@ -424,12 +450,14 @@ html[data-navigation-direction="back"]::view-transition-new(root) { --input: #1c1c20; --ring: #e8e8ec; - /* Charts: desaturated pastels that fit the cinema aesthetic */ - --chart-1: #8b9cf7; - --chart-2: #7ec8e3; - --chart-3: #81c995; - --chart-4: #e8a87c; - --chart-5: #c78dbd; + /* Charts: categorical set validated on this theme's card surface — + lightness band, chroma floor, adjacent-pair CVD separation, and + contrast all pass. Assigned in fixed entity order, never cycled. */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; /* Sidebar: darkest surface, creating depth */ --sidebar: #0f0f12; @@ -486,12 +514,13 @@ html[data-navigation-direction="back"]::view-transition-new(root) { --input: #e8e8ec; --ring: #1a1a1e; - /* Charts: richer tones for light backgrounds */ - --chart-1: #6366f1; - --chart-2: #0ea5e9; - --chart-3: #22c55e; - --chart-4: #f59e0b; - --chart-5: #a855f7; + /* Charts: light-mode steps of the same categorical set, validated against + the #ffffff card surface (dark mode is selected, never an auto flip). */ + --chart-1: #4c60dd; + --chart-2: #0b8a75; + --chart-3: #9c6410; + --chart-4: #a63f75; + --chart-5: #2b78ab; /* Sidebar: slightly darker than background for depth */ --sidebar: #eaeaee; @@ -514,6 +543,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { /* Hero: light-mode overrides */ --hero-backdrop-brightness: 1.15; --hero-backdrop-saturate: 0.6; + /* A heavier wash of the near-white background both lifts and desaturates, + matching what brightness(1.15)/saturate(0.6) did for this theme. */ + --hero-backdrop-scrim: 35%; --hero-text-shadow: 0 1px 3px rgb(255 255 255 / 60%); } @@ -540,11 +572,12 @@ html[data-navigation-direction="back"]::view-transition-new(root) { --border: #28384d; --input: #182231; --ring: #78aefc; - --chart-1: #78aefc; - --chart-2: #87d2ff; - --chart-3: #6ec7ba; - --chart-4: #b9c7ff; - --chart-5: #f18d8d; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #0c131d; --sidebar-foreground: #f4f8ff; --sidebar-primary: #78aefc; @@ -584,11 +617,12 @@ html[data-navigation-direction="back"]::view-transition-new(root) { --border: #3b2830; --input: #251a1f; --ring: #d16a78; - --chart-1: #d16a78; - --chart-2: #f08e7a; - --chart-3: #c8a0b8; - --chart-4: #8aa0c6; - --chart-5: #e1b86e; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #120d0f; --sidebar-foreground: #f8f2f3; --sidebar-primary: #d16a78; @@ -628,11 +662,12 @@ html[data-navigation-direction="back"]::view-transition-new(root) { --border: #284038; --input: #182420; --ring: #5bc39d; - --chart-1: #5bc39d; - --chart-2: #7dd8bd; - --chart-3: #7eb7a4; - --chart-4: #88a9cf; - --chart-5: #e0b66b; + /* Shared dark categorical set (see midnight-cinema). */ + --chart-1: #5f74ee; + --chart-2: #169e88; + --chart-3: #bf7f22; + --chart-4: #c2578f; + --chart-5: #3f97cf; --sidebar: #0c1210; --sidebar-foreground: #f2f8f5; --sidebar-primary: #5bc39d; @@ -697,6 +732,7 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } html[data-high-contrast="true"] .surface-panel-subtle, html[data-high-contrast="true"] .surface-panel, + html[data-high-contrast="true"] .surface-panel-raised, html[data-high-contrast="true"] .glass, html[data-high-contrast="true"] .glass-subtle { border-color: color-mix(in srgb, var(--border) 88%, white 12%); @@ -772,9 +808,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { font-size: calc(32px * var(--ui-text-scale-factor)) !important; } html[data-text-scale="large"] .text-\[38px\], - html[data-text-scale="x-large"] .text-\[38px\] { - font-size: calc(38px * var(--ui-text-scale-factor)) !important; - } + /* Buttons are activatable, so they get the activatable cursor. Declared in + `base` rather than as a utility, so an explicit `cursor-*` class — a + disabled player control, a non-interactive tooltip target — still wins. */ button:not(:disabled), [role="button"]:not([aria-disabled="true"]) { cursor: pointer; @@ -783,6 +819,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { [role="button"][aria-disabled="true"] { cursor: not-allowed; } + html[data-text-scale="x-large"] .text-\[38px\] { + font-size: calc(38px * var(--ui-text-scale-factor)) !important; + } body { background-color: var(--background); color: var(--foreground); @@ -974,8 +1013,8 @@ html[data-navigation-direction="back"]::view-transition-new(root) { transparent 60% ), linear-gradient( - color-mix(in srgb, var(--background) 25%, transparent), - color-mix(in srgb, var(--background) 25%, transparent) + color-mix(in srgb, var(--background) var(--hero-backdrop-scrim, 25%), transparent), + color-mix(in srgb, var(--background) var(--hero-backdrop-scrim, 25%), transparent) ); } @@ -1018,7 +1057,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { lightweight shell first; metadata and artwork stay gated until this compositor motion has completed. Nothing here touches width, margin, or any other layout property, so the - document is laid out once per navigation instead of once per frame. */ + document is laid out once per navigation instead of once per frame. Holding + the surface at a constant size also means its 40px backdrop blur is + rasterized against unchanging geometry rather than re-blurred each frame. */ /* Two transforms that cancel out. `.sidebar-surface` is a 260px frame with `overflow: hidden` that slides 196px left, so its right edge — and the border drawn on it — sweeps from x=260 down to x=64. `.sidebar-inner` @@ -1134,14 +1175,12 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } /* ── Glass Surfaces ──────────────────────────────────────── */ - /* Glass-like floating controls and overlay panels. These use opaque token - blends instead of live backdrop sampling so hover and viewport changes - remain compositor-friendly on every browser. + /* Frosted glass effect for floating controls, overlay panels. Usage:
...
*/ .glass { - background: color-mix(in srgb, var(--surface) 88%, var(--background)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--surface) 70%, transparent); + backdrop-filter: blur(20px) saturate(1.2); + -webkit-backdrop-filter: blur(20px) saturate(1.2); border: 1px solid color-mix(in srgb, var(--border) 50%, transparent); box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.06), @@ -1149,15 +1188,15 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } .glass-subtle { - background: color-mix(in srgb, var(--surface) 82%, var(--background)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--surface) 40%, transparent); + backdrop-filter: blur(12px) saturate(1.1); + -webkit-backdrop-filter: blur(12px) saturate(1.1); border: 1px solid color-mix(in srgb, var(--border) 30%, transparent); box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.05); } - /* Preserve the glass-like hover color while fading a composited overlay - instead of repainting the control surface on every frame. */ + /* Preserve glass hover color while fading a composited overlay + instead of repainting the backdrop-filtered element on every frame. */ .glass-hover { position: relative; isolation: isolate; @@ -1182,17 +1221,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { /* 25% surface over the existing 40% glass equals the previous 55% hover. */ --glass-hover-color: color-mix(in srgb, var(--surface) 25%, transparent); } - .glass-hover-surface-solid { - /* Opaque cards need the actual hover token; another surface tint would be - visually identical over their existing `--surface` background. */ - --glass-hover-color: var(--surface-hover); - } .glass-hover-accent { --glass-hover-color: var(--accent); } - .glass-hover-accent-subtle { - --glass-hover-color: color-mix(in srgb, var(--accent) 60%, transparent); - } /* Detail actions sit directly over large hero artwork. Repainting several live backdrop filters after every button or popover state change blocks @@ -1204,16 +1235,15 @@ html[data-navigation-direction="back"]::view-transition-new(root) { -webkit-backdrop-filter: none; } - /* Detail menus should be ready on the first frame. Their opaque background - avoids sampling the large hero behind them, and no entrance transition - delays pointer or keyboard interaction. */ + /* The detail overflow menu opens without an entrance transition so it is + immediately interactive. Pointer hover is CSS-only. */ .detail-overflow-menu { animation: none !important; transition: none !important; } - /* Rating previews are CSS-only so moving across stars never schedules a - React render or recomposites a live backdrop-filter surface. */ + /* Rating hover stays entirely in CSS so moving across stars does not + schedule React renders or recomposite a live backdrop-filter surface. */ .star-rating { contain: layout paint; background: color-mix(in srgb, var(--surface) 82%, transparent); @@ -1275,9 +1305,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { /* Less-transparent glass for small chips that sit over busy cover art (manga count/status pills) so the label stays legible. */ .glass-chip { - background: color-mix(in srgb, var(--surface) 88%, var(--background)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--surface) 78%, transparent); + backdrop-filter: blur(12px) saturate(1.1); + -webkit-backdrop-filter: blur(12px) saturate(1.1); border: 1px solid color-mix(in srgb, var(--border) 45%, transparent); box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.06); } @@ -1293,9 +1323,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } .glass-dark { - background: color-mix(in srgb, var(--background) 92%, var(--surface)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--background) 80%, transparent); + backdrop-filter: blur(24px) saturate(1.3); + -webkit-backdrop-filter: blur(24px) saturate(1.3); border: 1px solid color-mix(in srgb, var(--border) 40%, transparent); box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.04), @@ -1330,11 +1360,11 @@ html[data-navigation-direction="back"]::view-transition-new(root) { barely-there glass disc that gains opacity on hover so the frame stays clear during playback. */ .player-disc-secondary { - background: rgb(28 28 31); + background: rgb(255 255 255 / 0.06); color: rgb(255 255 255 / 0.92); border: 1px solid rgb(255 255 255 / 0.1); - backdrop-filter: none; - -webkit-backdrop-filter: none; + backdrop-filter: blur(14px) saturate(1.1); + -webkit-backdrop-filter: blur(14px) saturate(1.1); box-shadow: 0 10px 30px -14px rgb(0 0 0 / 0.6), inset 0 1px 0 rgb(255 255 255 / 0.08); @@ -1344,7 +1374,7 @@ html[data-navigation-direction="back"]::view-transition-new(root) { border-color var(--duration-fast) var(--ease-default); } .player-disc-secondary:hover { - background: rgb(43 43 47); + background: rgb(255 255 255 / 0.14); border-color: rgb(255 255 255 / 0.22); transform: scale(1.06); } @@ -1469,14 +1499,24 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } /* ── Media Card ──────────────────────────────────────────── */ - /* Hover effect for poster/backdrop cards. Keep feedback on the image-only - transform below: animating a filter and a second transform on every card - briefly keeps several rasterized card surfaces alive when the pointer is - swept across a row. + /* Hover effect for poster/backdrop cards. + Scale is applied to the image inside the overflow-hidden + .media-card-image container so it stays visually contained + and doesn't overlap adjacent carousels. Usage:
Title
*/ + .media-card { + transition: + transform var(--duration-normal) var(--ease-gentle), + filter var(--duration-fast) var(--ease-default); + } + .media-card:hover { + filter: brightness(1.1); + transform: translateY(-4px); + } + /* Home/library pages can contain hundreds of cards across off-screen rows. Let supporting engines skip layout and paint for those rows until they approach the viewport. Browsers without content-visibility ignore this @@ -1505,15 +1545,6 @@ html[data-navigation-direction="back"]::view-transition-new(root) { contain: layout paint style; } - /* Wide episode cards sit inside a clipped, transformed Embla track. During - a browser-chrome resize Chrome otherwise folds their artwork and text back - into the document paint on every frame. Card-level containment keeps the - same responsive row and hero artwork while bounding that repeated paint - to each card's stable surface. */ - .season-episode-card { - contain: layout paint style; - } - /* Card actions stay hidden at rest so touch devices get clean artwork; a long press opens the action sheet instead. Keyboard focus and an open menu reveal them on every device so keyboard and assistive-tech users @@ -1533,21 +1564,13 @@ html[data-navigation-direction="back"]::view-transition-new(root) { phone cannot uncover the controls. This deliberately uses a direct :hover selector: Tailwind's group-hover variant is limited to the primary pointer's (hover: hover) result, which can exclude Windows hybrid devices - even while an attached mouse is actively hovering the card. */ - @media (any-hover: hover) and (any-pointer: fine) { - .group\/media:hover .media-card-hover-dim { - opacity: 1; - } - .group\/card:hover .media-card-action-trigger, - .group\/media:hover .media-card-play-trigger { - opacity: 1; - pointer-events: auto; - } - } + even while an attached mouse is actively hovering the card. - /* Continue Watching uses the same compositor-only dim reveal as trailer - cards. Changing a full-card background color repaints the artwork when a - pointer is swept across the row; fading this fixed overlay does not. */ + Card actions sit at z-20 in the card wrapper's stacking context while the + overlay-badge layer sits at z-10 inside the artwork box, so a visible + action always covers the badges in that corner. Badges anchor flush in the + corners and never reserve room for the actions; the overlay layer is + pointer-events-none, so covered badges cannot steal an action's clicks. */ .media-card-hover-dim { background: rgb(0 0 0 / 0.3); opacity: 0; @@ -1557,6 +1580,16 @@ html[data-navigation-direction="back"]::view-transition-new(root) { .group\/media:focus-within .media-card-hover-dim { opacity: 1; } + @media (any-hover: hover) and (any-pointer: fine) { + .group\/media:hover .media-card-hover-dim { + opacity: 1; + } + .group\/card:hover .media-card-action-trigger, + .group\/media:hover .media-card-play-trigger { + opacity: 1; + pointer-events: auto; + } + } /* A long press on a card opens the action sheet, so the browser's own image and link callout has to stay out of the way. Text selection is only @@ -1583,7 +1616,7 @@ html[data-navigation-direction="back"]::view-transition-new(root) { } .media-card-image img { transition: - transform var(--duration-fast) var(--ease-gentle), + transform var(--duration-normal) var(--ease-gentle), opacity var(--duration-normal) var(--ease-default); } .media-card:hover .media-card-image img { @@ -1625,11 +1658,11 @@ html[data-navigation-direction="back"]::view-transition-new(root) { line-height: 1rem; letter-spacing: 0.04em; text-transform: uppercase; - background-color: color-mix(in srgb, var(--foreground) 8%, var(--background)); + background-color: color-mix(in srgb, var(--foreground) 8%, transparent); color: color-mix(in srgb, var(--foreground) 72%, var(--muted-foreground)); border: 1px solid color-mix(in srgb, var(--border) 55%, transparent); - backdrop-filter: none; - -webkit-backdrop-filter: none; + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); } /* ── Pill Button ─────────────────────────────────────────── */ @@ -1645,11 +1678,7 @@ html[data-navigation-direction="back"]::view-transition-new(root) { font-size: 0.84rem; font-weight: 700; letter-spacing: 0.01em; - transition: - background-color var(--duration-fast) var(--ease-default), - color var(--duration-fast) var(--ease-default), - transform var(--duration-fast) var(--ease-default), - box-shadow var(--duration-fast) var(--ease-default); + transition: all var(--duration-fast) var(--ease-default); } .pill-primary { background-color: var(--primary); @@ -1657,7 +1686,7 @@ html[data-navigation-direction="back"]::view-transition-new(root) { box-shadow: 0 16px 36px -20px color-mix(in srgb, var(--primary) 60%, transparent); } .pill-primary:hover { - background-color: color-mix(in srgb, var(--primary) 94%, black); + filter: brightness(0.96); transform: translateY(-1px); } .pill-secondary { @@ -1668,14 +1697,14 @@ html[data-navigation-direction="back"]::view-transition-new(root) { background-color: var(--surface-hover); } .pill-glass { - background: color-mix(in srgb, var(--foreground) 12%, var(--background)); + background: color-mix(in srgb, var(--foreground) 12%, transparent); color: var(--foreground); border: 1px solid color-mix(in srgb, var(--border) 50%, transparent); - backdrop-filter: none; - -webkit-backdrop-filter: none; + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); } .pill-glass:hover { - background: color-mix(in srgb, var(--foreground) 18%, var(--background)); + background: color-mix(in srgb, var(--foreground) 18%, transparent); } /* ── Action Circle ───────────────────────────────────────── */ @@ -1778,12 +1807,13 @@ html[data-navigation-direction="back"]::view-transition-new(root) { gap: 1.5rem; padding: 0.9rem 1rem; border-bottom: 1px solid transparent; - background: color-mix(in srgb, var(--background) 92%, var(--surface)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--background) 82%, transparent); + backdrop-filter: blur(18px) saturate(1.15); + -webkit-backdrop-filter: blur(18px) saturate(1.15); border-bottom-color: color-mix(in srgb, var(--border) 55%, transparent); transition: background var(--duration-normal) var(--ease-default), + backdrop-filter var(--duration-normal) var(--ease-default), border-color var(--duration-normal) var(--ease-default); } @media (min-width: 1024px) { @@ -1800,9 +1830,9 @@ html[data-navigation-direction="back"]::view-transition-new(root) { border-bottom-color: transparent; } .library-marquee-header.is-overlay[data-scrolled="true"] { - background: color-mix(in srgb, var(--background) 92%, var(--surface)); - backdrop-filter: none; - -webkit-backdrop-filter: none; + background: color-mix(in srgb, var(--background) 72%, transparent); + backdrop-filter: blur(18px) saturate(1.15); + -webkit-backdrop-filter: blur(18px) saturate(1.15); border-bottom-color: color-mix(in srgb, var(--border) 50%, transparent); } @media (min-width: 640px) { @@ -1935,10 +1965,10 @@ html[data-navigation-direction="back"]::view-transition-new(root) { gap: 0.15rem; padding: 0.25rem; border-radius: 9999px; - background: color-mix(in srgb, #fff 8%, var(--background)); + background: color-mix(in srgb, #fff 8%, transparent); border: 1px solid color-mix(in srgb, #fff 10%, transparent); - backdrop-filter: none; - -webkit-backdrop-filter: none; + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); } .library-marquee-header:not(.is-overlay) .marquee-tab-bar { background: color-mix(in srgb, var(--foreground) 6%, transparent); @@ -2059,6 +2089,17 @@ html[data-navigation-direction="back"]::view-transition-new(root) { box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.04); } + /* One surface step above a surface-panel shell. Settings groups sit on + this so they read as defined panels inside the shell — in light themes + the shell and --surface are near-identical, so the raised fill plus the + hairline are what keep group boundaries visible. */ + .surface-panel-raised { + background: color-mix(in srgb, var(--surface-raised) 65%, transparent); + border: 1px solid color-mix(in srgb, var(--border) 72%, transparent); + border-radius: 1.7rem; + box-shadow: inset 0 1px 0 rgb(255 255 255 / 0.04); + } + /* Search controls repaint on the hottest route in the app. Keep them on the same opaque paint surface as the page: no backdrop sampling, alpha blend, or blur layer needs to be recomposited while results replace underneath. */ @@ -2185,10 +2226,83 @@ html[data-navigation-direction="back"]::view-transition-new(root) { padding-inline: 0; } - .admin-dashboard-stats { + /* Admin dashboard widget grid: single column on small screens, two columns + on tablets (where .admin-widget-wide keeps the full row), and a 12-column + grid from lg up where each widget spans var(--widget-span) columns and + var(--widget-rows) fixed-height rows. + + Rows are only fixed from lg up. Below that the grid is one or two + content-sized columns, so a widget is already as narrow as it gets and a + row span picked for the wide layout would clip it — the same reason column + spans stop applying there. */ + .admin-widget-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(min(100%, 10.5rem), 1fr)); + grid-template-columns: 1fr; gap: 0.875rem; + /* Height of one widget row. Declared here rather than in the media query so + DashboardGrid can read it back for a drag; it only takes effect at lg. */ + --admin-row-h: 6.25rem; + } + + .admin-widget { + grid-column: 1 / -1; + position: relative; + min-width: 0; + } + + /* ── Dashboard Widget Scrollbars ──────────────────────────── */ + /* Widget bodies scroll inside their row, so the default OS bar + sits right on top of the theme. One rule covers every widget: + `scrollbar-width`/`scrollbar-color` inherit, so Firefox picks + them up from the grid, and WebKit gets a descendant rule. + Matches .overlay-scroll so the dashboard and its Add-widget + sheet look the same. */ + .admin-widget-grid { + scrollbar-width: thin; + scrollbar-color: color-mix(in srgb, var(--foreground) 15%, transparent) transparent; + } + .admin-widget-grid ::-webkit-scrollbar { + width: 6px; + height: 6px; + } + .admin-widget-grid ::-webkit-scrollbar-track { + background: transparent; + } + .admin-widget-grid ::-webkit-scrollbar-thumb { + background: color-mix(in srgb, var(--foreground) 15%, transparent); + border-radius: 3px; + } + .admin-widget-grid ::-webkit-scrollbar-thumb:hover { + background: color-mix(in srgb, var(--foreground) 30%, transparent); + } + .admin-widget-grid ::-webkit-scrollbar-corner { + background: transparent; + } + + @media (min-width: 640px) and (max-width: 1023px) { + .admin-widget-grid { + grid-template-columns: repeat(2, 1fr); + } + + .admin-widget { + grid-column: span 1; + } + + .admin-widget-wide { + grid-column: 1 / -1; + } + } + + @media (min-width: 1024px) { + .admin-widget-grid { + grid-template-columns: repeat(12, 1fr); + grid-auto-rows: var(--admin-row-h); + } + + .admin-widget { + grid-column: span var(--widget-span) / span var(--widget-span); + grid-row: span var(--widget-rows) / span var(--widget-rows); + } } @media (max-width: 639px) { @@ -2316,11 +2430,11 @@ html[data-navigation-direction="back"]::view-transition-new(root) { position: sticky; background: linear-gradient( 180deg, - color-mix(in srgb, var(--background) 96%, var(--surface)), - color-mix(in srgb, var(--background) 92%, var(--surface)) + color-mix(in srgb, var(--background) 94%, transparent), + color-mix(in srgb, var(--background) 88%, transparent) ); - backdrop-filter: none; - -webkit-backdrop-filter: none; + backdrop-filter: blur(20px) saturate(1.3); + -webkit-backdrop-filter: blur(20px) saturate(1.3); border-bottom: 1px solid color-mix(in srgb, var(--impersonation-accent) 16%, var(--border)); box-shadow: inset 0 1px 0 color-mix(in srgb, var(--impersonation-accent) 22%, transparent), diff --git a/web/src/components/AdminLayout.test.tsx b/web/src/components/AdminLayout.test.tsx new file mode 100644 index 000000000..dd511189a --- /dev/null +++ b/web/src/components/AdminLayout.test.tsx @@ -0,0 +1,132 @@ +import { act, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { createMemoryRouter, RouterProvider } from "react-router"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + useAdminServerStatus: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/settings", () => ({ + useAdminServerStatus: () => mocks.useAdminServerStatus(), +})); +vi.mock("@/hooks/queries/admin/plugins", () => ({ + useAdminPluginInstallations: () => ({ data: undefined }), +})); +vi.mock("@/hooks/queries/admin/policy", () => ({ + usePolicyCapability: () => ({ data: undefined }), +})); +vi.mock("@/components/AdminSidebar", () => ({ default: () => null })); +vi.mock("@/components/AdminSectionCommandDialog", () => ({ + AdminSectionCommandDialog: () => null, +})); +vi.mock("@/components/ServerActivity", () => ({ default: () => null })); +vi.mock("@/playback/watchPlaybackContext", () => ({ + useWatchPlaybackController: () => ({ isBackgroundBarVisible: false }), +})); +vi.mock("@/pages/audiobooks/player/audiobookPlaybackContext", () => ({ + useAudiobookPlaybackController: () => null, +})); + +import AdminLayout from "./AdminLayout"; + +// The dashboard and the users page stand in for "any admin page that is not +// settings" — the shell is the only thing that renders the restart prompt, so +// both must show it. +function renderAdmin(initialPath = "/admin") { + const router = createMemoryRouter( + [ + { + path: "/admin", + element: , + children: [ + { index: true, element:

Admin dashboard

}, + { path: "users", element:

Admin users

}, + ], + }, + ], + { initialEntries: [initialPath] }, + ); + + return { router, ...render() }; +} + +beforeEach(() => { + mocks.useAdminServerStatus.mockReturnValue({ data: { restart_required: true } }); + vi.stubGlobal("matchMedia", (query: string) => ({ + matches: query === "(min-width: 64rem)", + media: query, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + })); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("AdminLayout search shortcut hint", () => { + function stubUserAgent(value: string) { + vi.spyOn(window.navigator, "userAgent", "get").mockReturnValue(value); + } + + // The dialog opens on Cmd or Ctrl, so the advertised hint has to name the key + // this keyboard actually has — a hardcoded ⌘ is a dead instruction on Windows + // and Linux, which is most self-hosters. + it("names Ctrl off Apple platforms", () => { + stubUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"); + renderAdmin(); + + const [search] = screen.getAllByRole("button", { name: "Search admin sections" }); + expect(search).toHaveAttribute("title", "Search admin sections (Ctrl K)"); + expect(screen.getByText("Ctrl K")).toBeInTheDocument(); + expect(screen.queryByText(/⌘/)).not.toBeInTheDocument(); + }); + + it("names the command glyph on Apple platforms", () => { + stubUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15"); + renderAdmin(); + + const [search] = screen.getAllByRole("button", { name: "Search admin sections" }); + expect(search).toHaveAttribute("title", "Search admin sections (⌘ K)"); + expect(screen.getByText("⌘ K")).toBeInTheDocument(); + }); +}); + +describe("AdminLayout restart banner", () => { + it("stays quiet while no restart is owed", () => { + mocks.useAdminServerStatus.mockReturnValue({ data: { restart_required: false } }); + renderAdmin(); + + expect(screen.getByRole("heading", { name: "Admin dashboard" })).toBeInTheDocument(); + expect(screen.queryByText("Restart required")).not.toBeInTheDocument(); + }); + + it("prompts on a page outside settings, above the routed page", () => { + renderAdmin("/admin/users"); + + const banner = screen.getByRole("status"); + const page = screen.getByRole("heading", { name: "Admin users" }); + + expect(banner).toHaveTextContent("Restart required"); + // Node.DOCUMENT_POSITION_FOLLOWING: the page comes after the banner. + expect(banner.compareDocumentPosition(page) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it("keeps a dismissal across admin navigation", async () => { + const { router } = renderAdmin(); + + await userEvent.click(screen.getByRole("button", { name: "Later" })); + expect(screen.queryByText("Restart required")).not.toBeInTheDocument(); + + // The shell owns the banner, so moving between admin pages neither + // resurrects the prompt nor loses the admin's "Later". + await act(async () => { + await router.navigate("/admin/users"); + }); + + expect(screen.getByRole("heading", { name: "Admin users" })).toBeInTheDocument(); + expect(screen.queryByText("Restart required")).not.toBeInTheDocument(); + }); +}); diff --git a/web/src/components/AdminLayout.tsx b/web/src/components/AdminLayout.tsx index 9c749b815..9fc56e79c 100644 --- a/web/src/components/AdminLayout.tsx +++ b/web/src/components/AdminLayout.tsx @@ -1,7 +1,9 @@ -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Outlet, useLocation } from "react-router"; import AdminSidebar from "@/components/AdminSidebar"; +import { AdminSectionCommandDialog } from "@/components/AdminSectionCommandDialog"; import ServerActivity from "@/components/ServerActivity"; +import { RestartBanner } from "@/components/admin/RestartBanner"; import { Sheet, SheetClose, @@ -11,8 +13,14 @@ import { SheetTrigger, } from "@/components/ui/sheet"; import { useDocumentTitle } from "@/hooks/useDocumentTitle"; +import { useAdminPluginInstallations } from "@/hooks/queries/admin/plugins"; +import { usePolicyCapability } from "@/hooks/queries/admin/policy"; +import { useAdminServerStatus } from "@/hooks/queries/admin/settings"; +import { buildAdminCommandNavSections } from "@/lib/adminNavigation"; import { resolveAdminDocumentTitle } from "@/lib/documentTitle"; -import { Menu, X } from "lucide-react"; +import { searchShortcutLabel } from "@/lib/keyboardShortcut"; +import { cn } from "@/lib/utils"; +import { Menu, Search, X } from "lucide-react"; import { useWatchPlaybackController } from "@/playback/watchPlaybackContext"; import { useAudiobookPlaybackController } from "@/pages/audiobooks/player/audiobookPlaybackContext"; @@ -20,7 +28,25 @@ const ADMIN_DESKTOP_MEDIA_QUERY = "(min-width: 64rem)"; export default function AdminLayout() { const [mobileOpen, setMobileOpen] = useState(false); + const [commandOpen, setCommandOpen] = useState(false); const location = useLocation(); + const { data: adminInstallations } = useAdminPluginInstallations(); + const policyCapability = usePolicyCapability(); + // The one restart prompt for the admin area. Read here, not in a page: a + // restart is owed by the server, not by the page that happened to ask for + // it, so the prompt has to survive navigating away from settings. The query + // is shared and cached, so the settings overview reading the same status + // costs nothing extra. + const { data: serverStatus } = useAdminServerStatus(); + // Mounted here rather than on the dashboard so Cmd+K reaches every admin + // page, which is what the pages that advertise the shortcut assume. + const adminSearchSections = useMemo( + () => + buildAdminCommandNavSections(adminInstallations, { + policyEditorAvailable: policyCapability.data?.editor_available === true, + }), + [adminInstallations, policyCapability.data?.editor_available], + ); const { isBackgroundBarVisible } = useWatchPlaybackController(); const audiobookPlayback = useAudiobookPlaybackController(); const hasBackgroundBar = isBackgroundBarVisible || audiobookPlayback?.isBackgroundBarVisible; @@ -44,6 +70,11 @@ export default function AdminLayout() { return (
+
- +
+ setCommandOpen(true)} className="h-11 w-11" /> + +
{/* Mobile sidebar drawer */} @@ -113,8 +147,9 @@ export default function AdminLayout() { - {/* Desktop activity indicator */} -
+ {/* Desktop header controls */} +
+ setCommandOpen(true)} showShortcut />
@@ -126,9 +161,56 @@ export default function AdminLayout() { }`} >
+ {/* Above the routed page and inside the content column, so every + admin page carries the prompt and none of them can be covered by + it. `lg:mt-7` clears the fixed Search/activity controls in the + top-right corner (top-5, h-9 → they end 3.5rem down), which would + otherwise float over the banner's buttons. */} +
); } + +function AdminSearchButton({ + onClick, + className, + showShortcut = false, +}: { + onClick: () => void; + className?: string; + showShortcut?: boolean; +}) { + // Advertised, not hardcoded: the dialog opens on either modifier, so the hint + // has to name the one this keyboard actually has. + const shortcut = searchShortcutLabel(); + + return ( + + ); +} diff --git a/web/src/components/AdminSectionCommandDialog.test.tsx b/web/src/components/AdminSectionCommandDialog.test.tsx index e18cc6735..47cbb3e8f 100644 --- a/web/src/components/AdminSectionCommandDialog.test.tsx +++ b/web/src/components/AdminSectionCommandDialog.test.tsx @@ -66,18 +66,20 @@ describe("AdminSectionCommandDialog", () => { expect(screen.queryByRole("option", { name: /Settings/ })).not.toBeInTheDocument(); }); - it("searches individual admin setting labels from the dashboard dialog", async () => { + it("searches individual admin setting labels from the admin dialog", async () => { renderDialog(); const searchBox = await openDialog(); - await userEvent.type(searchBox, "pool max open"); + await userEvent.type(searchBox, "maximum postgres connections"); - expect(screen.getByRole("option", { name: /Database/ })).toBeInTheDocument(); - expect(screen.getByText("Pool Max Open")).toBeInTheDocument(); + expect(screen.getByRole("option", { name: /Storage & Database/ })).toBeInTheDocument(); + expect(screen.getByText("Maximum Postgres connections")).toBeInTheDocument(); - await userEvent.click(screen.getByRole("option", { name: /Database/ })); + await userEvent.click(screen.getByRole("option", { name: /Storage & Database/ })); - expect(screen.getByLabelText("Current path")).toHaveTextContent("/admin/settings?tab=database"); + expect(screen.getByLabelText("Current path")).toHaveTextContent( + "/admin/settings/infrastructure", + ); }); it("includes admin plugin app destinations", async () => { @@ -115,7 +117,7 @@ describe("AdminSectionCommandDialog", () => { const searchBox = await openDialog(); await userEvent.type(searchBox, "logs"); - await userEvent.click(screen.getByRole("option", { name: /Logs/ })); + await userEvent.click(screen.getByRole("option", { name: /^LogsServer log stream/ })); expect(screen.getByLabelText("Current path")).toHaveTextContent("/admin/logs"); expect(screen.queryByRole("searchbox", { name: "Search admin sections" })).toBeNull(); diff --git a/web/src/components/AdminSectionCommandDialog.tsx b/web/src/components/AdminSectionCommandDialog.tsx index a1cd5d8ad..95b8787f9 100644 --- a/web/src/components/AdminSectionCommandDialog.tsx +++ b/web/src/components/AdminSectionCommandDialog.tsx @@ -15,10 +15,26 @@ import { cn } from "@/lib/utils"; interface AdminSectionCommandDialogProps { sections: readonly AdminNavGroup[]; + /** Controlled open state, so a visible search button can open the palette. */ + open?: boolean; + onOpenChange?: (open: boolean) => void; } -export function AdminSectionCommandDialog({ sections }: AdminSectionCommandDialogProps) { - const [open, setOpen] = useState(false); +export function AdminSectionCommandDialog({ + sections, + open: openProp, + onOpenChange, +}: AdminSectionCommandDialogProps) { + const [uncontrolledOpen, setUncontrolledOpen] = useState(false); + const open = openProp ?? uncontrolledOpen; + const isControlled = openProp !== undefined; + const setOpen = useCallback( + (next: boolean) => { + if (!isControlled) setUncontrolledOpen(next); + onOpenChange?.(next); + }, + [isControlled, onOpenChange], + ); const [query, setQuery] = useState(""); const [selectedIndex, setSelectedIndex] = useState(0); const inputRef = useRef(null); @@ -56,12 +72,12 @@ export function AdminSectionCommandDialog({ sections }: AdminSectionCommandDialo setOpen(false); setQuery(""); setSelectedIndex(0); - }, []); + }, [setOpen]); const openDialog = useCallback(() => { setOpen(true); focusSearch(); - }, [focusSearch]); + }, [focusSearch, setOpen]); useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { diff --git a/web/src/components/AdminSidebar.test.tsx b/web/src/components/AdminSidebar.test.tsx index d69e6f91d..60fb1a002 100644 --- a/web/src/components/AdminSidebar.test.tsx +++ b/web/src/components/AdminSidebar.test.tsx @@ -82,11 +82,19 @@ describe("AdminSidebar", () => { it("renders the grouped navigation sections", () => { const markup = renderSidebar(); - for (const section of ["Overview", "Content", "Automation", "Users", "System"]) { + for (const section of ["Overview", "Content", "Automation", "Users", "Settings", "System"]) { expect(markup).toContain(`>${section}<`); } }); + it("keeps settings as one sidebar destination", () => { + const markup = renderSidebar(); + const settingsLinks = markup.match(/href="\/admin\/settings[^"]*"/g) ?? []; + + expect(settingsLinks).toEqual(['href="/admin/settings"']); + expect(markup).not.toContain("/admin/settings?tab="); + }); + it("renders as an embedded rail inside the mobile drawer", () => { const markup = renderSidebar(true); diff --git a/web/src/components/AdminSidebar.tsx b/web/src/components/AdminSidebar.tsx index ff9450a89..05de00306 100644 --- a/web/src/components/AdminSidebar.tsx +++ b/web/src/components/AdminSidebar.tsx @@ -3,6 +3,7 @@ import { ArrowLeft } from "lucide-react"; import type { ReactNode } from "react"; import { SideNavItem, SideNavSection } from "@/components/SideNav"; import { SiloBrand } from "@/components/SiloBrand"; +import ViewTransitionLink from "@/components/ViewTransitionLink"; import { buildAdminNavSections, buildAdminPluginNavItems, @@ -154,15 +155,17 @@ export default function AdminSidebar({ onNavigate, embedded = false }: AdminSide {buildDisplay} - {/* Back to app */} - Back to App - + ); diff --git a/web/src/components/AppSidebar.logic.ts b/web/src/components/AppSidebar.logic.ts index a14568b62..db9b071bd 100644 --- a/web/src/components/AppSidebar.logic.ts +++ b/web/src/components/AppSidebar.logic.ts @@ -20,7 +20,7 @@ export const SIDEBAR_SURFACE_WIDTH = 260; * The `