From 826c71f6261cb793a97f377b0780e9c3c0f9be91 Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:17:27 +0000 Subject: [PATCH 01/44] feat(playback): carry an immutable session creation time in stream tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session age is the ordering signal every later enforcement rule depends on ("cut the newest stream first"), but nothing carried a stable creation time. `Sign` overwrites `RegisteredClaims` wholesale on every mint, so `iat` is issue time, not session-start time, and a replan mints a replacement token that resets it. Reconstruction after a restart never set `StartedAt` at all, so `RegisterReconstructed` stamped `time.Now()`, and the proxy re-stamped `time.Now()` on every HLS touch. Session age therefore reset on every restart, reconnect and segment request. Adds an explicit `ostn` claim carrying the creation time in Unix nanoseconds. Nanoseconds rather than seconds because victim ordering is defined as (startedAtUnixNano, sessionID): at second precision, sessions started in the same second would fall back to sorting by random UUID. `int64` decodes exactly through golang-jwt's struct unmarshal. Resolution is centralised in `Claims.StartedAt`, which returns an explicit source rather than a bool: `Sign` always writes `iat`, so a legacy token always resolves *something*, and the caller must be able to tell an authoritative value from a degraded one. `iat` is treated as degraded because it is not stable across re-mints. A missing claim never invalidates an otherwise valid token. `RecipeCard` carries the value as a `time.Time` (RFC3339Nano, full precision), `ReconstructSession` seeds `Session.StartedAt` from it, and the proxy's node session record reports it. `SessionInfo.StartedAt` keeps its existing RFC3339 encoding — the `/api/v1` additive-only rule forbids re-encoding an existing field — so precision and provenance are exposed additively as `started_at_unix_nano` and `started_at_source`. Part of the stream telemetry and enforcement effort (P0a). AI-use disclosure: implemented with AI assistance (Claude planning and review, Codex gpt-5.6-sol implementing), verified against the repo's own build, vet, lint and test gates. --- internal/api/handlers/playback.go | 8 +-- internal/api/handlers/playback_v3.go | 2 + internal/api/handlers/playback_v3_test.go | 45 +++++++++++++++++ internal/nodesessions/tracker.go | 26 +++++----- internal/playback/recipecard.go | 53 ++++++++++++-------- internal/playback/recipecard_test.go | 34 +++++++++++++ internal/playback/transcode_manager.go | 1 + internal/proxy/server.go | 22 ++++++--- internal/proxy/session_info_test.go | 39 +++++++++++++++ internal/streamtoken/token.go | 28 +++++++++++ internal/streamtoken/token_test.go | 60 +++++++++++++++++++++++ 11 files changed, 274 insertions(+), 44 deletions(-) create mode 100644 internal/proxy/session_info_test.go create mode 100644 internal/streamtoken/token_test.go diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 5e7ac3cbc..cec144f93 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -506,16 +506,18 @@ func (h *PlaybackHandler) playbackStreamURL(s *playback.Session) string { // the bytes are served by HTTP Range / a re-spawned remux pipe at the // client-supplied position. func identityRecipeCard(s *playback.Session) playback.RecipeCard { + var card playback.RecipeCard switch s.PlayMethod { case playback.PlayRemux: - card := playback.NewRemuxRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID, s.TranscodeAudio, s.AudioTrackIndex, s.RemuxDVMode) + card = playback.NewRemuxRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID, s.TranscodeAudio, s.AudioTrackIndex, s.RemuxDVMode) card.TargetCodecAudio = s.TargetAudioCodec card.TargetAudioChannels = s.TargetAudioChannels card.TargetAudioBitrateKbps = s.TargetAudioBitrateKbps - return card default: - return playback.NewDirectRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID) + card = playback.NewDirectRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID) } + card.OriginalStartedAt = s.StartedAt + return card } func fileBitrateKbps(file *models.MediaFile) int { diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 80ff90f0a..c66d5a300 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -1256,6 +1256,7 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play } } card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) + card.OriginalStartedAt = session.StartedAt url := appendStreamToken(fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID), h.signSessionToken(card)) committed := false previousNodeURL := session.TranscodeNodeURL @@ -1326,6 +1327,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla } hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) + card.OriginalStartedAt = session.StartedAt url := h.buildProxyManifestURL(card, nodePlan.ProxyNode) // buildProxyManifestURL only returns an absolute proxy URL when a proxy was // planned and the token could be signed; otherwise the client fetches the diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 525c44164..ba2529446 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -708,6 +708,7 @@ func TestHandleReplanPlaybackV3UpdatesSelectedAudioAndReplaysIdempotently(t *tes file.AudioTracks = append(file.AudioTracks, models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", Language: "spa"}) manager := playback.NewSessionManager(0, 0) handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-secret" stubCopySeekAnchorV3(handler) handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} handler.ItemAccess = allowAllPlaybackItemAccess{} @@ -731,6 +732,17 @@ func TestHandleReplanPlaybackV3UpdatesSelectedAudioAndReplaysIdempotently(t *tes if started.PlaybackPlan == nil { t.Fatal("start returned no plan") } + originalSession, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + originalClaims := streamClaimsFromPlanURL(t, started.PlaybackPlan.Stream.URL, handler.JWTSecret) + if originalClaims.OriginalStartedAtUnixNano != originalSession.StartedAt.UnixNano() { + t.Fatalf("start ostn = %d, want %d", originalClaims.OriginalStartedAtUnixNano, originalSession.StartedAt.UnixNano()) + } + // Cross the JWT NumericDate second boundary so the replan proves ostn stays + // immutable even though Sign rewrites iat on the replacement token. + time.Sleep(time.Until(time.Unix(time.Now().Unix()+1, 0)) + 10*time.Millisecond) audioIndex := 1 bandwidthEstimate := 3_500 bandwidthCap := 4_000 @@ -760,6 +772,13 @@ func TestHandleReplanPlaybackV3UpdatesSelectedAudioAndReplaysIdempotently(t *tes if first.PlaybackPlan == nil || second.PlaybackPlan == nil || first.PlaybackPlan.PlanID != second.PlaybackPlan.PlanID { t.Fatalf("first=%#v second=%#v", first, second) } + replanClaims := streamClaimsFromPlanURL(t, first.PlaybackPlan.Stream.URL, handler.JWTSecret) + if replanClaims.OriginalStartedAtUnixNano != originalSession.StartedAt.UnixNano() { + t.Fatalf("replan ostn = %d, want original %d", replanClaims.OriginalStartedAtUnixNano, originalSession.StartedAt.UnixNano()) + } + if originalClaims.IssuedAt == nil || replanClaims.IssuedAt == nil || originalClaims.IssuedAt.Equal(replanClaims.IssuedAt.Time) { + t.Fatalf("iat did not move across replan: original=%v replan=%v", originalClaims.IssuedAt, replanClaims.IssuedAt) + } session, err := manager.GetSession(started.SessionID) if err != nil { t.Fatal(err) @@ -802,6 +821,32 @@ func TestHandleReplanPlaybackV3UpdatesSelectedAudioAndReplaysIdempotently(t *tes } } +func streamClaimsFromPlanURL(t *testing.T, rawURL, secret string) *streamtoken.Claims { + t.Helper() + u, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + token := u.Query().Get(streamTokenParam) + if token == "" { + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + for i, part := range parts { + if part == "transcode" && i+1 < len(parts) { + token = parts[i+1] + break + } + } + } + if token == "" { + t.Fatalf("plan URL has no stream token: %q", rawURL) + } + claims, err := streamtoken.Verify(token, secret) + if err != nil { + t.Fatal(err) + } + return claims +} + func TestHandleReplanPlaybackV3FailureDoesNotReplaceDurableStartDecision(t *testing.T) { file := v3HandlerFixtureFile(t) manager := playback.NewSessionManager(0, 0) diff --git a/internal/nodesessions/tracker.go b/internal/nodesessions/tracker.go index acdc709c6..c5ff7f6ee 100644 --- a/internal/nodesessions/tracker.go +++ b/internal/nodesessions/tracker.go @@ -20,18 +20,20 @@ const ( // SessionInfo represents an active streaming session stored in Redis. type SessionInfo struct { - SessionID string `json:"session_id"` - NodeURL string `json:"node_url"` - NodeName string `json:"node_name"` - UserID string `json:"user_id,omitempty"` - MediaItemID string `json:"media_item_id,omitempty"` - MediaTitle string `json:"media_title,omitempty"` - Type string `json:"type"` // "direct_play", "remux", "transcode", "download_prepare", "download" - CodecVideo string `json:"codec_video,omitempty"` - CodecAudio string `json:"codec_audio,omitempty"` - Resolution string `json:"resolution,omitempty"` - HWAccel string `json:"hw_accel,omitempty"` - StartedAt string `json:"started_at"` + SessionID string `json:"session_id"` + NodeURL string `json:"node_url"` + NodeName string `json:"node_name"` + UserID string `json:"user_id,omitempty"` + MediaItemID string `json:"media_item_id,omitempty"` + MediaTitle string `json:"media_title,omitempty"` + Type string `json:"type"` // "direct_play", "remux", "transcode", "download_prepare", "download" + CodecVideo string `json:"codec_video,omitempty"` + CodecAudio string `json:"codec_audio,omitempty"` + Resolution string `json:"resolution,omitempty"` + HWAccel string `json:"hw_accel,omitempty"` + StartedAt string `json:"started_at"` + StartedAtUnixNano int64 `json:"started_at_unix_nano,omitempty"` + StartedAtSource string `json:"started_at_source,omitempty"` // AuthUserID / ProfileID / MediaFileID are the numeric ownership keys the // node copies from the verified stream token. They enrich the live admin diff --git a/internal/playback/recipecard.go b/internal/playback/recipecard.go index b6a576aae..527ee274c 100644 --- a/internal/playback/recipecard.go +++ b/internal/playback/recipecard.go @@ -16,12 +16,13 @@ import ( // context, channels, log sink). Those are re-wired on reconstruct from the // live config and request. type RecipeCard struct { - SessionID string `json:"session_id"` - UserID int `json:"user_id"` - ProfileID string `json:"profile_id"` - MediaFileID int `json:"media_file_id"` - TranscodeNodeURL string `json:"transcode_node_url,omitempty"` - TranscodeTransportID string `json:"transcode_transport_id,omitempty"` + SessionID string `json:"session_id"` + UserID int `json:"user_id"` + ProfileID string `json:"profile_id"` + MediaFileID int `json:"media_file_id"` + TranscodeNodeURL string `json:"transcode_node_url,omitempty"` + TranscodeTransportID string `json:"transcode_transport_id,omitempty"` + OriginalStartedAt time.Time `json:"original_started_at,omitempty"` // PlayMethod discriminates which serve path reconstructs this session // (direct / remux / transcode). Empty decodes as PlayTranscode for @@ -209,20 +210,26 @@ const MaxTokenTTL = 24 * time.Hour // change applies to reconstructed sessions too. func (c RecipeCard) ToClaims() streamtoken.Claims { return streamtoken.Claims{ - SessionID: c.SessionID, - MediaPath: c.InputPath, - OutputSubdir: c.OutputSubdir, - PlayMethod: string(c.PlayMethod), - TranscodeAudio: c.TranscodeAudio, - RemuxDVMode: string(c.RemuxDVMode), - TranscodeNode: c.TranscodeNodeURL, - TranscodeTransportID: c.TranscodeTransportID, - TargetCodec: c.TargetCodecVideo, - TargetRes: c.TargetResolution, - AudioTrackIndex: c.AudioTrackIndex, - UserID: c.UserID, - ProfileID: c.ProfileID, - MediaFileID: c.MediaFileID, + SessionID: c.SessionID, + MediaPath: c.InputPath, + OutputSubdir: c.OutputSubdir, + PlayMethod: string(c.PlayMethod), + TranscodeAudio: c.TranscodeAudio, + RemuxDVMode: string(c.RemuxDVMode), + TranscodeNode: c.TranscodeNodeURL, + TranscodeTransportID: c.TranscodeTransportID, + TargetCodec: c.TargetCodecVideo, + TargetRes: c.TargetResolution, + AudioTrackIndex: c.AudioTrackIndex, + UserID: c.UserID, + ProfileID: c.ProfileID, + MediaFileID: c.MediaFileID, + OriginalStartedAtUnixNano: func() int64 { + if c.OriginalStartedAt.IsZero() { + return 0 + } + return c.OriginalStartedAt.UnixNano() + }(), SourceVideoCodec: c.SourceVideoCodec, SourceVideoProfile: c.SourceVideoProfile, SourceVideoBitDepth: c.SourceVideoBitDepth, @@ -257,7 +264,7 @@ func RecipeCardFromClaims(c *streamtoken.Claims) RecipeCard { if method == "" { method = PlayTranscode } - return RecipeCard{ + card := RecipeCard{ SessionID: c.SessionID, UserID: c.UserID, ProfileID: c.ProfileID, @@ -292,4 +299,8 @@ func RecipeCardFromClaims(c *streamtoken.Claims) RecipeCard { TotalDuration: c.TotalDuration, FastStart: c.FastStart, } + if c.OriginalStartedAtUnixNano != 0 { + card.OriginalStartedAt = time.Unix(0, c.OriginalStartedAtUnixNano).UTC() + } + return card } diff --git a/internal/playback/recipecard_test.go b/internal/playback/recipecard_test.go index 2628c6b6b..ab9f09240 100644 --- a/internal/playback/recipecard_test.go +++ b/internal/playback/recipecard_test.go @@ -3,6 +3,7 @@ package playback import ( "encoding/json" "testing" + "time" "github.com/Silo-Server/silo-server/internal/streamtoken" ) @@ -83,6 +84,39 @@ func TestRecipeCardRoundTripOpts(t *testing.T) { } } +func TestRecipeCardOriginalStartedAtRoundTripAndReconstruct(t *testing.T) { + started := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + card := NewRecipeCard(42, "profile-1", 77, "", TranscodeOpts{SessionID: "started", InputPath: "/media/movie.mkv"}) + card.OriginalStartedAt = started + encoded, err := json.Marshal(card) + if err != nil { + t.Fatal(err) + } + var stored RecipeCard + if err := json.Unmarshal(encoded, &stored); err != nil { + t.Fatal(err) + } + if !stored.OriginalStartedAt.Equal(started) { + t.Fatalf("stored-card round trip = %s, want %s", stored.OriginalStartedAt, started) + } + + claims := card.ToClaims() + if claims.OriginalStartedAtUnixNano != started.UnixNano() { + t.Fatalf("ostn = %d, want %d", claims.OriginalStartedAtUnixNano, started.UnixNano()) + } + back := RecipeCardFromClaims(&claims) + if !back.OriginalStartedAt.Equal(started) { + t.Fatalf("claim round trip = %s, want %s", back.OriginalStartedAt, started) + } + + tm := NewTranscodeManager() + tm.Sessions = NewSessionManager(0, 0) + session := tm.ReconstructSession(t.Context(), "started", 42, back) + if session == nil || !session.StartedAt.Equal(started) { + t.Fatalf("reconstructed StartedAt = %v, want %s", session, started) + } +} + func TestRecipeCardPlayMethodConstructors(t *testing.T) { if c := NewRecipeCard(1, "p", 2, "", TranscodeOpts{SessionID: "t"}); c.PlayMethod != PlayTranscode { t.Errorf("transcode card PlayMethod = %q, want transcode", c.PlayMethod) diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go index 19ace5069..afb1df55e 100644 --- a/internal/playback/transcode_manager.go +++ b/internal/playback/transcode_manager.go @@ -410,6 +410,7 @@ func (m *TranscodeManager) ReconstructSession(ctx context.Context, sessionID str UserID: card.UserID, ProfileID: card.ProfileID, MediaFileID: card.MediaFileID, + StartedAt: card.OriginalStartedAt, PlayMethod: method, BasePlayMethod: method, TranscodeNodeURL: card.TranscodeNodeURL, diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 32146eefc..22b5692c9 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -415,15 +415,21 @@ func (s *Server) touchTranscodeSession(r *http.Request, claims *streamtoken.Clai // sessionInfo builds the node-session tracker record for a verified token, // copying the numeric ownership keys the node-session tracker needs. func sessionInfo(tr *nodesessions.Tracker, claims *streamtoken.Claims, kind string) nodesessions.SessionInfo { + startedAt, source := claims.StartedAt() + if source == streamtoken.StartedAtSourceNone { + startedAt = time.Now().UTC() + } return nodesessions.SessionInfo{ - SessionID: claims.SessionID, - NodeURL: tr.NodeURL(), - NodeName: tr.NodeName(), - Type: kind, - StartedAt: time.Now().UTC().Format(time.RFC3339), - AuthUserID: claims.UserID, - ProfileID: claims.ProfileID, - MediaFileID: claims.MediaFileID, + SessionID: claims.SessionID, + NodeURL: tr.NodeURL(), + NodeName: tr.NodeName(), + Type: kind, + StartedAt: startedAt.Format(time.RFC3339), + StartedAtUnixNano: startedAt.UnixNano(), + StartedAtSource: string(source), + AuthUserID: claims.UserID, + ProfileID: claims.ProfileID, + MediaFileID: claims.MediaFileID, } } diff --git a/internal/proxy/session_info_test.go b/internal/proxy/session_info_test.go new file mode 100644 index 000000000..b55229e78 --- /dev/null +++ b/internal/proxy/session_info_test.go @@ -0,0 +1,39 @@ +package proxy + +import ( + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func TestSessionInfoPreservesStartedAtAcrossTouches(t *testing.T) { + started := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + tracker := nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy") + claims := &streamtoken.Claims{SessionID: "s", UserID: 42, ProfileID: "p", MediaFileID: 77, OriginalStartedAtUnixNano: started.UnixNano()} + + first := sessionInfo(tracker, claims, "transcode") + time.Sleep(time.Millisecond) + second := sessionInfo(tracker, claims, "transcode") + if first.StartedAtUnixNano != started.UnixNano() || second.StartedAtUnixNano != first.StartedAtUnixNano { + t.Fatalf("touch reset StartedAtUnixNano: first=%d second=%d want=%d", first.StartedAtUnixNano, second.StartedAtUnixNano, started.UnixNano()) + } + if first.StartedAtSource != string(streamtoken.StartedAtSourceClaim) { + t.Fatalf("StartedAtSource = %q, want claim", first.StartedAtSource) + } +} + +func TestSessionInfoLegacyUsesIssuedAt(t *testing.T) { + issued := time.Date(2026, 8, 16, 12, 34, 56, 0, time.UTC) + tracker := nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy") + claims := &streamtoken.Claims{SessionID: "legacy"} + claims.IssuedAt = jwt.NewNumericDate(issued) + + info := sessionInfo(tracker, claims, "direct_play") + if info.StartedAtUnixNano != issued.UnixNano() || info.StartedAtSource != string(streamtoken.StartedAtSourceIssuedAt) { + t.Fatalf("legacy session info = %+v", info) + } +} diff --git a/internal/streamtoken/token.go b/internal/streamtoken/token.go index 36be91409..485de0d49 100644 --- a/internal/streamtoken/token.go +++ b/internal/streamtoken/token.go @@ -49,6 +49,10 @@ type Claims struct { UserID int `json:"uid,omitempty"` ProfileID string `json:"pid,omitempty"` MediaFileID int `json:"mfid,omitempty"` + // OriginalStartedAtUnixNano is decoded directly into int64 by golang-jwt, + // preserving nanosecond precision. A future map[string]any decode path must + // not pass this through float64, which cannot represent this magnitude exactly. + OriginalStartedAtUnixNano int64 `json:"ostn,omitempty"` // DownloadArtifactID is an opaque transcode-node artifact handle. For // download tokens TranscodeNode is its authenticated origin; MediaPath stays // empty so node-local filesystem paths never leave the owning node. @@ -91,6 +95,30 @@ type Claims struct { jwt.RegisteredClaims } +type StartedAtSource string + +const ( + StartedAtSourceClaim StartedAtSource = "claim" + StartedAtSourceIssuedAt StartedAtSource = "issued_at" + StartedAtSourceNone StartedAtSource = "none" +) + +// StartedAt resolves the session's creation time from the explicit claim first, +// then the registered issue time. Only the explicit claim is authoritative: +// Sign rewrites RegisteredClaims on every mint, so iat is issue time. +func (c *Claims) StartedAt() (time.Time, StartedAtSource) { + if c == nil { + return time.Time{}, StartedAtSourceNone + } + if c.OriginalStartedAtUnixNano != 0 { + return time.Unix(0, c.OriginalStartedAtUnixNano).UTC(), StartedAtSourceClaim + } + if c.IssuedAt != nil { + return c.IssuedAt.UTC(), StartedAtSourceIssuedAt + } + return time.Time{}, StartedAtSourceNone +} + // Sign creates a signed JWT string from the given claims. func Sign(c Claims, secret string, ttl time.Duration) (string, error) { now := time.Now() diff --git a/internal/streamtoken/token_test.go b/internal/streamtoken/token_test.go new file mode 100644 index 000000000..277589aea --- /dev/null +++ b/internal/streamtoken/token_test.go @@ -0,0 +1,60 @@ +package streamtoken + +import ( + "testing" + "time" +) + +func TestStartedAtRoundTrip(t *testing.T) { + started := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + token, err := Sign(Claims{SessionID: "s", OriginalStartedAtUnixNano: started.UnixNano()}, "secret", time.Hour) + if err != nil { + t.Fatal(err) + } + claims, err := Verify(token, "secret") + if err != nil { + t.Fatal(err) + } + got, source := claims.StartedAt() + if source != StartedAtSourceClaim || !got.Equal(started) || claims.OriginalStartedAtUnixNano != started.UnixNano() { + t.Fatalf("StartedAt = (%s, %q), claim=%d; want (%s, %q), claim=%d", got, source, claims.OriginalStartedAtUnixNano, started, StartedAtSourceClaim, started.UnixNano()) + } +} + +func TestStartedAtLegacyAndAbsent(t *testing.T) { + token, err := Sign(Claims{SessionID: "legacy"}, "secret", time.Hour) + if err != nil { + t.Fatal(err) + } + claims, err := Verify(token, "secret") + if err != nil { + t.Fatal(err) + } + if got, source := claims.StartedAt(); got.IsZero() || source != StartedAtSourceIssuedAt { + t.Fatalf("legacy StartedAt = (%s, %q), want non-zero issued_at", got, source) + } + if got, source := (&Claims{}).StartedAt(); !got.IsZero() || source != StartedAtSourceNone { + t.Fatalf("empty StartedAt = (%s, %q), want zero none", got, source) + } +} + +func TestStartedAtSameSecondPreservesActualOrder(t *testing.T) { + older := time.Date(2026, 8, 16, 12, 0, 0, 100, time.UTC) + newer := older.Add(200 * time.Nanosecond) + resolve := func(id string, started time.Time) time.Time { + t.Helper() + token, err := Sign(Claims{SessionID: id, OriginalStartedAtUnixNano: started.UnixNano()}, "secret", time.Hour) + if err != nil { + t.Fatal(err) + } + claims, err := Verify(token, "secret") + if err != nil { + t.Fatal(err) + } + got, _ := claims.StartedAt() + return got + } + if !resolve("z-session", older).Before(resolve("a-session", newer)) { + t.Fatal("same-second token round trip lost nanosecond ordering") + } +} From d5d52c4e264e4d9d82b8c24f17d56352e6bff707 Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:17:45 +0000 Subject: [PATCH 02/44] fix(jellycompat): attribute proxied stream tokens to their owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Jellyfin-client stream served through a proxy node was attributed to nobody. In the admin "active streams" view those sessions showed a node, a type and a byte count, but no user, no profile and no media file — so an operator could see that something was streaming without being able to see who was watching what. `buildProxyRedirectURL` signed the stream token with the session id, media path, play method and the audio/DV fields, but never set `UserID`, `ProfileID` or `MediaFileID`, even though the claims struct carries all three and the compat session knows them. The proxy copies exactly those three claims into its node-session record, so the gap surfaced directly in the admin view. Populates the three ownership claims from the compat session (`StreamAppUserID`, `ProfileID`) and the negotiated source (`FileID`), and passes the play session's creation time so proxied sessions also carry the immutable start time added in the previous commit. For compat, the top-level `PlaybackSession.CreatedAt` is the source of truth and is overlaid onto the recipe card at every reconstruction and persistence point: the durable compat store unmarshals and rewrites the whole JSON document, so an older replica in a mixed-version deploy silently drops unknown *nested* fields, and a timestamp living only inside the nested recipe would be erased. Wire-safe: `Verify` decodes into a struct and does not require an exact claim set, so an older proxy binary ignores claims it does not model. Two accepted costs: the token grows, and claims are signed but not encrypted, so internal user/profile/file ids become readable to anyone already holding the (already sensitive) stream URL. Adds a claim-growth budget test, which the repo previously lacked, plus a mixed-version reconstruction test covering the nested-field-dropped case. Part of the stream telemetry and enforcement effort (P0a). AI-use disclosure: implemented with AI assistance (Claude planning and review, Codex gpt-5.6-sol implementing), verified against the repo's own build, vet, lint and test gates. --- internal/jellycompat/handlers_playback.go | 17 +++ internal/jellycompat/streams.go | 17 +-- internal/jellycompat/streams_test.go | 122 ++++++++++++++++++++++ 3 files changed, 150 insertions(+), 6 deletions(-) diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 5ce389347..38e304ce9 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -351,6 +351,8 @@ func (h *PlaybackHandler) buildProxyRedirectURL( method string, file *models.MediaFile, source PlaybackMediaSource, + compatSession *Session, + createdAt time.Time, transcodeNodeURL string, seekSeconds float64, proxyNode *nodepool.Node, @@ -374,6 +376,14 @@ func (h *PlaybackHandler) buildProxyRedirectURL( TranscodeNode: transcodeNodeURL, DVProfile: file.PrimaryDVProfile(), } + if compatSession != nil { + claims.UserID = compatSession.StreamAppUserID + claims.ProfileID = compatSession.ProfileID + claims.MediaFileID = source.FileID + } + if !createdAt.IsZero() { + claims.OriginalStartedAtUnixNano = createdAt.UnixNano() + } token, err := streamtoken.Sign(claims, h.JWTSecret, 24*time.Hour) if err != nil { return "", err @@ -545,6 +555,10 @@ func (h *PlaybackHandler) persistTranscodeRecipe( playSessionID, upstreamSessionID string, opts playback.TranscodeOpts, ) error { + var playSession *PlaybackSession + if h.playbackStore != nil { + playSession, _ = h.playbackStore.Get(playSessionID) + } var recipe *playback.RecipeCard if h.sessionMgr != nil { if upstream, err := h.sessionMgr.GetSession(upstreamSessionID); err == nil && upstream != nil { @@ -556,6 +570,9 @@ func (h *PlaybackHandler) persistTranscodeRecipe( card.ClientVersion = upstream.ClientVersion card.ClientUserAgent = upstream.ClientUserAgent card.IsJellyfinCompat = upstream.IsJellyfinCompat + if playSession != nil { + card.OriginalStartedAt = playSession.CreatedAt + } recipe = &card } } diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index e6ffd3633..dd03d7834 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -109,7 +109,7 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque } if h.NodePlanner != nil && h.JWTSecret != "" { plan := h.NodePlanner.PlanSession(playSession.UpstreamSessionID, "", false, source.Version.Bitrate) - if redirectURL, redirectErr := h.buildProxyRedirectURL(playSession.ID, playSession.UpstreamSessionID, method, file, *source, "", seekSeconds, plan.ProxyNode); redirectErr == nil { + if redirectURL, redirectErr := h.buildProxyRedirectURL(playSession.ID, playSession.UpstreamSessionID, method, file, *source, session, playSession.CreatedAt, "", seekSeconds, plan.ProxyNode); redirectErr == nil { http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) return } @@ -259,7 +259,7 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re writeError(w, http.StatusBadGateway, "TranscodeStartFailed", "Transcode node rejected the request") return } - redirectURL, redirectErr := h.buildProxyRedirectURL(playSession.ID, playSession.UpstreamSessionID, string(playback.PlayTranscode), file, *source, tcNode.URL, 0, plan.ProxyNode) + redirectURL, redirectErr := h.buildProxyRedirectURL(playSession.ID, playSession.UpstreamSessionID, string(playback.PlayTranscode), file, *source, session, playSession.CreatedAt, tcNode.URL, 0, plan.ProxyNode) if redirectErr != nil { failRemoteStart() writeError(w, http.StatusInternalServerError, "ServerError", "Failed to sign proxy stream URL") @@ -1354,13 +1354,18 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re // (PlaybackSession.Recipe); direct/remux need only identity, rebuilt here from // the compat session and the negotiated source. func (h *PlaybackHandler) upstreamRecipeCard(ps *PlaybackSession, cs *Session, source PlaybackMediaSource, method string) playback.RecipeCard { + var card playback.RecipeCard if ps != nil && ps.Recipe != nil { - return *ps.Recipe + card = *ps.Recipe + } else if method == "remux" { + card = playback.NewRemuxRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID, source.TranscodeAudio, compatAudioTrackIndexOrDefault(source)) + } else { + card = playback.NewDirectRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID) } - if method == "remux" { - return playback.NewRemuxRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID, source.TranscodeAudio, compatAudioTrackIndexOrDefault(source)) + if ps != nil && !ps.CreatedAt.IsZero() { + card.OriginalStartedAt = ps.CreatedAt } - return playback.NewDirectRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID) + return card } // reportMatchesPlaySession rejects an alias-resolved session whose item or diff --git a/internal/jellycompat/streams_test.go b/internal/jellycompat/streams_test.go index 5fa552fda..f00f9bdb8 100644 --- a/internal/jellycompat/streams_test.go +++ b/internal/jellycompat/streams_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -118,6 +119,8 @@ func TestBuildProxyRedirectURLRequestsSourceAlignedCompatManifest(t *testing.T) string(playback.PlayTranscode), &models.MediaFile{FilePath: "/media/movie.mkv"}, PlaybackMediaSource{}, + nil, + time.Time{}, "http://transcode-1", 0, &nodepool.Node{URL: "http://proxy-1"}, @@ -138,6 +141,8 @@ func TestBuildProxyRedirectURLCarriesAudioOnlyRemuxClaim(t *testing.T) { string(playback.PlayRemux), &models.MediaFile{FilePath: "/media/book.m4b", BaseType: "audiobook", CodecAudio: "aac"}, PlaybackMediaSource{}, + nil, + time.Time{}, "", 0, &nodepool.Node{URL: "http://proxy-1"}, @@ -155,6 +160,123 @@ func TestBuildProxyRedirectURLCarriesAudioOnlyRemuxClaim(t *testing.T) { } } +// maxProxyTokenClaimGrowthBytes covers the path plus query-string growth from +// uid, pid, mfid, and ostn after JWT base64 expansion. +const maxProxyTokenClaimGrowthBytes = 256 + +func TestProxyRedirectURLClaimGrowthBudget(t *testing.T) { + h := &PlaybackHandler{JWTSecret: "test-secret"} + file := &models.MediaFile{FilePath: "/" + strings.Repeat("p", 511), VideoTracks: []models.VideoTrack{{DVProfile: 7}}} + source := PlaybackMediaSource{FileID: 2147483647} + session := &Session{StreamAppUserID: 2147483647, ProfileID: "123e4567-e89b-12d3-a456-426614174000"} + createdAt := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + transcodeNodeURL := "http://" + strings.Repeat("n", 57) // 64 bytes. + proxyNode := &nodepool.Node{URL: "http://proxy"} + + for _, method := range []string{string(playback.PlayDirect), string(playback.PlayRemux), string(playback.PlayTranscode)} { + t.Run(method, func(t *testing.T) { + withClaims, err := h.buildProxyRedirectURL("play", "upstream", method, file, source, session, createdAt, transcodeNodeURL, 12.5, proxyNode) + if err != nil { + t.Fatal(err) + } + withoutClaims, err := h.buildProxyRedirectURL("play", "upstream", method, file, source, nil, time.Time{}, transcodeNodeURL, 12.5, proxyNode) + if err != nil { + t.Fatal(err) + } + if growth := len(withClaims) - len(withoutClaims); growth > maxProxyTokenClaimGrowthBytes { + t.Fatalf("path + query claim growth = %d bytes, budget %d", growth, maxProxyTokenClaimGrowthBytes) + } + + token := proxyTokenFromRedirect(t, withClaims, method) + claims, err := streamtoken.Verify(token, h.JWTSecret) + if err != nil { + t.Fatal(err) + } + if claims.UserID != session.StreamAppUserID || claims.ProfileID != session.ProfileID || claims.MediaFileID != source.FileID || claims.OriginalStartedAtUnixNano != createdAt.UnixNano() { + t.Fatalf("ownership/start claims did not round trip: %#v", claims) + } + }) + } +} + +func proxyTokenFromRedirect(t *testing.T, rawURL, method string) string { + t.Helper() + u, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + prefix := "/stream/" + method + "/" + token := strings.TrimPrefix(u.Path, prefix) + if method == string(playback.PlayTranscode) { + token = strings.TrimSuffix(token, "/master.m3u8") + } + if token == "" || token == u.Path { + t.Fatalf("cannot extract token from %q", rawURL) + } + return token +} + +func TestUpstreamRecipeCardOverlaysTopLevelCreatedAt(t *testing.T) { + createdAt := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + compatSession := &Session{StreamAppUserID: 42, ProfileID: "profile-1"} + source := PlaybackMediaSource{FileID: 77} + h := &PlaybackHandler{} + + for _, tt := range []struct { + name string + method string + recipe *playback.RecipeCard + }{ + {name: "nested recipe from old replica", method: "transcode", recipe: &playback.RecipeCard{SessionID: "upstream"}}, + {name: "direct fallback", method: "direct"}, + {name: "remux fallback", method: "remux"}, + } { + t.Run(tt.name, func(t *testing.T) { + ps := &PlaybackSession{UpstreamSessionID: "upstream", CreatedAt: createdAt, Recipe: tt.recipe} + card := h.upstreamRecipeCard(ps, compatSession, source, tt.method) + if !card.OriginalStartedAt.Equal(createdAt) { + t.Fatalf("OriginalStartedAt = %s, want %s", card.OriginalStartedAt, createdAt) + } + }) + } + + mixedVersion := &PlaybackSession{ + UpstreamSessionID: "upstream-reconstruct", + CreatedAt: createdAt, + Recipe: &playback.RecipeCard{ + SessionID: "upstream-reconstruct", UserID: 42, ProfileID: "profile-1", MediaFileID: 77, + }, + } + card := h.upstreamRecipeCard(mixedVersion, compatSession, source, "transcode") + tm := playback.NewTranscodeManager() + tm.Sessions = playback.NewSessionManager(0, 0) + reconstructed := tm.ReconstructSession(t.Context(), mixedVersion.UpstreamSessionID, compatSession.StreamAppUserID, card) + if reconstructed == nil || !reconstructed.StartedAt.Equal(createdAt) { + t.Fatalf("mixed-version reconstruction = %#v, want StartedAt %s", reconstructed, createdAt) + } +} + +func TestPersistTranscodeRecipeCarriesTopLevelCreatedAt(t *testing.T) { + createdAt := time.Date(2026, 8, 16, 12, 34, 56, 987654321, time.UTC) + store := NewPlaybackSessionStore(time.Hour, nil) + // ExpiresAt must be set explicitly: the store derives a zero ExpiresAt as + // CreatedAt+ttl (playback_sessions.go:228), so a frozen CreatedAt would make + // this session read as already expired once wall-clock passes it. + store.Put(PlaybackSession{ID: "play", CreatedAt: createdAt, ExpiresAt: time.Now().Add(time.Hour)}) + manager := playback.NewSessionManager(0, 0) + manager.RegisterReconstructed(&playback.Session{ID: "upstream", UserID: 42, ProfileID: "profile-1", MediaFileID: 77, PlayMethod: playback.PlayTranscode}) + h := &PlaybackHandler{playbackStore: store, sessionMgr: manager} + + err := h.persistTranscodeRecipe(t.Context(), "play", "upstream", playback.TranscodeOpts{SessionID: "upstream", InputPath: "/media/movie.mkv"}) + if err != nil { + t.Fatal(err) + } + got, ok := store.Get("play") + if !ok || got.Recipe == nil || !got.Recipe.OriginalStartedAt.Equal(createdAt) { + t.Fatalf("persisted recipe = %#v, want OriginalStartedAt %s", got, createdAt) + } +} + func TestRewriteManifest_PreservesPlaybackAndMediaSourceIDs(t *testing.T) { manifest := strings.Join([]string{ "#EXTM3U", From 828253360da750cd011e28255c1e91cb45dd22ca Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:18:16 +0000 Subject: [PATCH 03/44] fix(clientip): resolve viewer addresses on the proxy and ABS listeners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two viewer-facing entry points recorded the wrong address. The standalone proxy mounted only CORS and egress metering, and the dedicated Audiobookshelf listener only its own access log — neither ran the trusted-proxy resolver that the native and Jellyfin routers have always had. The ABS case was not an empty field but a wrong one: `requestClientIP` falls back to `RemoteAddr`, so behind a reverse proxy every audiobook session and every `RemoteAddr`-based log line recorded the *proxy peer* rather than the viewer. The proxy listener had no resolution at all. Mounts `clientip.Middleware` first on both, so it runs before anything that reads the address. Proxy mode already has a Postgres pool and a config watcher, so the trusted-CIDR list and its hot reload work there exactly as in integrated mode. Error semantics deliberately mirror the integrated path: a malformed CIDR list at startup is fatal rather than silently starting with an empty trust set, because failing open would make every forwarding header both untrusted and unverified; a malformed list on reload logs and retains the last valid CIDRs. The reload closure that integrated mode already used is extracted and shared rather than copied. Behavior change worth stating: recorded session IPs and RemoteAddr-based log lines on both listeners now show the resolved viewer address instead of the reverse-proxy peer, since the middleware overwrites RemoteAddr. Adds trust-boundary tests over the mounted proxy router on a real socket — trusted forwarding header honored, spoofed header from an untrusted peer ignored, and a runtime narrowing of the trusted set taking effect — because the resolver reads RemoteAddr, which only a real connection populates. Part of the stream telemetry and enforcement effort (P0a). AI-use disclosure: implemented with AI assistance (Claude planning and review, Codex gpt-5.6-sol implementing), verified against the repo's own build, vet, lint and test gates. --- cmd/silo/main.go | 58 ++++-- internal/clientip/resolver_test.go | 53 ++++++ internal/proxy/router_socket_test.go | 264 +++++++++++++++++++++++++++ internal/proxy/server.go | 11 ++ 4 files changed, 371 insertions(+), 15 deletions(-) create mode 100644 internal/clientip/resolver_test.go create mode 100644 internal/proxy/router_socket_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 71a9da4ce..b4a6780b6 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -133,6 +133,39 @@ func resolveNodeIdentity() string { return h } +func clientIPResolverFromConfig(cfg *config.Config) (*clientip.Resolver, error) { + if cfg == nil { + return nil, fmt.Errorf("config is not loaded") + } + raw := cfg.ClientIP.TrustedProxies + if raw == "" { + raw = clientip.DefaultTrustedProxies + } + cidrs, err := clientip.ParseCIDRs(raw) + if err != nil { + return nil, err + } + return clientip.NewResolver(cidrs), nil +} + +func registerClientIPConfigReload(watcher *nodeconfig.Watcher, resolver *clientip.Resolver) { + watcher.OnChange(func(old, updated *config.Config) { + if old != nil && old.ClientIP.TrustedProxies == updated.ClientIP.TrustedProxies { + return + } + raw := updated.ClientIP.TrustedProxies + if raw == "" { + raw = clientip.DefaultTrustedProxies + } + cidrs, err := clientip.ParseCIDRs(raw) + if err != nil { + slog.WarnContext(context.Background(), "clientip config reload failed", "component", "app", "error", err) + return + } + resolver.UpdateTrustedCIDRs(cidrs) + }) +} + func resolvePluginCacheDir() string { if v := strings.TrimSpace(os.Getenv("SILO_PLUGIN_CACHE_DIR")); v != "" { return v @@ -718,6 +751,12 @@ func main() { var handler http.Handler if mode == "proxy" { srv := proxy.NewServer(watcher, tracker) + proxyIPResolver, resolverErr := clientIPResolverFromConfig(watcher.Config()) + if resolverErr != nil { + log.Fatalf("load trusted CIDRs: %v", resolverErr) + } + registerClientIPConfigReload(watcher, proxyIPResolver) + srv.SetClientIPResolver(proxyIPResolver) srv.SetRemoteArtifactMissReporter(downloads.NewArtifactManager( downloads.NewArtifactRepository(pool), downloads.NewRepository(pool), @@ -1846,21 +1885,7 @@ func main() { }) // The config watcher covers the Redis-less poll/RequestReload path, so // admin UI edits apply without a restart on single-node deployments too. - configWatcher.OnChange(func(old, updated *config.Config) { - if old != nil && old.ClientIP.TrustedProxies == updated.ClientIP.TrustedProxies { - return - } - raw := updated.ClientIP.TrustedProxies - if raw == "" { - raw = clientip.DefaultTrustedProxies - } - cidrs, parseErr := clientip.ParseCIDRs(raw) - if parseErr != nil { - slog.WarnContext(context.Background(), "clientip config reload failed", "component", "app", "error", parseErr) - return - } - ipResolver.UpdateTrustedCIDRs(cidrs) - }) + registerClientIPConfigReload(configWatcher, ipResolver) // Step 6b: Create rate limiter. if cfg.RateLimit.Enabled && deps.DB != nil { @@ -2701,6 +2726,9 @@ func main() { var absSrv *http.Server if (mode == "integrated" || mode == "api") && deps.ABSHandler != nil && cfg.AudiobookshelfCompat.Listen != "" { absRouter := chi.NewRouter() + if ipResolver != nil { + absRouter.Use(clientip.Middleware(ipResolver)) + } absRouter.Use(chimiddleware.Recoverer) absRouter.Use(chimiddleware.Compress(5)) deps.ABSHandler.Mount(absRouter) diff --git a/internal/clientip/resolver_test.go b/internal/clientip/resolver_test.go new file mode 100644 index 000000000..59d8f3c4b --- /dev/null +++ b/internal/clientip/resolver_test.go @@ -0,0 +1,53 @@ +package clientip + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestMiddlewareTrustBoundaryAndHotReload(t *testing.T) { + trusted, err := ParseCIDRs("127.0.0.0/8,10.0.0.0/8") + if err != nil { + t.Fatal(err) + } + resolver := NewResolver(trusted) + resolve := func(remote, xff, realIP string) string { + t.Helper() + var got string + h := Middleware(resolver)(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + got = r.RemoteAddr + if contextIP := FromContext(r.Context()); contextIP != got { + t.Fatalf("context IP = %q, RemoteAddr = %q", contextIP, got) + } + })) + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = remote + r.Header.Set("X-Forwarded-For", xff) + r.Header.Set("X-Real-IP", realIP) + h.ServeHTTP(httptest.NewRecorder(), r) + return got + } + + if got := resolve("127.0.0.1:1234", "198.51.100.9", ""); got != "198.51.100.9" { + t.Fatalf("trusted peer XFF = %q", got) + } + if got := resolve("203.0.113.7:1234", "198.51.100.9", ""); got != "203.0.113.7" { + t.Fatalf("untrusted spoof = %q", got) + } + if got := resolve("127.0.0.1:1234", "10.1.1.1, 127.0.0.2", ""); got != "10.1.1.1" { + t.Fatalf("all-trusted chain = %q", got) + } + if got := resolve("127.0.0.1:1234", "", "198.51.100.10"); got != "198.51.100.10" { + t.Fatalf("X-Real-IP fallback = %q", got) + } + + narrowed, err := ParseCIDRs("10.0.0.0/8") + if err != nil { + t.Fatal(err) + } + resolver.UpdateTrustedCIDRs(narrowed) + if got := resolve("127.0.0.1:1234", "198.51.100.9", ""); got != "127.0.0.1" { + t.Fatalf("hot reload did not narrow trust: %q", got) + } +} diff --git a/internal/proxy/router_socket_test.go b/internal/proxy/router_socket_test.go new file mode 100644 index 000000000..05e198329 --- /dev/null +++ b/internal/proxy/router_socket_test.go @@ -0,0 +1,264 @@ +package proxy + +import ( + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +const socketProxyMedia = "0123456789abcdefghijklmnopqrstuvwxyz" + +// newSocketProxyServer builds a proxy Server whose Handler() is mounted on a +// real listener, so the tests below exercise the assembled middleware chain +// (client IP resolution -> CORS -> egress metering) rather than a handler in +// isolation. That chain is the thing P0a changed; a handler-level test would +// bypass all of it. +func newSocketProxyServer(t *testing.T, secret string, resolver *clientip.Resolver) *Server { + t.Helper() + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = secret + w.SetConfigForTest(cfg) + srv := NewServer(w, nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy")) + srv.SetClientIPResolver(resolver) + return srv +} + +func socketProxyMediaToken(t *testing.T, secret, path string) string { + t.Helper() + token, err := streamtoken.Sign(streamtoken.Claims{ + SessionID: "socket-proxy-1", + MediaPath: path, + PlayMethod: "direct", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + }, secret, time.Minute) + if err != nil { + t.Fatalf("sign: %v", err) + } + return token +} + +func writeSocketProxyMedia(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "movie.mp4") + if err := os.WriteFile(path, []byte(socketProxyMedia), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// socketProxyResult is the fully-drained result of one request. The body is read +// and closed inside the helper so each case can assert on it directly. +type socketProxyResult struct { + status int + header http.Header + body string +} + +func socketProxyRequest(t *testing.T, client *http.Client, method, url string, headers map[string]string) socketProxyResult { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("%s %s: read body: %v", method, url, err) + } + return socketProxyResult{status: resp.StatusCode, header: resp.Header, body: string(body)} +} + +// TestMountedProxyRouterServesMediaOverSocket covers the HTTP surface of the +// proxy's mounted chain: GET/HEAD, ranges, conditional requests, and the fact +// that no compression middleware is mounted there (so media keeps its +// io.ReaderFrom path through the egress meter). +func TestMountedProxyRouterServesMediaOverSocket(t *testing.T) { + const secret = "socket-proxy-secret" + path := writeSocketProxyMedia(t) + srv := newSocketProxyServer(t, secret, nil) + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + mediaURL := server.URL + "/stream/direct/" + socketProxyMediaToken(t, secret, path) + + got := socketProxyRequest(t, client, http.MethodGet, mediaURL, nil) + if got.status != http.StatusOK || got.body != socketProxyMedia { + t.Fatalf("GET = %d %q, want 200 %q", got.status, got.body, socketProxyMedia) + } + etag := got.header.Get("ETag") + + if got = socketProxyRequest(t, client, http.MethodHead, mediaURL, nil); got.status != http.StatusOK { + t.Fatalf("HEAD = %d, want 200", got.status) + } + + got = socketProxyRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5"}) + if got.status != http.StatusPartialContent || got.body != "2345" { + t.Fatalf("Range = %d %q, want 206 %q", got.status, got.body, "2345") + } + + // The proxy mounts no compressor, so an Accept-Encoding request must still + // come back identity-encoded and byte-identical. + got = socketProxyRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Accept-Encoding": "gzip"}) + if enc := got.header.Get("Content-Encoding"); enc != "" { + t.Fatalf("media Content-Encoding = %q, want empty", enc) + } + if got.body != socketProxyMedia { + t.Fatalf("gzip-offered body = %q, want %q", got.body, socketProxyMedia) + } + + if etag != "" { + got = socketProxyRequest(t, client, http.MethodGet, mediaURL, map[string]string{"If-None-Match": etag}) + if got.status != http.StatusNotModified { + t.Fatalf("If-None-Match = %d, want 304", got.status) + } + } +} + +// TestMountedProxyRouterResolvesViewerIPOverSocket is the trust-boundary test for +// the resolver P0a mounted on the proxy. It runs over a real socket because the +// resolver reads RemoteAddr, which only a real connection populates: the peer is +// loopback, so a forwarding header is honored only when loopback is trusted. +func TestMountedProxyRouterResolvesViewerIPOverSocket(t *testing.T) { + const secret = "socket-proxy-ip-secret" + + trusted, err := clientip.ParseCIDRs("127.0.0.0/8,::1/128") + if err != nil { + t.Fatalf("ParseCIDRs: %v", err) + } + resolver := clientip.NewResolver(trusted) + + var seen string + srv := newSocketProxyServer(t, secret, resolver) + mounted := srv.Handler() + // No proxy route consumes the resolved address yet — that arrives with the + // telemetry phase — so observe it from a NotFound handler, which chi still + // runs through the full mounted middleware chain. That keeps this a test of + // the real chain rather than of clientip.Middleware in isolation. + router, ok := mounted.(chi.Router) + if !ok { + t.Fatalf("proxy Handler() is %T, want chi.Router", mounted) + } + router.NotFound(func(w http.ResponseWriter, r *http.Request) { + seen = clientip.FromContext(r.Context()) + w.WriteHeader(http.StatusNoContent) + }) + server := httptest.NewServer(mounted) + t.Cleanup(server.Close) + + probe := func(t *testing.T, headers map[string]string) string { + t.Helper() + seen = "" + client := &http.Client{} + defer client.CloseIdleConnections() + req, err := http.NewRequest(http.MethodGet, server.URL+"/unrouted-probe", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("GET: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return seen + } + + // Loopback is trusted here, so the forwarded viewer address wins. + if got := probe(t, map[string]string{"X-Forwarded-For": "203.0.113.9"}); got != "203.0.113.9" { + t.Fatalf("trusted XFF resolved to %q, want 203.0.113.9", got) + } + if got := probe(t, map[string]string{"X-Real-IP": "203.0.113.10"}); got != "203.0.113.10" { + t.Fatalf("trusted X-Real-IP resolved to %q, want 203.0.113.10", got) + } + + // Narrow the trust set at runtime: loopback is no longer a trusted proxy, so + // the same spoofed header must be ignored in favor of the real peer. + narrowed, err := clientip.ParseCIDRs("10.0.0.0/8") + if err != nil { + t.Fatalf("ParseCIDRs: %v", err) + } + resolver.UpdateTrustedCIDRs(narrowed) + got := probe(t, map[string]string{"X-Forwarded-For": "203.0.113.9"}) + if got == "203.0.113.9" { + t.Fatal("spoofed X-Forwarded-For was honored from an untrusted peer") + } + if ip := net.ParseIP(got); ip == nil || !ip.IsLoopback() { + t.Fatalf("untrusted peer resolved to %q, want the loopback peer address", got) + } +} + +// TestMountedProxyRouterRelaysToNode covers the proxy->node hop over real +// sockets: the proxy must stream the upstream node's bytes back to the viewer +// through the egress meter without corrupting them. +func TestMountedProxyRouterRelaysToNode(t *testing.T) { + const secret = "socket-proxy-relay-secret" + const segment = "segment-bytes-from-node" + + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/segment/") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "video/mp2t") + _, _ = io.WriteString(w, segment) + })) + t.Cleanup(node.Close) + + token, err := streamtoken.Sign(streamtoken.Claims{ + SessionID: "socket-relay-1", + PlayMethod: "transcode", + TranscodeNode: node.URL, + TranscodeTransportID: "transport-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + }, secret, time.Minute) + if err != nil { + t.Fatalf("sign: %v", err) + } + + srv := newSocketProxyServer(t, secret, nil) + before := srv.egress.RateKbps() + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + + got := socketProxyRequest(t, client, http.MethodGet, server.URL+"/stream/transcode/"+token+"/segment/000.ts", nil) + if got.status != http.StatusOK || got.body != segment { + t.Fatalf("relayed segment = %d %q, want 200 %q", got.status, got.body, segment) + } + if srv.egress.RateKbps() < before { + t.Fatal("relayed bytes were not counted by the egress meter") + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 22b5692c9..7baf192ae 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -18,6 +18,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/cors" + "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/downloads" "github.com/Silo-Server/silo-server/internal/nodeconfig" @@ -33,6 +34,7 @@ type Server struct { httpClient *http.Client artifactMissReporter remoteArtifactMissReporter egress *egressMeter + clientIP *clientip.Resolver // subCache stores full-track PGS (.sup) extracts under the transcode dir // so repeat selections skip the whole-file ffmpeg demux. subCache *playback.SubtitleCache @@ -77,6 +79,12 @@ func (s *Server) SetRemoteArtifactMissReporter(reporter remoteArtifactMissReport s.artifactMissReporter = reporter } +// SetClientIPResolver wires trusted-proxy client IP resolution. It must be +// called during construction, before the server begins handling requests. +func (s *Server) SetClientIPResolver(resolver *clientip.Resolver) { + s.clientIP = resolver +} + // newStreamTransport tunes the proxy→transcode-node connection pool. Many // concurrent viewers fan their segment fetches through one proxy→node pair, // and Go's default of 2 idle connections per host causes constant connection @@ -94,6 +102,9 @@ func newStreamTransport() *http.Transport { // Handler returns the chi.Router with all proxy routes mounted. func (s *Server) Handler() http.Handler { r := chi.NewRouter() + if s.clientIP != nil { + r.Use(clientip.Middleware(s.clientIP)) + } // hls.js uses XHR for manifest/segment fetches which are subject to // CORS when the proxy runs on a different origin than the web app. r.Use(cors.Handler(cors.Options{ From 760287d6337fd351fb337f57d1307bb4f6b55d8c Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:18:47 +0000 Subject: [PATCH 04/44] fix(httpstream): keep sendfile and the write deadline alive through wrappers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Media was being served the slow way, and one middleware silently disabled the server's ability to interrupt a stuck stream. `io.Copy` — and therefore `http.ServeContent` — finds `io.ReaderFrom` by direct type assertion and never through `Unwrap()`. Every status/logging/metrics wrapper on a media route that did not forward `ReadFrom` turned off the zero-copy path for everything below it, so large direct-play and download bodies were copied through the application instead of handed to the kernel. Separately, a wrapper without `Unwrap()` dead-ends `http.ResponseController`, which is how the rolling write deadline is set — the same deadline the enforcement phase will use as its in-flight interrupt. Adds shared helpers in `httpstream` (`ReaderFromOf`, `CopyChunked`, `WriterOnly`) and forwards `ReadFrom` through every wrapper on a live media chain, preserving each one's own accounting: byte-counting wrappers (the proxy egress meter, the ABS access log, the jellycompat debug writer) transfer in bounded slices and credit each one, so the meter's rolling per-second window is not collapsed into a single bucket by one large transfer. Also adds the `Unwrap` the ABS access log never had (GAP-10) and the `Unwrap`/`Hijack` the jellycompat image-proxy writer never had. chi's `compressResponseWriter` implements `Unwrap`, `Flush`, `Hijack` and `Push` but not `ReadFrom`, and its handler wraps unconditionally — the encoder is chosen later, so even a non-compressible content type gets a wrapper that kills sendfile. It is third-party, so it cannot be repaired. Compression is therefore bypassed on exact bulk-media routes via `CompressExcept`, matching only the registered GET/HEAD methods with exact segment counts and exact casing, so a wrong-method or child path is never swallowed. Blanket bypass would have been wrong: subtitle font bundles are JSON served under the same global compressor, and bypassing them would change the wire contract. Also fixes a pre-existing reap of healthy streams. The deadline was refreshed only between 64 MiB slices against a 180s stall window, so any client sustaining less than ~3 Mbit/s had its deadline expire mid-slice and was killed despite continuous progress. The slice is now 4 MiB (~186 kbit/s floor), with tests covering both a steadily-progressing slow stream and the oversized-slice failure mode, plus a guard on the constant itself. Verified over real sockets against the mounted routers — GET, HEAD, single and multi-range, conditional responses, Accept-Encoding present and absent, HTTP/2, proxy-to-node relay, the ABS socket.io upgrade, and the jellycompat image-proxy client path — because handler-level tests bypass exactly the middleware this changes. Adds direct-play, remux and high-RPS HLS benchmarks as a baseline for the hot path. Part of the stream telemetry and enforcement effort (P0a). AI-use disclosure: implemented with AI assistance (Claude planning and review, Codex gpt-5.6-sol implementing), verified against the repo's own build, vet, lint and test gates. --- cmd/silo/main.go | 4 +- internal/activitylog/middleware.go | 13 + internal/activitylog/readfrom_test.go | 25 ++ internal/api/middleware/metrics.go | 21 ++ internal/api/middleware/readfrom_test.go | 39 +++ internal/api/middleware/request_logger.go | 13 + internal/api/router.go | 74 ++++-- internal/api/router_compression_test.go | 34 +++ internal/api/router_http2_test.go | 80 ++++++ internal/api/router_socket_test.go | 195 +++++++++++++++ internal/audiobooks/abs/access_log.go | 21 ++ internal/audiobooks/abs/compression_test.go | 34 +++ internal/audiobooks/abs/handler.go | 36 +++ internal/audiobooks/abs/readfrom_test.go | 28 +++ internal/audiobooks/abs/router_socket_test.go | 59 +++++ internal/httpstream/compress.go | 22 ++ internal/httpstream/readfrom.go | 49 ++++ internal/httpstream/readfrom_bench_test.go | 146 +++++++++++ internal/httpstream/readfrom_deadline_test.go | 152 ++++++++++++ internal/httpstream/readfrom_test.go | 69 ++++++ internal/httpstream/rolling_deadline.go | 34 +-- internal/jellycompat/image_proxy_tags.go | 39 +++ internal/jellycompat/logging.go | 38 +++ internal/jellycompat/readfrom_test.go | 83 +++++++ internal/jellycompat/router.go | 25 +- .../jellycompat/router_compression_test.go | 32 +++ internal/jellycompat/router_socket_test.go | 234 ++++++++++++++++++ internal/proxy/egress.go | 18 +- internal/proxy/egress_readfrom_test.go | 29 +++ 29 files changed, 1599 insertions(+), 47 deletions(-) create mode 100644 internal/activitylog/readfrom_test.go create mode 100644 internal/api/middleware/readfrom_test.go create mode 100644 internal/api/router_compression_test.go create mode 100644 internal/api/router_http2_test.go create mode 100644 internal/api/router_socket_test.go create mode 100644 internal/audiobooks/abs/compression_test.go create mode 100644 internal/audiobooks/abs/readfrom_test.go create mode 100644 internal/audiobooks/abs/router_socket_test.go create mode 100644 internal/httpstream/compress.go create mode 100644 internal/httpstream/readfrom.go create mode 100644 internal/httpstream/readfrom_bench_test.go create mode 100644 internal/httpstream/readfrom_deadline_test.go create mode 100644 internal/httpstream/readfrom_test.go create mode 100644 internal/jellycompat/readfrom_test.go create mode 100644 internal/jellycompat/router_compression_test.go create mode 100644 internal/jellycompat/router_socket_test.go create mode 100644 internal/proxy/egress_readfrom_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index b4a6780b6..8ebadeb12 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -40,6 +40,7 @@ import ( "github.com/Silo-Server/silo-server/internal/api" "github.com/Silo-Server/silo-server/internal/api/handlers" "github.com/Silo-Server/silo-server/internal/audiobooks" + "github.com/Silo-Server/silo-server/internal/audiobooks/abs" "github.com/Silo-Server/silo-server/internal/audiobooks/podcastfeed" "github.com/Silo-Server/silo-server/internal/auth" "github.com/Silo-Server/silo-server/internal/autoscan" @@ -56,6 +57,7 @@ import ( "github.com/Silo-Server/silo-server/internal/ebooks" evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/httpstream" "github.com/Silo-Server/silo-server/internal/imagecache" "github.com/Silo-Server/silo-server/internal/intromarkers" "github.com/Silo-Server/silo-server/internal/jellycompat" @@ -2730,7 +2732,7 @@ func main() { absRouter.Use(clientip.Middleware(ipResolver)) } absRouter.Use(chimiddleware.Recoverer) - absRouter.Use(chimiddleware.Compress(5)) + absRouter.Use(httpstream.CompressExcept(5, abs.SkipMediaCompression)) deps.ABSHandler.Mount(absRouter) absSrv = &http.Server{ Addr: cfg.AudiobookshelfCompat.Listen, diff --git a/internal/activitylog/middleware.go b/internal/activitylog/middleware.go index 26c6674ab..e6ccb8757 100644 --- a/internal/activitylog/middleware.go +++ b/internal/activitylog/middleware.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "io" "net" "net/http" "strings" @@ -13,6 +14,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/httpstream" ) // excludedPrefixes are paths that should not be logged. @@ -201,6 +203,17 @@ func (w *statusWriter) Write(b []byte) (int, error) { return w.ResponseWriter.Write(b) } +func (w *statusWriter) ReadFrom(src io.Reader) (int64, error) { + if !w.wroteHeader { + w.status, w.wroteHeader = http.StatusOK, true + } + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, 0, nil) +} + // Hijack implements http.Hijacker, required for WebSocket upgrades. func (w *statusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := w.ResponseWriter.(http.Hijacker); ok { diff --git a/internal/activitylog/readfrom_test.go b/internal/activitylog/readfrom_test.go new file mode 100644 index 000000000..d7b6dae08 --- /dev/null +++ b/internal/activitylog/readfrom_test.go @@ -0,0 +1,25 @@ +package activitylog + +import ( + "bytes" + "io" + "net/http" + "testing" +) + +type readerFromSpy struct{ bytes.Buffer } + +func (w *readerFromSpy) Header() http.Header { return make(http.Header) } +func (w *readerFromSpy) WriteHeader(int) {} +func (w *readerFromSpy) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(&w.Buffer, r) +} + +func TestStatusWriterPreservesReaderFrom(t *testing.T) { + spy := &readerFromSpy{} + w := &statusWriter{ResponseWriter: spy} + n, err := w.ReadFrom(bytes.NewBufferString("media")) + if err != nil || n != 5 || spy.String() != "media" || w.status != http.StatusOK { + t.Fatalf("ReadFrom = n=%d err=%v status=%d body=%q", n, err, w.status, spy.String()) + } +} diff --git a/internal/api/middleware/metrics.go b/internal/api/middleware/metrics.go index b2aa0127b..9c401b97b 100644 --- a/internal/api/middleware/metrics.go +++ b/internal/api/middleware/metrics.go @@ -3,6 +3,7 @@ package middleware import ( "bufio" "fmt" + "io" "net" "net/http" "regexp" @@ -11,6 +12,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) var ( @@ -64,6 +67,24 @@ func (w *statusWriter) WriteHeader(status int) { w.ResponseWriter.WriteHeader(status) } +func (w *statusWriter) Write(b []byte) (int, error) { + if !w.written { + w.status, w.written = http.StatusOK, true + } + return w.ResponseWriter.Write(b) +} + +func (w *statusWriter) ReadFrom(src io.Reader) (int64, error) { + if !w.written { + w.status, w.written = http.StatusOK, true + } + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, 0, nil) +} + // Hijack implements http.Hijacker, required for WebSocket upgrades. func (w *statusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := w.ResponseWriter.(http.Hijacker); ok { diff --git a/internal/api/middleware/readfrom_test.go b/internal/api/middleware/readfrom_test.go new file mode 100644 index 000000000..4d06e0c0d --- /dev/null +++ b/internal/api/middleware/readfrom_test.go @@ -0,0 +1,39 @@ +package middleware + +import ( + "bytes" + "io" + "net/http" + "testing" +) + +type readerFromSpy struct { + bytes.Buffer + header http.Header + calls int +} + +func (w *readerFromSpy) Header() http.Header { return w.header } +func (w *readerFromSpy) WriteHeader(int) {} +func (w *readerFromSpy) ReadFrom(r io.Reader) (int64, error) { + w.calls++ + return io.Copy(&w.Buffer, r) +} + +func TestStatusWritersPreserveReaderFrom(t *testing.T) { + for _, wrap := range []struct { + name string + new func(http.ResponseWriter) io.ReaderFrom + }{ + {"request logger", func(w http.ResponseWriter) io.ReaderFrom { return &requestStatusWriter{ResponseWriter: w} }}, + {"metrics", func(w http.ResponseWriter) io.ReaderFrom { return &statusWriter{ResponseWriter: w} }}, + } { + t.Run(wrap.name, func(t *testing.T) { + spy := &readerFromSpy{header: make(http.Header)} + n, err := wrap.new(spy).ReadFrom(bytes.NewBufferString("media")) + if err != nil || n != 5 || spy.calls != 1 || spy.String() != "media" { + t.Fatalf("ReadFrom = n=%d err=%v calls=%d body=%q", n, err, spy.calls, spy.String()) + } + }) + } +} diff --git a/internal/api/middleware/request_logger.go b/internal/api/middleware/request_logger.go index 7a8f3fc79..271dd3655 100644 --- a/internal/api/middleware/request_logger.go +++ b/internal/api/middleware/request_logger.go @@ -3,6 +3,7 @@ package middleware import ( "bufio" "fmt" + "io" "log/slog" "net" "net/http" @@ -14,6 +15,7 @@ import ( "github.com/Silo-Server/silo-server/internal/activitylog" "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/httpstream" ) func RequestLogger(nodeID string) func(http.Handler) http.Handler { @@ -95,6 +97,17 @@ func (w *requestStatusWriter) Write(b []byte) (int, error) { return w.ResponseWriter.Write(b) } +func (w *requestStatusWriter) ReadFrom(src io.Reader) (int64, error) { + if !w.wroteHeader { + w.status, w.wroteHeader = http.StatusOK, true + } + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, 0, nil) +} + func (w *requestStatusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := w.ResponseWriter.(http.Hijacker); ok { return hj.Hijack() diff --git a/internal/api/router.go b/internal/api/router.go index 60febacc3..5e5d777a8 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -36,6 +36,7 @@ import ( "github.com/Silo-Server/silo-server/internal/downloads" evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/httpstream" "github.com/Silo-Server/silo-server/internal/intromarkers" "github.com/Silo-Server/silo-server/internal/invitations" "github.com/Silo-Server/silo-server/internal/libraryingest" @@ -217,26 +218,7 @@ func (d *Dependencies) CurrentConfig() *config.Config { func NewRouter(deps Dependencies) chi.Router { r := chi.NewRouter() - // Standard middleware. - r.Use(middleware.RequestID) - - // Client IP resolution must run before request logging. - if deps.ClientIPResolver != nil { - r.Use(clientip.Middleware(deps.ClientIPResolver)) - } - - r.Use(apimw.RequestLogger(deps.NodeID)) - r.Use(middleware.Recoverer) - r.Use(apimw.Metrics) - - // Compress text-like responses (JSON, SVG, …); media content types are - // not in the middleware's allowlist and stream through untouched. - r.Use(middleware.Compress(5)) - - // Activity logging (before auth — captures all requests including failed auth). - if deps.ActivityLogWriter != nil { - r.Use(activitylog.NewMiddleware(deps.ActivityLogWriter, deps.NodeID)) - } + useBaseMiddleware(r, deps) // Build the readiness handler with optional S3 check. var s3Checker handlers.S3HealthChecker @@ -3277,6 +3259,58 @@ func NewRouter(deps Dependencies) chi.Router { return r } +// useBaseMiddleware mounts the middleware chain every native request passes +// through, in order. It is factored out of NewRouter so a test can drive the +// real chain over a real socket: re-declaring the stack in a test would let the +// two drift, and a drifted copy is exactly how a broken writer chain passes its +// own tests (see the §4.4 conformance requirement in the stream-telemetry design). +func useBaseMiddleware(r chi.Router, deps Dependencies) { + // Standard middleware. + r.Use(middleware.RequestID) + + // Client IP resolution must run before request logging. + if deps.ClientIPResolver != nil { + r.Use(clientip.Middleware(deps.ClientIPResolver)) + } + + r.Use(apimw.RequestLogger(deps.NodeID)) + r.Use(middleware.Recoverer) + r.Use(apimw.Metrics) + + // Compress text-like responses (JSON, SVG, …), while leaving exact bulk + // media routes unwrapped so their io.ReaderFrom/sendfile path survives. + r.Use(httpstream.CompressExcept(5, skipNativeMediaCompression)) + + // Activity logging (before auth — captures all requests including failed auth). + if deps.ActivityLogWriter != nil { + r.Use(activitylog.NewMiddleware(deps.ActivityLogWriter, deps.NodeID)) + } +} + +func skipNativeMediaCompression(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + p := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + if len(p) < 3 || p[0] != "api" || p[1] != "v1" { + return false + } + switch { + case len(p) == 4 && p[2] == "stream" && p[3] != "": + return true + case len(p) == 7 && p[2] == "playback" && p[3] == "transcode" && p[4] != "" && p[5] == "segment" && p[6] != "": + return true + case len(p) == 5 && p[2] == "downloads" && p[3] != "" && (p[4] == "file" || p[4] == "file-proxy"): + return true + case len(p) == 3 && (p[2] == "direct-download" || p[2] == "direct-download-proxy"): + return true + case len(p) == 7 && p[2] == "ebooks" && p[3] != "" && p[4] == "files" && p[5] != "" && p[6] == "read": + return true + default: + return false + } +} + // optionalProfileViewerAccess preserves the established profile-less plugin // launch path while validating any profile a newer caller asks the launch // cookie to carry. A missing viewer resolver must not remove this existing v1 diff --git a/internal/api/router_compression_test.go b/internal/api/router_compression_test.go new file mode 100644 index 000000000..47d44988c --- /dev/null +++ b/internal/api/router_compression_test.go @@ -0,0 +1,34 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestSkipNativeMediaCompression(t *testing.T) { + tests := []struct { + method, path string + want bool + }{ + {http.MethodGet, "/api/v1/stream/s1", true}, + {http.MethodHead, "/api/v1/playback/transcode/s1/segment/000.ts", true}, + {http.MethodGet, "/api/v1/downloads/d1/file", true}, + {http.MethodGet, "/api/v1/downloads/d1/file-proxy", true}, + {http.MethodGet, "/api/v1/direct-download", true}, + {http.MethodHead, "/api/v1/direct-download-proxy", true}, + {http.MethodGet, "/api/v1/ebooks/c1/files/f1/read", true}, + {http.MethodGet, "/api/v1/stream/s1/subtitles/1", false}, + {http.MethodGet, "/api/v1/stream/s1/subtitles/1/fonts", false}, + {http.MethodGet, "/api/v1/stream/s1/", false}, + {http.MethodPost, "/api/v1/stream/s1", false}, + {http.MethodGet, "/API/v1/stream/s1", false}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.path, func(t *testing.T) { + if got := skipNativeMediaCompression(httptest.NewRequest(tt.method, tt.path, nil)); got != tt.want { + t.Fatalf("skipNativeMediaCompression = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/api/router_http2_test.go b/internal/api/router_http2_test.go new file mode 100644 index 000000000..082406d64 --- /dev/null +++ b/internal/api/router_http2_test.go @@ -0,0 +1,80 @@ +package api + +import ( + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/config" +) + +// TestMountedNativeRouterServesMediaOverHTTP2 is a does-not-regress check. +// HTTP/2 has no sendfile: the h2 layer frames every response body itself, so +// the io.ReaderFrom fast path the writer chain now preserves is simply unused. +// The wrappers must still behave — correct status, ranges, and an unwrapped +// (uncompressed) media body on a bypassed route — rather than assuming the +// HTTP/1.1 shape of the connection. +func TestMountedNativeRouterServesMediaOverHTTP2(t *testing.T) { + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatalf("LoadFromDB: %v", err) + } + + routes := &socketRoutes{} + root := chi.NewRouter() + useBaseMiddleware(root, Dependencies{ + Config: cfg, + ActivityLogWriter: socketActivityWriter{}, + }) + routes.Mount(root) + + server := httptest.NewUnstartedServer(root) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + + client := server.Client() + if tr, ok := client.Transport.(*http.Transport); ok { + tr.DisableCompression = true + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // httptest self-signed cert + } + t.Cleanup(client.CloseIdleConnections) + + mediaURL := server.URL + "/api/v1/stream/socket-test" + + resp, err := client.Get(mediaURL) + if err != nil { + t.Fatalf("GET over HTTP/2: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.ProtoMajor != 2 { + t.Fatalf("negotiated HTTP/%d.%d, want HTTP/2 — the test is not exercising h2", + resp.ProtoMajor, resp.ProtoMinor) + } + if resp.StatusCode != http.StatusOK || string(body) != "0123456789abcdefghijklmnopqrstuvwxyz" { + t.Fatalf("HTTP/2 GET = %d %q", resp.StatusCode, body) + } + if enc := resp.Header.Get("Content-Encoding"); enc != "" { + t.Fatalf("HTTP/2 media Content-Encoding = %q, want empty on a bypassed route", enc) + } + + req, err := http.NewRequest(http.MethodGet, mediaURL, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Range", "bytes=2-5") + resp, err = client.Do(req) + if err != nil { + t.Fatalf("ranged GET over HTTP/2: %v", err) + } + body, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent || string(body) != "2345" { + t.Fatalf("HTTP/2 Range = %d %q, want 206 %q", resp.StatusCode, body, "2345") + } +} diff --git a/internal/api/router_socket_test.go b/internal/api/router_socket_test.go new file mode 100644 index 000000000..4245a6e30 --- /dev/null +++ b/internal/api/router_socket_test.go @@ -0,0 +1,195 @@ +package api + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/activitylog" + "github.com/Silo-Server/silo-server/internal/config" +) + +const nativeSocketMediaETag = `"native-socket-media-v1"` + +type socketActivityWriter struct{} + +func (socketActivityWriter) Write(activitylog.LogEntry) {} +func (socketActivityWriter) Close() error { return nil } + +type socketRoutes struct { + readerFromSeen atomic.Bool +} + +func (h *socketRoutes) Mount(r chi.Router) { + media := []byte("0123456789abcdefghijklmnopqrstuvwxyz") + serveMedia := func(w http.ResponseWriter, req *http.Request) { + _, ok := w.(io.ReaderFrom) + h.readerFromSeen.Store(ok) + w.Header().Set("Content-Type", "video/mp4") + w.Header().Set("ETag", nativeSocketMediaETag) + http.ServeContent(w, req, "movie.mp4", time.Unix(1_700_000_000, 0), bytes.NewReader(media)) + } + r.Get("/api/v1/stream/socket-test", serveMedia) + r.Head("/api/v1/stream/socket-test", serveMedia) + r.Get("/api/v1/stream/socket-test/subtitles/1/fonts", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"payload": strings.Repeat("compressible-json-", 128)}) + }) +} + +func TestMountedNativeRouterPreservesMediaHTTPAndCompression(t *testing.T) { + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatalf("LoadFromDB: %v", err) + } + routes := &socketRoutes{} + // Drive the real middleware chain (useBaseMiddleware is what NewRouter + // itself mounts) rather than NewRouter's full route tree: the media routes + // are only registered when their handler dependencies are non-nil, which a + // unit test cannot supply. Mounting the shared chain keeps this test honest + // about ordering while still exercising stub media routes end to end. + root := chi.NewRouter() + useBaseMiddleware(root, Dependencies{ + Config: cfg, + ActivityLogWriter: socketActivityWriter{}, + }) + routes.Mount(root) + server := httptest.NewUnstartedServer(root) + server.Start() + t.Cleanup(server.Close) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + + mediaURL := server.URL + "/api/v1/stream/socket-test" + assertNativeSocketResponse(t, client, http.MethodGet, mediaURL, nil, http.StatusOK, "0123456789abcdefghijklmnopqrstuvwxyz") + assertNativeSocketResponse(t, client, http.MethodHead, mediaURL, nil, http.StatusOK, "") + assertNativeSocketResponse(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5"}, http.StatusPartialContent, "2345") + assertNativeMultiRange(t, client, mediaURL) + assertNativeSocketResponse(t, client, http.MethodGet, mediaURL, map[string]string{"If-None-Match": nativeSocketMediaETag}, http.StatusNotModified, "") + assertNativeSocketResponse(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5", "If-Range": nativeSocketMediaETag}, http.StatusPartialContent, "2345") + assertNativeSocketResponse(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5", "If-Range": `"stale"`}, http.StatusOK, "0123456789abcdefghijklmnopqrstuvwxyz") + + resp := nativeSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Accept-Encoding": "gzip"}) + body := readNativeSocketBody(t, resp) + if encoding := resp.Header.Get("Content-Encoding"); encoding != "" { + t.Fatalf("media Content-Encoding = %q, want empty", encoding) + } + if string(body) != "0123456789abcdefghijklmnopqrstuvwxyz" { + t.Fatalf("media body = %q", body) + } + if !routes.readerFromSeen.Load() { + t.Fatal("bypassed media handler ResponseWriter does not implement io.ReaderFrom through mounted middleware") + } + + jsonURL := server.URL + "/api/v1/stream/socket-test/subtitles/1/fonts" + resp = nativeSocketRequest(t, client, http.MethodGet, jsonURL, map[string]string{"Accept-Encoding": "gzip"}) + defer func() { _ = resp.Body.Close() }() + if encoding := resp.Header.Get("Content-Encoding"); encoding != "gzip" { + t.Fatalf("JSON Content-Encoding = %q, want gzip", encoding) + } + if vary := resp.Header.Values("Vary"); !headerValuesContain(vary, "Accept-Encoding") { + t.Fatalf("JSON Vary = %q, want Accept-Encoding", vary) + } + zr, err := gzip.NewReader(resp.Body) + if err != nil { + t.Fatalf("gzip.NewReader: %v", err) + } + defer func() { _ = zr.Close() }() + if body, err := io.ReadAll(zr); err != nil || !bytes.Contains(body, []byte("compressible-json")) { + t.Fatalf("compressed JSON body invalid: body=%q err=%v", body, err) + } +} + +func assertNativeSocketResponse(t *testing.T, client *http.Client, method, url string, headers map[string]string, wantStatus int, wantBody string) { + t.Helper() + resp := nativeSocketRequest(t, client, method, url, headers) + body := readNativeSocketBody(t, resp) + _ = resp.Body.Close() + if resp.StatusCode != wantStatus || string(body) != wantBody { + t.Fatalf("%s status/body = %d, %q; want %d, %q", method, resp.StatusCode, body, wantStatus, wantBody) + } + if encoding := resp.Header.Get("Content-Encoding"); encoding != "" { + t.Fatalf("%s Content-Encoding = %q, want empty", method, encoding) + } +} + +func assertNativeMultiRange(t *testing.T, client *http.Client, url string) { + t.Helper() + resp := nativeSocketRequest(t, client, http.MethodGet, url, map[string]string{"Range": "bytes=0-1,4-6"}) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusPartialContent { + t.Fatalf("multi-range status = %d, want 206", resp.StatusCode) + } + mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil || mediaType != "multipart/byteranges" { + t.Fatalf("multi-range Content-Type = %q: %v", resp.Header.Get("Content-Type"), err) + } + mr := multipart.NewReader(resp.Body, params["boundary"]) + var bodies []string + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("read multipart range: %v", err) + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("read multipart body: %v", err) + } + bodies = append(bodies, string(body)) + } + if strings.Join(bodies, ",") != "01,456" { + t.Fatalf("multi-range bodies = %q, want [01 456]", bodies) + } +} + +func nativeSocketRequest(t *testing.T, client *http.Client, method, url string, headers map[string]string) *http.Response { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + for key, value := range headers { + req.Header.Set(key, value) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("request %s %s: %v", method, url, err) + } + return resp +} + +func readNativeSocketBody(t *testing.T, resp *http.Response) []byte { + t.Helper() + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + return body +} + +func headerValuesContain(values []string, want string) bool { + for _, value := range values { + for _, part := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(part), want) { + return true + } + } + } + return false +} diff --git a/internal/audiobooks/abs/access_log.go b/internal/audiobooks/abs/access_log.go index 8416732f1..0fa5c6291 100644 --- a/internal/audiobooks/abs/access_log.go +++ b/internal/audiobooks/abs/access_log.go @@ -3,11 +3,14 @@ package abs import ( "bufio" "errors" + "io" "log/slog" "net" "net/http" "strings" "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) // accessLog is a minimal chi middleware that emits one structured line @@ -93,11 +96,29 @@ func (s *statusRecorder) WriteHeader(code int) { } func (s *statusRecorder) Write(b []byte) (int, error) { + if s.status == 0 { + s.status = http.StatusOK + } n, err := s.ResponseWriter.Write(b) s.bytes += n return n, err } +func (s *statusRecorder) ReadFrom(src io.Reader) (int64, error) { + if s.status == 0 { + s.status = http.StatusOK + } + rf, ok := httpstream.ReaderFromOf(s.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(s), src) + } + return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + s.bytes += int(n) + }) +} + +func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter } + // Hijack passes through to the wrapped ResponseWriter so socket.io // WebSocket upgrades can take ownership of the raw connection. // Without this, the underlying engine.io transport sees a diff --git a/internal/audiobooks/abs/compression_test.go b/internal/audiobooks/abs/compression_test.go new file mode 100644 index 000000000..7836c2807 --- /dev/null +++ b/internal/audiobooks/abs/compression_test.go @@ -0,0 +1,34 @@ +package abs + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestSkipMediaCompression(t *testing.T) { + tests := []struct { + method, path string + want bool + }{ + {http.MethodGet, "/api/items/i1/file/2", true}, + {http.MethodGet, "/api/items/i1/file/2/download", true}, + {http.MethodHead, "/abs/api/items/i1/file/2", true}, + {http.MethodGet, "/abs/api/items/i1/file/2/download", true}, + {http.MethodGet, "/public/session/s1/track/2", true}, + {http.MethodGet, "/abs/public/session/s1/track/2", true}, + {http.MethodGet, "/feed/books/file/2", true}, + {http.MethodGet, "/socket.io/", false}, + {http.MethodGet, "/api/items/i1/file/2/extra", false}, + {http.MethodPut, "/api/items/i1/file/2", false}, + {http.MethodGet, "/API/items/i1/file/2", false}, + {http.MethodGet, "/api/items/i1/file/2/", false}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.path, func(t *testing.T) { + if got := SkipMediaCompression(httptest.NewRequest(tt.method, tt.path, nil)); got != tt.want { + t.Fatalf("SkipMediaCompression = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/audiobooks/abs/handler.go b/internal/audiobooks/abs/handler.go index b615a8f77..2a1d2e378 100644 --- a/internal/audiobooks/abs/handler.go +++ b/internal/audiobooks/abs/handler.go @@ -292,6 +292,42 @@ type Handler struct { deps Dependencies } +// SkipMediaCompression reports whether an ABS media route must retain the +// server's original ResponseWriter for sendfile and optional interface support. +func SkipMediaCompression(r *http.Request) bool { + const ( + apiSegment = "api" + absSegment = "abs" + downloadSegment = "download" + fileSegment = "file" + itemsSegment = "items" + publicSegment = "public" + sessionSegment = "session" + ) + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + p := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + switch { + case len(p) == 5 && p[0] == apiSegment && p[1] == itemsSegment && p[2] != "" && p[3] == fileSegment && p[4] != "": + return true + case len(p) == 6 && p[0] == apiSegment && p[1] == itemsSegment && p[2] != "" && p[3] == fileSegment && p[4] != "" && p[5] == downloadSegment: + return true + case len(p) == 6 && p[0] == absSegment && p[1] == apiSegment && p[2] == itemsSegment && p[3] != "" && p[4] == fileSegment && p[5] != "": + return true + case len(p) == 7 && p[0] == absSegment && p[1] == apiSegment && p[2] == itemsSegment && p[3] != "" && p[4] == fileSegment && p[5] != "" && p[6] == downloadSegment: + return true + case len(p) == 5 && p[0] == publicSegment && p[1] == sessionSegment && p[2] != "" && p[3] == "track" && p[4] != "": + return true + case len(p) == 6 && p[0] == absSegment && p[1] == publicSegment && p[2] == sessionSegment && p[3] != "" && p[4] == "track" && p[5] != "": + return true + case len(p) == 4 && p[0] == "feed" && p[1] != "" && p[2] == fileSegment && p[3] != "": + return true + default: + return false + } +} + // New constructs an ABS Handler. Sensible defaults are applied for optional // fields (LoginLimiter, InstallID). // diff --git a/internal/audiobooks/abs/readfrom_test.go b/internal/audiobooks/abs/readfrom_test.go new file mode 100644 index 000000000..05dc7490a --- /dev/null +++ b/internal/audiobooks/abs/readfrom_test.go @@ -0,0 +1,28 @@ +package abs + +import ( + "bytes" + "io" + "net/http" + "testing" +) + +type readerFromSpy struct{ bytes.Buffer } + +func (w *readerFromSpy) Header() http.Header { return make(http.Header) } +func (w *readerFromSpy) WriteHeader(int) {} +func (w *readerFromSpy) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(&w.Buffer, r) +} + +func TestStatusRecorderPreservesReaderFromAndAccounting(t *testing.T) { + spy := &readerFromSpy{} + w := &statusRecorder{ResponseWriter: spy} + n, err := w.ReadFrom(bytes.NewBufferString("media")) + if err != nil || n != 5 || w.bytes != 5 || w.status != http.StatusOK || spy.String() != "media" { + t.Fatalf("ReadFrom = n=%d err=%v status=%d bytes=%d body=%q", n, err, w.status, w.bytes, spy.String()) + } + if w.Unwrap() != spy { + t.Fatal("Unwrap did not return underlying writer") + } +} diff --git a/internal/audiobooks/abs/router_socket_test.go b/internal/audiobooks/abs/router_socket_test.go new file mode 100644 index 000000000..4e5a331ba --- /dev/null +++ b/internal/audiobooks/abs/router_socket_test.go @@ -0,0 +1,59 @@ +package abs + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type hijackingSocketIOServer struct{} + +func (hijackingSocketIOServer) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijack unavailable", http.StatusInternalServerError) + return + } + conn, rw, err := hijacker.Hijack() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _, _ = rw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _ = rw.Flush() + }) +} + +func TestMountedStandaloneRouterPreservesSocketIOHijack(t *testing.T) { + router := chi.NewRouter() + router.Use(middleware.Recoverer) + router.Use(httpstream.CompressExcept(5, SkipMediaCompression)) + New(Dependencies{MediaStore: noopMediaStore{}, SocketIO: hijackingSocketIOServer{}}).Mount(router) + + server := httptest.NewUnstartedServer(router) + server.Start() + t.Cleanup(server.Close) + + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + req, err := http.NewRequest(http.MethodGet, server.URL+"/socket.io/", nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("socket.io upgrade: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("upgrade status = %d, want 101", resp.StatusCode) + } +} diff --git a/internal/httpstream/compress.go b/internal/httpstream/compress.go new file mode 100644 index 000000000..64e7a8e2a --- /dev/null +++ b/internal/httpstream/compress.go @@ -0,0 +1,22 @@ +package httpstream + +import ( + "net/http" + + "github.com/go-chi/chi/v5/middleware" +) + +// CompressExcept mounts chi's compressor but leaves the ResponseWriter +// untouched when skip reports true, preserving io.ReaderFrom on bulk routes. +func CompressExcept(level int, skip func(*http.Request) bool, types ...string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + compressed := middleware.Compress(level, types...)(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if skip != nil && skip(r) { + next.ServeHTTP(w, r) + return + } + compressed.ServeHTTP(w, r) + }) + } +} diff --git a/internal/httpstream/readfrom.go b/internal/httpstream/readfrom.go new file mode 100644 index 000000000..fea3dd075 --- /dev/null +++ b/internal/httpstream/readfrom.go @@ -0,0 +1,49 @@ +package httpstream + +import ( + "io" + "net/http" +) + +// ReadFromChunkDefault bounds one zero-copy slice so a wrapper's byte counters +// and write deadlines stay current during a multi-gigabyte transfer. +const ReadFromChunkDefault int64 = 4 << 20 + +// ReaderFromOf reports w's own io.ReaderFrom, by direct assertion only. It +// deliberately does not traverse Unwrap: skipping an intermediate wrapper +// would bypass that wrapper's byte accounting. +func ReaderFromOf(w http.ResponseWriter) (io.ReaderFrom, bool) { + rf, ok := w.(io.ReaderFrom) + return rf, ok +} + +// CopyChunked drives rf.ReadFrom in slices of chunk bytes, calling record after +// each slice. A non-positive chunk performs a single unbounded transfer. +func CopyChunked(rf io.ReaderFrom, src io.Reader, chunk int64, record func(n int64, err error)) (int64, error) { + if chunk <= 0 { + n, err := rf.ReadFrom(src) + if record != nil { + record(n, err) + } + return n, err + } + + var total int64 + for { + n, err := rf.ReadFrom(io.LimitReader(src, chunk)) + total += n + if record != nil { + record(n, err) + } + if err != nil { + return total, err + } + if n < chunk { + return total, nil + } + } +} + +// WriterOnly hides every method except Write, so io.Copy cannot rediscover the +// caller's own ReadFrom and recurse into it. +func WriterOnly(w io.Writer) io.Writer { return struct{ io.Writer }{w} } diff --git a/internal/httpstream/readfrom_bench_test.go b/internal/httpstream/readfrom_bench_test.go new file mode 100644 index 000000000..b9bcec93f --- /dev/null +++ b/internal/httpstream/readfrom_bench_test.go @@ -0,0 +1,146 @@ +package httpstream + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +// The benchmarks below cover the three shapes P0a's writer changes touch, as +// required by the design's "P0 is not zero-risk" note: a large sequential body +// (direct play), a progressively written body (remux), and many small requests +// (high-RPS HLS). They exist to catch a regression in the wrapper chain, not to +// produce an absolute throughput number — run them before and after a change to +// the writer chain and compare. + +func benchMediaFile(b *testing.B, size int) string { + b.Helper() + path := filepath.Join(b.TempDir(), "bench.bin") + f, err := os.Create(path) + if err != nil { + b.Fatal(err) + } + defer func() { _ = f.Close() }() + if err := f.Truncate(int64(size)); err != nil { + b.Fatal(err) + } + return path +} + +// BenchmarkDirectPlayReadFrom measures a whole-file transfer through the rolling +// deadline writer — the direct-play shape, and the one that depends on the +// io.ReaderFrom fast path surviving the wrapper chain. +func BenchmarkDirectPlayReadFrom(b *testing.B) { + const size = 32 << 20 + path := benchMediaFile(b, size) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f, err := os.Open(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer func() { _ = f.Close() }() + stat, err := f.Stat() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + http.ServeContent(NewRollingDeadlineWriter(w), r, "bench.bin", stat.ModTime(), f) + })) + defer srv.Close() + + client := srv.Client() + b.SetBytes(size) + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := client.Get(srv.URL) + if err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + b.Fatal(err) + } + _ = resp.Body.Close() + } +} + +// BenchmarkRemuxProgressiveWrite measures a body written incrementally and +// flushed, which is the remux shape: it never uses ReadFrom, so it isolates the +// per-Write overhead the wrapper chain adds. +func BenchmarkRemuxProgressiveWrite(b *testing.B) { + const ( + chunk = 64 << 10 + chunks = 512 // 32 MiB + ) + buf := make([]byte, chunk) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + sw := NewRollingDeadlineWriter(w) + sw.WriteHeader(http.StatusOK) + for i := 0; i < chunks; i++ { + if _, err := sw.Write(buf); err != nil { + return + } + } + sw.Flush() + })) + defer srv.Close() + + client := srv.Client() + b.SetBytes(chunk * chunks) + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := client.Get(srv.URL) + if err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + b.Fatal(err) + } + _ = resp.Body.Close() + } +} + +// BenchmarkHLSSegmentRPS measures many small segment-sized responses, the shape +// where per-request wrapper allocation dominates and where a regression would +// show up as increased allocs/op rather than reduced throughput. +func BenchmarkHLSSegmentRPS(b *testing.B) { + const segment = 512 << 10 + path := benchMediaFile(b, segment) + modTime := time.Now() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f, err := os.Open(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer func() { _ = f.Close() }() + http.ServeContent(NewRollingDeadlineWriter(w), r, "000.ts", modTime, f) + })) + defer srv.Close() + + client := srv.Client() + b.SetBytes(segment) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + resp, err := client.Get(srv.URL) + if err != nil { + b.Error(err) + return + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + b.Error(err) + return + } + _ = resp.Body.Close() + } + }) +} diff --git a/internal/httpstream/readfrom_deadline_test.go b/internal/httpstream/readfrom_deadline_test.go new file mode 100644 index 000000000..cb632b42c --- /dev/null +++ b/internal/httpstream/readfrom_deadline_test.go @@ -0,0 +1,152 @@ +package httpstream + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// pacedReader delivers src in fixed-size pieces with a pause between each, so a +// transfer takes a predictable wall-clock time regardless of socket buffering. +// It models a slow disk or a rate-limited upstream, which is what makes a single +// zero-copy slice long-lived. +type pacedReader struct { + remaining int64 + piece int64 + pause time.Duration +} + +func (r *pacedReader) Read(p []byte) (int, error) { + if r.remaining <= 0 { + return 0, io.EOF + } + time.Sleep(r.pause) + n := r.piece + if n > int64(len(p)) { + n = int64(len(p)) + } + if n > r.remaining { + n = r.remaining + } + r.remaining -= n + return int(n), nil +} + +// sliceDuration is how long one readFromChunk-sized slice takes to be produced +// by the pacedReader configured below. The tests derive their stall windows from +// it so they stay correct if readFromChunk changes. +const ( + testPiece = 64 << 10 + testPiecePause = 2 * time.Millisecond +) + +func sliceDuration() time.Duration { + return time.Duration(readFromChunk/testPiece) * testPiecePause +} + +// TestReadFromRollsDeadlineBetweenSlices is the regression test for the reap of +// healthy-but-slow streams. The write deadline is an absolute time, so any write +// attempted after it fails immediately: a transfer that outlives the stall window +// survives only because the deadline is pushed forward *between* slices. Before +// the slice was reduced to ReadFromChunkDefault, a single 64 MiB slice at a +// modest rate outlasted the whole window and the stream was reaped mid-transfer +// despite making continuous progress. +func TestReadFromRollsDeadlineBetweenSlices(t *testing.T) { + slice := sliceDuration() + // Window comfortably exceeds one slice but is far shorter than the whole + // transfer, so only per-slice bumping can carry it to completion. + window := slice * 3 + total := readFromChunk * 3 + + done := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + sw := newRollingDeadlineWriter(w, window, 0 /* bump every slice */) + sw.WriteHeader(http.StatusOK) + _, err := sw.ReadFrom(&pacedReader{remaining: total, piece: testPiece, pause: testPiecePause}) + done <- err + })) + srv.Config.WriteTimeout = 0 // isolate: only the rolling deadline may reap + srv.Start() + defer srv.Close() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + n, err := io.Copy(io.Discard, resp.Body) + if err != nil { + t.Fatalf("slow but continuously progressing stream died after %d/%d bytes: %v", n, total, err) + } + if n != total { + t.Fatalf("short body: got %d bytes, want %d", n, total) + } + if handlerErr := <-done; handlerErr != nil { + t.Fatalf("handler ReadFrom returned %v; a steadily progressing stream must not be reaped", handlerErr) + } +} + +// TestOversizedReadFromSliceIsReaped pins down *why* the slice size matters: with +// a slice long enough to outlast the stall window, the deadline set before it +// expires part-way through and the transfer dies even though it never stopped +// making progress. This is the behavior the 64 MiB default produced, and it must +// stay reproducible so nobody restores a large slice without noticing. +func TestOversizedReadFromSliceIsReaped(t *testing.T) { + slice := sliceDuration() + window := slice * 3 + oversized := readFromChunk * 8 // one slice ≈ 8x slice duration >> window + + done := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + sw := newRollingDeadlineWriter(w, window, 0) + sw.WriteHeader(http.StatusOK) + rf, ok := ReaderFromOf(sw.w) + if !ok { + done <- nil + t.Error("test server ResponseWriter does not implement io.ReaderFrom") + return + } + // Deliberately drive a single oversized slice: no bump can happen inside it. + _, err := CopyChunked(rf, &pacedReader{remaining: oversized, piece: testPiece, pause: testPiecePause}, oversized, nil) + done <- err + })) + srv.Config.WriteTimeout = 0 + srv.Start() + defer srv.Close() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, resp.Body) + + select { + case err := <-done: + if err == nil { + t.Fatal("a slice longer than the stall window completed; the deadline is no longer enforced mid-slice") + } + if !isTimeoutError(err) { + t.Fatalf("oversized slice failed with %v, want a deadline timeout", err) + } + case <-time.After(30 * time.Second): + t.Fatal("oversized slice never returned") + } +} + +// TestReadFromChunkAllowsSlowClients guards the constant itself. The reap +// threshold is readFromChunk / DefaultStallWindow: any client sustaining less +// than that is killed mid-slice despite healthy progress. 64 MiB over the 180s +// default worked out to ~3 Mbit/s, which reaps ordinary mobile connections. +func TestReadFromChunkAllowsSlowClients(t *testing.T) { + const maxAcceptableFloorBitsPerSec = 256 << 10 // 256 kbit/s + + floor := float64(readFromChunk) * 8 / DefaultStallWindow.Seconds() + if floor > maxAcceptableFloorBitsPerSec { + t.Fatalf("readFromChunk %d over a %s window reaps clients below %.0f bit/s; keep it under %d bit/s", + readFromChunk, DefaultStallWindow, floor, maxAcceptableFloorBitsPerSec) + } +} diff --git a/internal/httpstream/readfrom_test.go b/internal/httpstream/readfrom_test.go new file mode 100644 index 000000000..ff130190d --- /dev/null +++ b/internal/httpstream/readfrom_test.go @@ -0,0 +1,69 @@ +package httpstream + +import ( + "bytes" + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +type readerFromResponseWriter struct { + bytes.Buffer + called int + header http.Header +} + +func (w *readerFromResponseWriter) Header() http.Header { return w.header } +func (w *readerFromResponseWriter) WriteHeader(int) {} +func (w *readerFromResponseWriter) ReadFrom(r io.Reader) (int64, error) { + w.called++ + return io.Copy(&w.Buffer, r) +} + +func TestCopyChunkedUsesReaderFromPerSlice(t *testing.T) { + w := &readerFromResponseWriter{header: make(http.Header)} + rf, ok := ReaderFromOf(w) + if !ok { + t.Fatal("ReaderFromOf did not report direct implementation") + } + var recorded int64 + n, err := CopyChunked(rf, bytes.NewReader(make([]byte, 10)), 4, func(n int64, _ error) { recorded += n }) + if err != nil || n != 10 || recorded != 10 || w.called != 3 { + t.Fatalf("CopyChunked = n=%d err=%v recorded=%d calls=%d", n, err, recorded, w.called) + } +} + +func TestWriterOnlyHidesReaderFrom(t *testing.T) { + w := &readerFromResponseWriter{header: make(http.Header)} + if _, ok := WriterOnly(w).(io.ReaderFrom); ok { + t.Fatal("WriterOnly exposed io.ReaderFrom") + } +} + +func TestCompressExceptBypassPreservesReaderFromAndKeptRouteCompresses(t *testing.T) { + handlerSawReaderFrom := false + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, handlerSawReaderFrom = w.(io.ReaderFrom) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(bytes.Repeat([]byte("x"), 2048)) + }) + handler := CompressExcept(gzip.BestSpeed, func(r *http.Request) bool { return r.URL.Path == "/media" })(next) + + mediaWriter := &readerFromResponseWriter{header: make(http.Header)} + mediaReq := httptest.NewRequest(http.MethodGet, "/media", nil) + mediaReq.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(mediaWriter, mediaReq) + if !handlerSawReaderFrom || mediaWriter.Header().Get("Content-Encoding") != "" || mediaWriter.Len() != 2048 { + t.Fatalf("bypass: saw ReaderFrom=%v encoding=%q bytes=%d", handlerSawReaderFrom, mediaWriter.Header().Get("Content-Encoding"), mediaWriter.Len()) + } + + recorder := httptest.NewRecorder() + jsonReq := httptest.NewRequest(http.MethodGet, "/json", nil) + jsonReq.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(recorder, jsonReq) + if recorder.Header().Get("Content-Encoding") != "gzip" || recorder.Header().Get("Vary") != "Accept-Encoding" { + t.Fatalf("kept route headers: encoding=%q vary=%q", recorder.Header().Get("Content-Encoding"), recorder.Header().Get("Vary")) + } +} diff --git a/internal/httpstream/rolling_deadline.go b/internal/httpstream/rolling_deadline.go index f406aa4a0..399e97d02 100644 --- a/internal/httpstream/rolling_deadline.go +++ b/internal/httpstream/rolling_deadline.go @@ -32,9 +32,10 @@ const ( // SetWriteDeadline per step rather than one per 32 KB chunk. bumpStep = 15 * time.Second - // readFromChunk bounds each ReadFrom slice so the deadline keeps rolling - // during zero-copy (sendfile) transfers of large files. - readFromChunk int64 = 64 << 20 + // readFromChunk bounds each ReadFrom slice so the deadline keeps rolling. + // At the default 180s window, 4 MiB permits steady clients down to roughly + // 186 kbit/s without expiring mid-slice. + readFromChunk int64 = ReadFromChunkDefault ) // StreamOutcome classifies how a streaming response ended. @@ -131,28 +132,19 @@ func (s *RollingDeadlineWriter) Write(p []byte) (int, error) { // (sendfile for *os.File bodies, as used by http.ServeContent) while still // rolling the deadline between bounded slices. func (s *RollingDeadlineWriter) ReadFrom(r io.Reader) (int64, error) { - rf, ok := s.w.(io.ReaderFrom) + rf, ok := ReaderFromOf(s.w) if !ok { - // writerOnly hides this method so io.Copy doesn't recurse into it. + // WriterOnly hides this method so io.Copy doesn't recurse into it. s.bump() - return io.Copy(writerOnly{s}, r) + return io.Copy(WriterOnly(s), r) } - var total int64 - for { + if s.statusCode == 0 { + s.statusCode = http.StatusOK + } + return CopyChunked(rf, r, readFromChunk, func(n int64, err error) { s.bump() - if s.statusCode == 0 { - s.statusCode = http.StatusOK - } - n, err := rf.ReadFrom(io.LimitReader(r, readFromChunk)) - total += n s.recordWrite(n, err) - if err != nil { - return total, err - } - if n < readFromChunk { - return total, nil - } - } + }) } func (s *RollingDeadlineWriter) Flush() { @@ -203,5 +195,3 @@ func isTimeoutError(err error) bool { var netErr net.Error return errors.As(err, &netErr) && netErr.Timeout() } - -type writerOnly struct{ io.Writer } diff --git a/internal/jellycompat/image_proxy_tags.go b/internal/jellycompat/image_proxy_tags.go index b91eb1092..bdf411126 100644 --- a/internal/jellycompat/image_proxy_tags.go +++ b/internal/jellycompat/image_proxy_tags.go @@ -1,10 +1,16 @@ package jellycompat import ( + "bufio" "bytes" "encoding/json" + "fmt" + "io" + "net" "net/http" "strings" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) func compatImageProxyTagVariantMiddleware(codec *ResourceIDCodec) func(http.Handler) http.Handler { @@ -53,6 +59,39 @@ func (w *compatImageProxyTagResponseWriter) Write(p []byte) (int, error) { return w.body.Write(p) } +func (w *compatImageProxyTagResponseWriter) ReadFrom(src io.Reader) (int64, error) { + if w.passthrough { + return w.readFromPassthrough(src) + } + if w.status == 0 { + w.status = http.StatusOK + } + if isJSONResponse(w.Header().Get("Content-Type")) { + return io.Copy(&w.body, src) + } + w.passthrough = true + w.ResponseWriter.WriteHeader(w.status) + return w.readFromPassthrough(src) +} + +func (w *compatImageProxyTagResponseWriter) readFromPassthrough(src io.Reader) (int64, error) { + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, 0, nil) +} + +func (w *compatImageProxyTagResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +func (w *compatImageProxyTagResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj, ok := w.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, fmt.Errorf("underlying ResponseWriter does not implement http.Hijacker") + } + return hj.Hijack() +} + // Flush implements http.Flusher for the passthrough (non-JSON) path only. // While buffering a JSON body for tag rewriting there is nothing downstream // to flush, and flushing the inner writer would commit headers before diff --git a/internal/jellycompat/logging.go b/internal/jellycompat/logging.go index 28136679d..20355dd48 100644 --- a/internal/jellycompat/logging.go +++ b/internal/jellycompat/logging.go @@ -16,6 +16,8 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) type loggingResponseWriter struct { @@ -35,6 +37,17 @@ func (w *loggingResponseWriter) Write(b []byte) (int, error) { return w.ResponseWriter.Write(b) } +func (w *loggingResponseWriter) ReadFrom(src io.Reader) (int64, error) { + if w.status == 0 { + w.status = http.StatusOK + } + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, 0, nil) +} + func (w *loggingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := w.ResponseWriter.(http.Hijacker); ok { return hj.Hijack() @@ -141,6 +154,31 @@ func (w *debugResponseWriter) Write(b []byte) (int, error) { return w.ResponseWriter.Write(b) } +func (w *debugResponseWriter) ReadFrom(src io.Reader) (int64, error) { + ct := strings.TrimSpace(w.Header().Get("Content-Type")) + if !w.detected && ct == "" { + return io.Copy(httpstream.WriterOnly(w), src) + } + if !w.detected { + w.contentType = ct + w.skipBody = !isTextualContentType(ct) + w.detected = true + } + if !w.skipBody { + return io.Copy(httpstream.WriterOnly(w), src) + } + if w.status == 0 { + w.status = http.StatusOK + } + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + w.totalBytes += int(n) + }) +} + func (w *debugResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := w.ResponseWriter.(http.Hijacker); ok { return hj.Hijack() diff --git a/internal/jellycompat/readfrom_test.go b/internal/jellycompat/readfrom_test.go new file mode 100644 index 000000000..9995f9a5d --- /dev/null +++ b/internal/jellycompat/readfrom_test.go @@ -0,0 +1,83 @@ +package jellycompat + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type readerFromSpy struct { + bytes.Buffer + header http.Header + calls int +} + +func (w *readerFromSpy) Header() http.Header { return w.header } +func (w *readerFromSpy) WriteHeader(int) {} +func (w *readerFromSpy) ReadFrom(r io.Reader) (int64, error) { + w.calls++ + return io.Copy(&w.Buffer, r) +} + +func TestResponseWritersPreserveReaderFrom(t *testing.T) { + for _, tt := range []struct { + name string + new func(*readerFromSpy) io.ReaderFrom + }{ + {"request log", func(spy *readerFromSpy) io.ReaderFrom { return &loggingResponseWriter{ResponseWriter: spy} }}, + {"debug media", func(spy *readerFromSpy) io.ReaderFrom { + spy.header.Set("Content-Type", "video/mp4") + return &debugResponseWriter{ResponseWriter: spy} + }}, + {"image proxy passthrough", func(spy *readerFromSpy) io.ReaderFrom { + spy.header.Set("Content-Type", "video/mp4") + return &compatImageProxyTagResponseWriter{ResponseWriter: spy} + }}, + } { + t.Run(tt.name, func(t *testing.T) { + spy := &readerFromSpy{header: make(http.Header)} + n, err := tt.new(spy).ReadFrom(bytes.NewBufferString("media")) + if err != nil || n != 5 || spy.calls != 1 || spy.String() != "media" { + t.Fatalf("ReadFrom = n=%d err=%v calls=%d body=%q", n, err, spy.calls, spy.String()) + } + }) + } +} + +func TestDebugResponseWriterReadFromKeepsTextCapture(t *testing.T) { + spy := &readerFromSpy{header: make(http.Header)} + spy.header.Set("Content-Type", "application/json") + w := &debugResponseWriter{ResponseWriter: spy} + _, err := w.ReadFrom(bytes.NewBufferString(`{"ok":true}`)) + if err != nil || w.body.String() != `{"ok":true}` || w.totalBytes != 11 { + t.Fatalf("text capture = body=%q bytes=%d err=%v", w.body.String(), w.totalBytes, err) + } +} + +func TestDebugLogMiddlewareReadFromLogsTextAndMedia(t *testing.T) { + for _, tt := range []struct { + name, contentType, body, want string + }{ + {"json", "application/json", `{"ok":true}`, `Response (content-type=application/json, 11 bytes)`}, + {"media", "video/mp4", "media", `Response: [binary content-type=video/mp4 bytes=5]`}, + } { + t.Run(tt.name, func(t *testing.T) { + var log bytes.Buffer + h := newDebugLogMiddleware(&log, "")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", tt.contentType) + _, _ = io.Copy(w, bytes.NewBufferString(tt.body)) + })) + spy := &readerFromSpy{header: make(http.Header)} + h.ServeHTTP(spy, httptest.NewRequest(http.MethodGet, "/", nil)) + if !strings.Contains(log.String(), tt.want) { + t.Fatalf("debug log = %q, want %q", log.String(), tt.want) + } + if tt.name == "media" && strings.Contains(log.String(), "media\n") { + t.Fatalf("binary body leaked into debug log: %q", log.String()) + } + }) + } +} diff --git a/internal/jellycompat/router.go b/internal/jellycompat/router.go index 577bdbd7d..2937e0ae0 100644 --- a/internal/jellycompat/router.go +++ b/internal/jellycompat/router.go @@ -15,6 +15,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/httpstream" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/recommendations" "github.com/Silo-Server/silo-server/internal/sections" @@ -42,7 +43,7 @@ func NewRouter(deps Dependencies) chi.Router { MaxAge: 86400, })) r.Use(normalizeCompatPathMiddleware) - r.Use(middleware.Compress(5, "application/json")) + r.Use(httpstream.CompressExcept(5, skipCompatMediaCompression, "application/json")) if debugPath := os.Getenv("JELLYCOMPAT_DEBUG_LOG"); debugPath != "" { rotator := &lumberjack.Logger{ Filename: debugPath, @@ -273,6 +274,28 @@ func NewRouter(deps Dependencies) chi.Router { return r } +func skipCompatMediaCompression(r *http.Request) bool { + const ( + videosSegment = "Videos" + hlsSegment = "hls" + hlsManifest = "stream.m3u8" + ) + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + p := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + switch { + case len(p) == 3 && p[0] == videosSegment && p[1] != "" && (p[2] == "stream" || strings.HasPrefix(p[2], "stream.")): + return p[2] == "stream" || len(strings.TrimPrefix(p[2], "stream.")) > 0 + case len(p) == 5 && p[0] == videosSegment && p[1] != "" && p[2] == hlsSegment && p[3] != "" && p[4] != "": + return p[4] != hlsManifest && strings.Contains(p[4], ".") + case len(p) == 3 && p[0] == "Items" && p[1] != "" && p[2] == "Download": + return true + default: + return false + } +} + func withDefaults(deps Dependencies) Dependencies { if deps.Now == nil { deps.Now = timeNow diff --git a/internal/jellycompat/router_compression_test.go b/internal/jellycompat/router_compression_test.go new file mode 100644 index 000000000..c0370b17d --- /dev/null +++ b/internal/jellycompat/router_compression_test.go @@ -0,0 +1,32 @@ +package jellycompat + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestSkipCompatMediaCompression(t *testing.T) { + tests := []struct { + method, path string + want bool + }{ + {http.MethodGet, "/Videos/i1/stream", true}, + {http.MethodHead, "/Videos/i1/stream.mkv", true}, + {http.MethodGet, "/Videos/i1/hls/p1/000.ts", true}, + {http.MethodGet, "/Items/i1/Download", true}, + {http.MethodGet, "/Videos/i1/master.m3u8", false}, + {http.MethodGet, "/Videos/i1/hls/p1/stream.m3u8", false}, + {http.MethodGet, "/Videos/i1/stream/subtitles/1", false}, + {http.MethodGet, "/Videos/i1/stream/", false}, + {http.MethodPost, "/Videos/i1/stream", false}, + {http.MethodGet, "/videos/i1/stream", false}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.path, func(t *testing.T) { + if got := skipCompatMediaCompression(httptest.NewRequest(tt.method, tt.path, nil)); got != tt.want { + t.Fatalf("skipCompatMediaCompression = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/jellycompat/router_socket_test.go b/internal/jellycompat/router_socket_test.go new file mode 100644 index 000000000..ef8a40fd3 --- /dev/null +++ b/internal/jellycompat/router_socket_test.go @@ -0,0 +1,234 @@ +package jellycompat + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/models" +) + +const compatSocketToken = "compat-socket-token" + +func TestMountedCompatRouterPreservesMediaHTTPAndCompression(t *testing.T) { + server, client, mediaURL, _ := newCompatSocketServer(t) + + full := compatSocketRequest(t, client, http.MethodGet, mediaURL, nil) + etag := full.Header.Get("ETag") + assertCompatSocketResponse(t, full, http.StatusOK, "0123456789abcdefghijklmnopqrstuvwxyz") + _ = full.Body.Close() + if etag == "" { + t.Fatal("media response omitted ETag") + } + head := compatSocketRequest(t, client, http.MethodHead, mediaURL, nil) + assertCompatSocketResponse(t, head, http.StatusOK, "") + _ = head.Body.Close() + singleRange := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5"}) + assertCompatSocketResponse(t, singleRange, http.StatusPartialContent, "2345") + _ = singleRange.Body.Close() + multiRange := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=0-1,4-6"}) + assertCompatMultiRange(t, multiRange) + _ = multiRange.Body.Close() + notModified := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"If-None-Match": etag}) + assertCompatSocketResponse(t, notModified, http.StatusNotModified, "") + _ = notModified.Body.Close() + ifRangeHit := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5", "If-Range": etag}) + assertCompatSocketResponse(t, ifRangeHit, http.StatusPartialContent, "2345") + _ = ifRangeHit.Body.Close() + ifRangeMiss := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Range": "bytes=2-5", "If-Range": `"stale"`}) + assertCompatSocketResponse(t, ifRangeMiss, http.StatusOK, "0123456789abcdefghijklmnopqrstuvwxyz") + _ = ifRangeMiss.Body.Close() + + gzipMedia := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{"Accept-Encoding": "gzip"}) + if encoding := gzipMedia.Header.Get("Content-Encoding"); encoding != "" { + t.Fatalf("media Content-Encoding = %q, want empty", encoding) + } + assertCompatSocketResponse(t, gzipMedia, http.StatusOK, "0123456789abcdefghijklmnopqrstuvwxyz") + _ = gzipMedia.Body.Close() + + jsonResp := compatSocketRequest(t, client, http.MethodGet, server.URL+"/System/Info/Public", map[string]string{"Accept-Encoding": "gzip"}) + defer func() { _ = jsonResp.Body.Close() }() + if encoding := jsonResp.Header.Get("Content-Encoding"); encoding != "gzip" { + t.Fatalf("JSON Content-Encoding = %q, want gzip", encoding) + } + if vary := jsonResp.Header.Values("Vary"); !compatHeaderContains(vary, "Accept-Encoding") { + t.Fatalf("JSON Vary = %q, want Accept-Encoding", vary) + } + zr, err := gzip.NewReader(jsonResp.Body) + if err != nil { + t.Fatalf("gzip.NewReader: %v", err) + } + defer func() { _ = zr.Close() }() + body, err := io.ReadAll(zr) + if err != nil || !bytes.Contains(body, []byte(`"ProductName":"Jellyfin Server"`)) { + t.Fatalf("compressed JSON body invalid: body=%q err=%v", body, err) + } +} + +func TestMountedCompatRouterImageProxyUARewritesJSONButStreamsVideo(t *testing.T) { + _, client, mediaURL, itemURL := newCompatSocketServer(t) + const userAgent = "Infuse-Direct/8.4.6" + + itemResp := compatSocketRequest(t, client, http.MethodGet, itemURL, map[string]string{ + "User-Agent": userAgent, + "X-Emby-Token": compatSocketToken, + }) + defer func() { _ = itemResp.Body.Close() }() + if itemResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(itemResp.Body) + t.Fatalf("item status = %d, body=%s", itemResp.StatusCode, body) + } + var item baseItemDTO + if err := json.NewDecoder(itemResp.Body).Decode(&item); err != nil { + t.Fatalf("decode item: %v", err) + } + if primary := item.ImageTags["Primary"]; primary == "" || !strings.HasSuffix(primary, compatImageProxyTagSuffix) { + t.Fatalf("rewritten primary image tag = %q, want %q suffix", primary, compatImageProxyTagSuffix) + } + + videoResp := compatSocketRequest(t, client, http.MethodGet, mediaURL, map[string]string{ + "User-Agent": userAgent, + "Range": "bytes=3-7", + }) + if got := videoResp.Header.Get("Content-Range"); got != "bytes 3-7/36" { + t.Fatalf("video Content-Range = %q, want bytes 3-7/36", got) + } + assertCompatSocketResponse(t, videoResp, http.StatusPartialContent, "34567") + _ = videoResp.Body.Close() +} + +func newCompatSocketServer(t *testing.T) (*httptest.Server, *http.Client, string, string) { + t.Helper() + dir := t.TempDir() + filePath := filepath.Join(dir, "movie.mp4") + if err := os.WriteFile(filePath, []byte("0123456789abcdefghijklmnopqrstuvwxyz"), 0o644); err != nil { + t.Fatalf("write media: %v", err) + } + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatalf("LoadFromDB: %v", err) + } + store := NewSessionStore(time.Hour, nil) + if err := store.Put(Session{Token: compatSocketToken, StreamAppUserID: 1, ProfileID: "profile-1"}); err != nil { + t.Fatalf("put compat session: %v", err) + } + codec := NewResourceIDCodec() + contentID := "socket-movie" + detail := &upstreamItemDetail{ + ContentID: contentID, + Type: "movie", + Title: "Socket Movie", + PosterURL: "https://images.invalid/poster.jpg", + Versions: []catalog.FileVersion{{ + FileID: 42, + FilePath: filePath, + Container: "mp4", + Duration: 3600, + FileSize: 36, + AddedAt: time.Now(), + }}, + } + router := NewRouter(Dependencies{ + Config: cfg, + SessionStore: store, + IDCodec: codec, + ContentService: &stubContentService{detail: detail}, + FileResolver: testCompatFileResolver{file: &models.MediaFile{ID: 42, FilePath: filePath}}, + SessionMgr: &testCompatSessionManager{}, + }) + server := httptest.NewUnstartedServer(router) + server.Start() + t.Cleanup(server.Close) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + itemID := codec.EncodeStringID(EncodedIDItem, contentID) + mediaURL := server.URL + "/Videos/" + itemID + "/stream.mp4?static=true&api_key=" + compatSocketToken + itemURL := server.URL + "/Items/" + itemID + return server, client, mediaURL, itemURL +} + +func compatSocketRequest(t *testing.T, client *http.Client, method, url string, headers map[string]string) *http.Response { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + for key, value := range headers { + req.Header.Set(key, value) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("request %s %s: %v", method, url, err) + } + return resp +} + +func assertCompatSocketResponse(t *testing.T, resp *http.Response, wantStatus int, wantBody string) { + t.Helper() + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if resp.StatusCode != wantStatus || string(body) != wantBody { + t.Fatalf("status/body = %d, %q; want %d, %q", resp.StatusCode, body, wantStatus, wantBody) + } + if encoding := resp.Header.Get("Content-Encoding"); encoding != "" { + t.Fatalf("media Content-Encoding = %q, want empty", encoding) + } +} + +func assertCompatMultiRange(t *testing.T, resp *http.Response) { + t.Helper() + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusPartialContent { + t.Fatalf("multi-range status = %d, want 206", resp.StatusCode) + } + mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil || mediaType != "multipart/byteranges" { + t.Fatalf("multi-range Content-Type = %q: %v", resp.Header.Get("Content-Type"), err) + } + mr := multipart.NewReader(resp.Body, params["boundary"]) + var bodies []string + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("read multipart range: %v", err) + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("read multipart body: %v", err) + } + bodies = append(bodies, string(body)) + } + if strings.Join(bodies, ",") != "01,456" { + t.Fatalf("multi-range bodies = %q, want [01 456]", bodies) + } +} + +func compatHeaderContains(values []string, want string) bool { + for _, value := range values { + for _, part := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(part), want) { + return true + } + } + } + return false +} diff --git a/internal/proxy/egress.go b/internal/proxy/egress.go index 1ae0d9d16..b38072e24 100644 --- a/internal/proxy/egress.go +++ b/internal/proxy/egress.go @@ -1,9 +1,12 @@ package proxy import ( + "io" "net/http" "sync" "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) // meterWindowSeconds is the averaging window for the egress rate. HLS clients @@ -57,9 +60,8 @@ func (m *egressMeter) RateKbps() int { return int(total * 8 / 1000 / meterWindowSeconds) } -// meteredResponseWriter counts every byte written to the client. -// Embedding the interface intentionally hides optimizations like -// io.ReaderFrom so all writes flow through Write. +// meteredResponseWriter counts every byte written to the client. Chunked +// ReaderFrom delegation preserves both sendfile and the rolling rate window. type meteredResponseWriter struct { http.ResponseWriter meter *egressMeter @@ -71,6 +73,16 @@ func (w *meteredResponseWriter) Write(b []byte) (int, error) { return n, err } +func (w *meteredResponseWriter) ReadFrom(src io.Reader) (int64, error) { + rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) + if !ok { + return io.Copy(httpstream.WriterOnly(w), src) + } + return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + w.meter.Add(n) + }) +} + func (w *meteredResponseWriter) Flush() { if f, ok := w.ResponseWriter.(http.Flusher); ok { f.Flush() diff --git a/internal/proxy/egress_readfrom_test.go b/internal/proxy/egress_readfrom_test.go new file mode 100644 index 000000000..59d0292fe --- /dev/null +++ b/internal/proxy/egress_readfrom_test.go @@ -0,0 +1,29 @@ +package proxy + +import ( + "bytes" + "io" + "net/http" + "testing" +) + +type egressReaderFromSpy struct{ bytes.Buffer } + +func (w *egressReaderFromSpy) Header() http.Header { return make(http.Header) } +func (w *egressReaderFromSpy) WriteHeader(int) {} +func (w *egressReaderFromSpy) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(&w.Buffer, r) +} + +func TestMeteredResponseWriterReadFromCountsBytes(t *testing.T) { + spy := &egressReaderFromSpy{} + meter := newEgressMeter() + w := &meteredResponseWriter{ResponseWriter: spy, meter: meter} + n, err := w.ReadFrom(bytes.NewReader(make([]byte, 8<<20))) + if err != nil || n != 8<<20 || len(spy.Bytes()) != 8<<20 { + t.Fatalf("ReadFrom = n=%d err=%v body=%d", n, err, len(spy.Bytes())) + } + if got := meter.RateKbps(); got <= 0 { + t.Fatalf("meter rate = %d, want > 0", got) + } +} From a9d54b6b2c8b27c21cadf3662c02f724d9aa01d0 Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:16:47 +0000 Subject: [PATCH 05/44] feat(streamtelemetry): add local shadow telemetry for native media routes P0b of the stream telemetry and enforcement design. Adds process-local, observation-only telemetry behind SILO_STREAM_TELEMETRY_ENABLED (default off). Nothing is rejected, delayed, throttled or cut, and neither PostgreSQL nor Redis is written. internal/streamtelemetry carries the three-level model from the design's 2.2: Observation per in-flight request, logicalSession keyed by canonical session id, and transfer for download-class pours. Every observation folds its final byte total in on release under the session lock, so a short HLS transfer that lives and dies between sweeps is still counted and can neither double-count nor lose growth. Retention, session/transfer/observation counts and every per-session set are bounded; saturation serves through and is reported through Truncated, monotonic dropped counters and a rate-limited warning. Observe counts bytes but creates no logical activity. The handler calls Attach only after it has loaded and authorized the session, because ownership is established inside the handler and the transcode serve routes deliberately allow an unauthenticated caller. 401/403/404 therefore create nothing, and never-attached bytes land in the unattributed counters. Media routes are declared as typed MediaRoute values for all five router families, each with a route-manifest test that walks the mounted router and fails the build on any route that is neither declared nor in that family's checked-in non-media allowlist. Only the native family is enrolled; the other four are classified and will be enrolled one at a time. observedWriter obeys the P0a writer-chain conformance rules, so sendfile, deadline traversal and the optional interfaces survive the extra wrapper. --- cmd/silo/main.go | 8 + internal/api/handlers/downloads.go | 27 +- internal/api/handlers/ebook_reader.go | 1 + internal/api/handlers/playback.go | 83 +++- internal/api/handlers/playback_realtime.go | 3 + internal/api/handlers/session_ws_test.go | 26 ++ internal/api/handlers/stream.go | 5 +- internal/api/handlers/streamtelemetry_test.go | 55 +++ internal/api/media_routes.go | 75 ++++ internal/api/media_routes_test.go | 79 ++++ internal/api/router.go | 40 +- internal/api/router_http2_test.go | 8 +- internal/api/router_socket_test.go | 35 +- internal/api/testdata/media_routes.txt | 358 +++++++++++++++ internal/audiobooks/abs/handler.go | 1 + internal/audiobooks/abs/media_routes.go | 35 ++ internal/audiobooks/abs/media_routes_test.go | 47 ++ .../audiobooks/abs/testdata/media_routes.txt | 368 ++++++++++++++++ internal/downloads/offline.go | 1 + internal/downloads/serve_observer.go | 25 ++ internal/downloads/service.go | 4 +- internal/httpstream/rolling_deadline.go | 15 +- internal/httpstream/rolling_deadline_test.go | 21 + internal/jellycompat/media_routes.go | 30 ++ internal/jellycompat/media_routes_test.go | 49 +++ internal/jellycompat/router.go | 1 + .../jellycompat/testdata/media_routes.txt | 226 ++++++++++ internal/proxy/media_routes.go | 29 ++ internal/proxy/media_routes_test.go | 51 +++ internal/proxy/server.go | 1 + internal/proxy/testdata/media_routes.txt | 32 ++ internal/streamtelemetry/benchmark_test.go | 96 +++++ internal/streamtelemetry/config.go | 103 +++++ internal/streamtelemetry/config_test.go | 50 +++ internal/streamtelemetry/doc.go | 17 + internal/streamtelemetry/identity.go | 77 ++++ internal/streamtelemetry/manifest.go | 79 ++++ internal/streamtelemetry/observation.go | 90 ++++ internal/streamtelemetry/registry.go | 408 ++++++++++++++++++ internal/streamtelemetry/registry_test.go | 275 ++++++++++++ internal/streamtelemetry/route.go | 132 ++++++ internal/streamtelemetry/route_test.go | 17 + internal/streamtelemetry/session.go | 190 ++++++++ internal/streamtelemetry/store.go | 32 ++ internal/streamtelemetry/view.go | 206 +++++++++ internal/streamtelemetry/writer.go | 116 +++++ internal/streamtelemetry/writer_test.go | 143 ++++++ internal/transcodenode/media_routes.go | 22 + internal/transcodenode/media_routes_test.go | 49 +++ internal/transcodenode/server.go | 1 + .../transcodenode/testdata/media_routes.txt | 28 ++ 51 files changed, 3815 insertions(+), 55 deletions(-) create mode 100644 internal/api/handlers/streamtelemetry_test.go create mode 100644 internal/api/media_routes.go create mode 100644 internal/api/media_routes_test.go create mode 100644 internal/api/testdata/media_routes.txt create mode 100644 internal/audiobooks/abs/media_routes.go create mode 100644 internal/audiobooks/abs/media_routes_test.go create mode 100644 internal/audiobooks/abs/testdata/media_routes.txt create mode 100644 internal/downloads/serve_observer.go create mode 100644 internal/jellycompat/media_routes.go create mode 100644 internal/jellycompat/media_routes_test.go create mode 100644 internal/jellycompat/testdata/media_routes.txt create mode 100644 internal/proxy/media_routes.go create mode 100644 internal/proxy/media_routes_test.go create mode 100644 internal/proxy/testdata/media_routes.txt create mode 100644 internal/streamtelemetry/benchmark_test.go create mode 100644 internal/streamtelemetry/config.go create mode 100644 internal/streamtelemetry/config_test.go create mode 100644 internal/streamtelemetry/doc.go create mode 100644 internal/streamtelemetry/identity.go create mode 100644 internal/streamtelemetry/manifest.go create mode 100644 internal/streamtelemetry/observation.go create mode 100644 internal/streamtelemetry/registry.go create mode 100644 internal/streamtelemetry/registry_test.go create mode 100644 internal/streamtelemetry/route.go create mode 100644 internal/streamtelemetry/route_test.go create mode 100644 internal/streamtelemetry/session.go create mode 100644 internal/streamtelemetry/store.go create mode 100644 internal/streamtelemetry/view.go create mode 100644 internal/streamtelemetry/writer.go create mode 100644 internal/streamtelemetry/writer_test.go create mode 100644 internal/transcodenode/media_routes.go create mode 100644 internal/transcodenode/media_routes_test.go create mode 100644 internal/transcodenode/testdata/media_routes.txt diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 8ebadeb12..1b65a8a77 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -99,6 +99,7 @@ import ( "github.com/Silo-Server/silo-server/internal/sections" "github.com/Silo-Server/silo-server/internal/server" "github.com/Silo-Server/silo-server/internal/settingscontract" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/taskmanager" taskrepository "github.com/Silo-Server/silo-server/internal/taskmanager/repository" @@ -690,6 +691,12 @@ func main() { appCtx, appCancel := context.WithCancel(ctx) defer appCancel() + var streamTelemetryRegistry *streamtelemetry.Registry + if mode == "" || mode == "integrated" || mode == "api" { + streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) + streamTelemetryRegistry = streamtelemetry.NewRegistry(streamTelemetryConfig, streamtelemetry.NewLocalStore(), slog.Default()) + streamTelemetryRegistry.Start(appCtx) + } restartReqCh := make(chan struct{}, 1) var restartRequested atomic.Bool @@ -859,6 +866,7 @@ func main() { BootstrapSensitiveValues: bootstrapSensitiveValues, RedisBootstrapAvailable: redisBootstrapAvailable, AppContext: appCtx, + StreamTelemetry: streamTelemetryRegistry, DB: pool, SecretCipher: dataCipher, EventBus: eventBus, diff --git a/internal/api/handlers/downloads.go b/internal/api/handlers/downloads.go index ce7fa5302..75f0977e1 100644 --- a/internal/api/handlers/downloads.go +++ b/internal/api/handlers/downloads.go @@ -458,6 +458,9 @@ func (h *DownloadHandler) handleDownloadFile(w http.ResponseWriter, r *http.Requ profileID, deviceID, _, _ := managedIdentity(r) filter := requestAccessFilter(r) + serveCtx := downloads.WithServeAuthorized(r.Context(), func(target downloads.FileTarget) { + attachTransfer(r.Context(), userID, profileID, target.MediaFileID) + }) if delegate && deviceID != "" { handled, err := h.redirectManagedDownload(r.Context(), w, r, userID, profileID, deviceID, id, filter) if err != nil { @@ -471,7 +474,7 @@ func (h *DownloadHandler) handleDownloadFile(w http.ResponseWriter, r *http.Requ // Full media downloads outlive the server's absolute WriteTimeout; roll // the write deadline with progress instead. sw := httpstream.NewRollingDeadlineWriter(w) - if err := h.svc.ServeFile(r.Context(), sw, r, userID, profileID, deviceID, id, filter); err != nil { + if err := h.svc.ServeFile(serveCtx, sw, r, userID, profileID, deviceID, id, filter); err != nil { if errors.Is(err, downloads.ErrResponseCommitted) { return } @@ -534,7 +537,10 @@ func (h *DownloadHandler) handleDirectDownload(w http.ResponseWriter, r *http.Re return } } - if err := h.svc.ServeDirect(r.Context(), w, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil { + serveCtx := downloads.WithServeAuthorized(r.Context(), func(target downloads.FileTarget) { + attachTransfer(r.Context(), userID, apimw.GetProfileID(r.Context()), target.MediaFileID) + }) + if err := h.svc.ServeDirect(serveCtx, w, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil { h.writeDownloadError(w, err) return } @@ -549,7 +555,11 @@ func (h *DownloadHandler) redirectDirectDownload(ctx context.Context, w http.Res if err != nil { return false, err } - return h.redirectToProxy(w, r, secret, target, userID, "") + handled, err := h.redirectToProxy(w, r, secret, target, userID, "") + if handled { + attachTransfer(ctx, userID, "", target.MediaFileID) + } + return handled, err } func (h *DownloadHandler) redirectManagedDownload(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) (bool, error) { @@ -561,7 +571,11 @@ func (h *DownloadHandler) redirectManagedDownload(ctx context.Context, w http.Re if err != nil { return false, err } - return h.redirectToProxy(w, r, secret, target, userID, profileID) + handled, err := h.redirectToProxy(w, r, secret, target, userID, profileID) + if handled { + attachTransfer(ctx, userID, profileID, target.MediaFileID) + } + return handled, err } func (h *DownloadHandler) proxyTarget() (downloadFileResolver, string, bool) { @@ -779,7 +793,10 @@ func (h *DownloadHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) return } ref := chi.URLParam(r, "ref") - if err := h.svc.ServeSubtitle(r.Context(), w, r, userID, profileID, deviceID, id, ref, requestAccessFilter(r)); err != nil { + serveCtx := downloads.WithServeAuthorized(r.Context(), func(target downloads.FileTarget) { + attachTransfer(r.Context(), userID, profileID, target.MediaFileID) + }) + if err := h.svc.ServeSubtitle(serveCtx, w, r, userID, profileID, deviceID, id, ref, requestAccessFilter(r)); err != nil { h.writeAssetError(w, "subtitle", id, err) return } diff --git a/internal/api/handlers/ebook_reader.go b/internal/api/handlers/ebook_reader.go index 87e7fe842..3f4f2a67a 100644 --- a/internal/api/handlers/ebook_reader.go +++ b/internal/api/handlers/ebook_reader.go @@ -133,6 +133,7 @@ func (h *EbookReaderHandler) HandleReadFile(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusNotFound, "not_found", "Ebook file not found") return } + attachTransfer(r.Context(), apimw.GetUserID(r.Context()), apimw.GetProfileID(r.Context()), fileID) if err := h.serveEbook(w, r, file); err != nil { if errors.Is(err, catalog.ErrItemNotFound) { diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index cec144f93..17e40bab9 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -31,6 +31,7 @@ import ( "github.com/Silo-Server/silo-server/internal/settingscontract" "github.com/Silo-Server/silo-server/internal/settingskeys" "github.com/Silo-Server/silo-server/internal/settingsresolve" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/userstore" @@ -159,6 +160,7 @@ type PlaybackHandler struct { MissingMarker MissingFileMarker NodePlanner nodepool.SessionPlanner // optional; enables proxy/transcode node selection JWTSecret string // needed for signing stream tokens + StreamTelemetry *streamtelemetry.Registry // local observation-only telemetry ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item @@ -417,15 +419,6 @@ func (h *PlaybackHandler) signStreamClaims(claims streamtoken.Claims) string { return token } -// streamCardFromQuery verifies the stream token in the request's ?st= parameter -// and returns the decoded reconstruction recipe, or nil when the token is -// absent, invalid/expired, or bound to a different session. A live session needs -// no token (the result is simply nil); the recipe is consumed only on -// reconstruct. -func (h *PlaybackHandler) streamCardFromQuery(r *http.Request, sessionID string) *playback.RecipeCard { - return streamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) -} - // loadTranscodeServeSession resolves the playback Session for the transcode // manifest/segment serve routes while keeping stream-token verification off the // hot path. The overwhelmingly common case is a live in-memory session, which @@ -435,7 +428,7 @@ func (h *PlaybackHandler) streamCardFromQuery(r *http.Request, sessionID string) // LoadOrReconstructSession front door so reconstruct/ownership semantics stay // identical. The returned card (nil on the live-session path) is the decoded // recipe the caller's own reconstruct branch consumes. -func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID string) (*playback.Session, playback.SessionLoadStatus, *playback.RecipeCard) { +func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID string) (*playback.Session, playback.SessionLoadStatus, *playback.RecipeCard, *streamtoken.Claims) { requestUserID := apimw.GetUserID(r.Context()) session, err := h.sessionMgr.GetSession(sessionID) if err == nil { @@ -443,34 +436,72 @@ func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID s // (a zero caller is allowed; a non-zero mismatch is refused). No token // verification on this hot path. if requestUserID != 0 && session.UserID != requestUserID { - return nil, playback.SessionForbidden, nil + return nil, playback.SessionForbidden, nil, nil } - return session, playback.SessionLoaded, nil + return session, playback.SessionLoaded, nil, nil } if !errors.Is(err, playback.ErrSessionNotFound) { - return nil, playback.SessionLoadFailed, nil + return nil, playback.SessionLoadFailed, nil, nil } // Genuine miss (e.g. after a restart): now — and only now — pay for the token // decode so the recipe is available for reconstruction. - card := h.streamCardFromQuery(r, sessionID) + card, claims := verifiedStreamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) session, status := h.tm.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, requestUserID, card) - return session, status, card + return session, status, card, claims } // streamCardFromToken verifies a stream token and decodes its reconstruction // recipe, returning nil when the token is absent, unparseable/expired, or bound // to a different session id. Shared by the native serve handlers (PlaybackHandler // and StreamHandler). -func streamCardFromToken(tokenStr, sessionID, secret string) *playback.RecipeCard { +func verifiedStreamCardFromToken(tokenStr, sessionID, secret string) (*playback.RecipeCard, *streamtoken.Claims) { if tokenStr == "" || secret == "" { - return nil + return nil, nil } claims, err := streamtoken.Verify(tokenStr, secret) if err != nil || claims.SessionID != sessionID { - return nil + return nil, nil } card := playback.RecipeCardFromClaims(claims) - return &card + return &card, claims +} + +func attachPlaybackSession(ctx context.Context, session *playback.Session, claims *streamtoken.Claims) { + if session == nil { + return + } + startedAt := session.StartedAt + startedSource := streamtelemetry.StartedAtSourceSession + tokenIssuedAt := time.Time{} + tokenSource := streamtelemetry.TokenIssuedAtSourceNone + if claims != nil { + if resolved, source := claims.StartedAt(); !resolved.IsZero() { + switch source { + case streamtoken.StartedAtSourceClaim: + startedAt = resolved + startedSource = streamtelemetry.StartedAtSourceClaim + case streamtoken.StartedAtSourceIssuedAt: + if startedAt.IsZero() { + startedAt = resolved + startedSource = streamtelemetry.StartedAtSourceIssuedAt + } + } + } + if claims.IssuedAt != nil { + tokenIssuedAt = claims.IssuedAt.Time + tokenSource = streamtelemetry.TokenIssuedAtSourceVerified + } + } + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{Subject: streamtelemetry.UserSubject(session.UserID), + ProfileID: session.ProfileID, SessionID: session.ID, MediaFileID: session.MediaFileID, + PlayMethod: string(session.PlayMethod), StartedAt: startedAt, StartedAtSource: startedSource, + TokenIssuedAt: tokenIssuedAt, TokenIssuedAtSource: tokenSource}) +} + +func attachTransfer(ctx context.Context, userID int, profileID string, mediaFileID int) { + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{Subject: streamtelemetry.UserSubject(userID), + ProfileID: profileID, MediaFileID: mediaFileID, StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, + TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone}) } // appendStreamToken adds the ?st= parameter to a native serve URL. @@ -1077,7 +1108,9 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req h.handleStartPlaybackV3(w, r, body) } -func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo { +// PlaybackClientInfoFromRequest captures and normalizes playback client headers +// at the HTTP request boundary. +func PlaybackClientInfoFromRequest(r *http.Request) playback.ClientInfo { if r == nil { return playback.ClientInfo{} } @@ -1095,6 +1128,10 @@ func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo { }.Normalized() } +func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo { + return PlaybackClientInfoFromRequest(r) +} + // HandleUpdateProgress handles POST /playback/{session_id}/progress. func (h *PlaybackHandler) HandleUpdateProgress(w http.ResponseWriter, r *http.Request) { userID := apimw.GetUserID(r.Context()) @@ -1308,7 +1345,7 @@ func alignedSeekSeconds(seekSeconds float64, segmentDuration int, targetVideoCod // reports as the timeline's stream_origin_seconds. func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") - session, status, card := h.loadTranscodeServeSession(r, sessionID) + session, status, card, claims := h.loadTranscodeServeSession(r, sessionID) switch status { case playback.SessionMissing: writePlaybackSessionNotFound(w) @@ -1320,6 +1357,7 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } + attachPlaybackSession(r.Context(), session, claims) transcodeSession := h.tm.GetTranscodeSession(sessionID) if transcodeSession == nil { @@ -1363,7 +1401,7 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h // Auth is optional — the session UUID serves as an access token. func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") - session, status, card := h.loadTranscodeServeSession(r, sessionID) + session, status, card, claims := h.loadTranscodeServeSession(r, sessionID) switch status { case playback.SessionMissing: writePlaybackSessionNotFound(w) @@ -1375,6 +1413,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } + attachPlaybackSession(r.Context(), session, claims) transcodeSession := h.tm.GetTranscodeSession(sessionID) if transcodeSession == nil { diff --git a/internal/api/handlers/playback_realtime.go b/internal/api/handlers/playback_realtime.go index 80cf0c3c4..47c0f16d5 100644 --- a/internal/api/handlers/playback_realtime.go +++ b/internal/api/handlers/playback_realtime.go @@ -123,5 +123,8 @@ func (h *PlaybackHandler) setRealtimeConnectionState(sessionID string, connected slog.Warn("failed to update realtime connection state", "session", sessionID, "connected", connected, "error", err) return false } + if h.StreamTelemetry != nil { + h.StreamTelemetry.SetRealtimeConnection(sessionID, connected) + } return true } diff --git a/internal/api/handlers/session_ws_test.go b/internal/api/handlers/session_ws_test.go index 5c526ce66..9ead61d6f 100644 --- a/internal/api/handlers/session_ws_test.go +++ b/internal/api/handlers/session_ws_test.go @@ -13,6 +13,7 @@ import ( apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/auth" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) func TestHandleSessionWebSocket_RequiresHelloBeforeRealtimeReady(t *testing.T) { @@ -24,6 +25,15 @@ func TestHandleSessionWebSocket_RequiresHelloBeforeRealtimeReady(t *testing.T) { handler := NewPlaybackHandler(sessionMgr) handler.RealtimeHub = playback.NewRealtimeHub() + telemetryConfig := streamtelemetry.DefaultConfig("websocket-test") + telemetryConfig.Enabled = true + handler.StreamTelemetry = streamtelemetry.NewRegistry(telemetryConfig, streamtelemetry.NewLocalStore(), nil) + seedRoute := streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyNative, Method: http.MethodGet, + Pattern: "/stream/{session_id}", Class: streamtelemetry.ClassPlayback, + Role: streamtelemetry.RoleViewerEgress, CapRelevant: true, Enrolled: true} + handler.StreamTelemetry.Observe(seedRoute)(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + attachPlaybackSession(r.Context(), session, nil) + })).ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/stream/"+session.ID, nil)) router := chi.NewRouter() router.Get("/playback/ws/{session_id}", func(w http.ResponseWriter, r *http.Request) { @@ -72,12 +82,28 @@ func TestHandleSessionWebSocket_RequiresHelloBeforeRealtimeReady(t *testing.T) { } waitForPlaybackRealtimeState(t, sessionMgr, session.ID, true) + waitForTelemetryRealtimeState(t, handler.StreamTelemetry, session.ID, true) if err := conn.Close(); err != nil { t.Fatalf("Close websocket: %v", err) } waitForPlaybackRealtimeState(t, sessionMgr, session.ID, false) + waitForTelemetryRealtimeState(t, handler.StreamTelemetry, session.ID, false) +} + +func waitForTelemetryRealtimeState(t *testing.T, registry *streamtelemetry.Registry, sessionID string, want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + for _, session := range registry.Snapshot().Sessions { + if session.SessionID == sessionID && session.RealtimeConnectionAlive == want { + return + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("telemetry realtime state for %s did not become %t", sessionID, want) } func waitForPlaybackRealtimeState(t *testing.T, sessionMgr *playback.SessionManager, sessionID string, want bool) { diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index cf7c29027..8863c2279 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -105,7 +105,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { // ?seek= query for remux), so no runtime beyond the Session needs rebuilding. // Without a token (or signing secret) reconstruct is off, collapsing to a // plain GetSession + ownership check. - card := streamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) + card, claims := verifiedStreamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) session, status := h.TM.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, userID, card) switch status { case playback.SessionMissing: @@ -141,6 +141,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { writePlaybackFilePreflightError(w, err) return } + attachPlaybackSession(r.Context(), session, claims) switch session.PlayMethod { case playback.PlayDirect: @@ -224,6 +225,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } + attachPlaybackSession(r.Context(), session, nil) fileID, err := subtitleSourceFileID(r, session) if err != nil { @@ -490,6 +492,7 @@ func (h *StreamHandler) HandleSubtitleFonts(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } + attachPlaybackSession(r.Context(), session, nil) fileID, err := subtitleSourceFileID(r, session) if err != nil { diff --git a/internal/api/handlers/streamtelemetry_test.go b/internal/api/handlers/streamtelemetry_test.go new file mode 100644 index 000000000..ed66b9eef --- /dev/null +++ b/internal/api/handlers/streamtelemetry_test.go @@ -0,0 +1,55 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func TestUnauthenticatedTranscodeObservationUsesSessionOwner(t *testing.T) { + cfg := streamtelemetry.DefaultConfig("test") + cfg.Enabled = true + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + route := streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyNative, Method: http.MethodGet, + Pattern: "/api/v1/playback/transcode/{session_id}/segment/{name}", Class: streamtelemetry.ClassPlayback, + Role: streamtelemetry.RoleViewerEgress, CapRelevant: true, Enrolled: true} + handler := registry.Observe(route)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attachPlaybackSession(r.Context(), &playback.Session{ID: "session", UserID: 91, ProfileID: "profile", + MediaFileID: 42, PlayMethod: playback.PlayTranscode, StartedAt: time.Unix(100, 0)}, nil) + _, _ = w.Write([]byte("segment")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/segment", nil)) + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 || snapshot.Sessions[0].Subject != streamtelemetry.UserSubject(91) { + t.Fatalf("unauthenticated transcode owner = %+v", snapshot) + } +} + +func TestPlaybackAttachmentPrefersSessionStartOverTokenIssuedAt(t *testing.T) { + started := time.Unix(100, 0) + issued := time.Unix(200, 0) + claims := &streamtoken.Claims{RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(issued)}} + cfg := streamtelemetry.DefaultConfig("test") + cfg.Enabled = true + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + route := streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyNative, Method: http.MethodGet, + Pattern: "/stream", Class: streamtelemetry.ClassPlayback, Role: streamtelemetry.RoleViewerEgress, Enrolled: true} + handler := registry.Observe(route)(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + attachPlaybackSession(r.Context(), &playback.Session{ID: "session", UserID: 1, StartedAt: started}, claims) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/stream", nil)) + session := registry.Sweep().Sessions[0] + if session.StartedAtSource != streamtelemetry.StartedAtSourceSession || !session.StartedAt.Equal(started) { + t.Fatalf("started-at = %s (%s)", session.StartedAt, session.StartedAtSource) + } + if session.TokenIssuedAtSources[streamtelemetry.TokenIssuedAtSourceVerified] != 1 { + t.Fatalf("token sources = %+v", session.TokenIssuedAtSources) + } +} diff --git a/internal/api/media_routes.go b/internal/api/media_routes.go new file mode 100644 index 000000000..0c25ca398 --- /dev/null +++ b/internal/api/media_routes.go @@ -0,0 +1,75 @@ +package api + +import ( + "net" + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/api/handlers" + "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var nativeMediaRoutes = []streamtelemetry.MediaRoute{ + nativeRoute(http.MethodGet, "/api/v1/stream/{session_id}", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodHead, "/api/v1/stream/{session_id}", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodGet, "/api/v1/stream/{session_id}/subtitles/{track}", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodHead, "/api/v1/stream/{session_id}/subtitles/{track}", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodGet, "/api/v1/stream/{session_id}/subtitles/{track}/fonts", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodGet, "/api/v1/playback/transcode/{session_id}/master.m3u8", streamtelemetry.ClassManifest, true), + nativeRoute(http.MethodGet, "/api/v1/playback/transcode/{session_id}/segment/{name}", streamtelemetry.ClassPlayback, true), + nativeRoute(http.MethodGet, "/api/v1/downloads/{id}/file", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodHead, "/api/v1/downloads/{id}/file", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodGet, "/api/v1/downloads/{id}/file-proxy", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodHead, "/api/v1/downloads/{id}/file-proxy", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodGet, "/api/v1/downloads/{id}/subtitles/{ref}", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodGet, "/api/v1/direct-download", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodHead, "/api/v1/direct-download", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodGet, "/api/v1/direct-download-proxy", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodHead, "/api/v1/direct-download-proxy", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodGet, "/api/v1/ebooks/{content_id}/files/{file_id}/read", streamtelemetry.ClassTransfer, false), + nativeRoute(http.MethodHead, "/api/v1/ebooks/{content_id}/files/{file_id}/read", streamtelemetry.ClassTransfer, false), +} + +func nativeRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { + return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyNative, Method: method, Pattern: pattern, + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "handler_attachment", + CapRelevant: capRelevant, Enrolled: true, Capture: nativeCapture(pattern)} +} + +func nativeCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { + return func(r *http.Request) streamtelemetry.CaptureSet { + client := handlers.PlaybackClientInfoFromRequest(r) + viewerIP := clientip.FromContext(r.Context()) + if viewerIP == "" { + viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) + if viewerIP == "" { + viewerIP = r.RemoteAddr + } + } + return streamtelemetry.CaptureSet{ + Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, + DeviceID: r.Header.Get("X-Silo-Device-ID"), + Client: streamtelemetry.ClientVariant{Name: client.Name, Version: client.Version, Build: client.Build, Channel: client.Channel}, + UserAgent: client.UserAgent, ReceivedAt: time.Now(), + } + } +} + +func declareNativeMediaRoutes() { streamtelemetry.DeclareRoutes(nativeMediaRoutes...) } + +func nativeMediaRoute(method, pattern string) streamtelemetry.MediaRoute { + for _, route := range nativeMediaRoutes { + if route.Method == method && route.Pattern == pattern { + return route + } + } + panic("undeclared native media route: " + method + " " + pattern) +} + +func observeNative(registry *streamtelemetry.Registry, method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if registry == nil { + return handler + } + return registry.Observe(nativeMediaRoute(method, pattern))(handler).ServeHTTP +} diff --git a/internal/api/media_routes_test.go b/internal/api/media_routes_test.go new file mode 100644 index 000000000..8a3475626 --- /dev/null +++ b/internal/api/media_routes_test.go @@ -0,0 +1,79 @@ +package api + +import ( + "context" + "flag" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/scanner" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var updateRouteManifest = flag.Bool("update-route-manifest", false, "update checked-in route manifest") + +func TestMediaRouteManifest(t *testing.T) { + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatal(err) + } + pool, err := pgxpool.New(context.Background(), "postgres://nobody:nobody@127.0.0.1:1/none?sslmode=disable") + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + declareNativeMediaRoutes() + minimal := NewRouter(Dependencies{Config: cfg}) + maximal := NewRouter(Dependencies{DB: pool, Config: cfg, FileRepo: scanner.NewFileRepository(pool), FolderRepo: catalog.NewFolderRepository(pool), SessionMgr: playback.NewSessionManager(0, 0)}) + actual, err := streamtelemetry.BuildRouteManifest([]chi.Routes{minimal, maximal}, nativeMediaRoutes) + if err != nil { + t.Fatal(err) + } + const path = "testdata/media_routes.txt" + if *updateRouteManifest { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(actual), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != actual { + t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") + } + for _, route := range nativeMediaRoutes { + if !route.Enrolled { + t.Fatalf("native route not enrolled: %s %s", route.Method, route.Pattern) + } + } +} + +func TestNativeRejectedAndMissingRequestsRemainProvisional(t *testing.T) { + for _, route := range nativeMediaRoutes { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound} { + t.Run(route.Method+" "+route.Pattern+" "+http.StatusText(status), func(t *testing.T) { + cfg := streamtelemetry.DefaultConfig("test") + cfg.Enabled = true + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + handler := registry.Observe(route)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(route.Method, "/", nil)) + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("status %d created logical activity: %+v", status, snapshot) + } + }) + } + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 5e5d777a8..3db42277c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -66,6 +66,7 @@ import ( "github.com/Silo-Server/silo-server/internal/secret" "github.com/Silo-Server/silo-server/internal/sections" "github.com/Silo-Server/silo-server/internal/settingscontract" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/subtitles" subtitleai "github.com/Silo-Server/silo-server/internal/subtitles/ai" "github.com/Silo-Server/silo-server/internal/subtitles/opensubtitles" @@ -108,6 +109,7 @@ type Dependencies struct { ProbeEnsurer handlers.PlaybackProbeEnsurer // on-demand probe repair for playback/detail (may be nil) UserStoreProvider userstore.UserStoreProvider // user store provider (may be nil) SessionMgr *playback.SessionManager // playback session manager (may be nil) + StreamTelemetry *streamtelemetry.Registry // local observation-only stream telemetry (may be nil) SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) MovieMatchQueueRepo *metadata.MovieMatchQueueRepository @@ -216,6 +218,7 @@ func (d *Dependencies) CurrentConfig() *config.Config { // under /api/v1/. ABS-compat routes (/abs/*, /login, /socket.io/*) are // mounted at the root level when deps.ABSHandler is non-nil. func NewRouter(deps Dependencies) chi.Router { + declareNativeMediaRoutes() r := chi.NewRouter() useBaseMiddleware(r, deps) @@ -927,6 +930,7 @@ func NewRouter(deps Dependencies) chi.Router { } else { playbackHandler = handlers.NewPlaybackHandler(deps.SessionMgr) } + playbackHandler.StreamTelemetry = deps.StreamTelemetry if deps.DB != nil { playbackHandler.PlanStoreV3 = planstore.NewPostgres(deps.DB) } @@ -2498,8 +2502,8 @@ func NewRouter(deps Dependencies) chi.Router { r.Route("/ebooks", func(r chi.Router) { r.Use(apimw.RequireProfile) r.Get("/capability", ebookReaderHandler.HandleConversionCapability) - r.Get("/{content_id}/files/{file_id}/read", ebookReaderHandler.HandleReadFile) - r.Head("/{content_id}/files/{file_id}/read", ebookReaderHandler.HandleReadFile) + r.Get("/{content_id}/files/{file_id}/read", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/ebooks/{content_id}/files/{file_id}/read", ebookReaderHandler.HandleReadFile)) + r.Head("/{content_id}/files/{file_id}/read", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/ebooks/{content_id}/files/{file_id}/read", ebookReaderHandler.HandleReadFile)) r.Get("/{content_id}/progress", ebookReaderHandler.HandleGetProgress) r.Put("/{content_id}/progress", ebookReaderHandler.HandleSaveProgress) r.Get("/{content_id}/reader-config", ebookReaderHandler.HandleGetConfig) @@ -2619,8 +2623,8 @@ func NewRouter(deps Dependencies) chi.Router { // HLS transcode delivery — no profile auth needed; // session ID (UUID) serves as the access token, same // pattern as /stream/{session_id}. - r.Get("/transcode/{session_id}/master.m3u8", playbackHandler.HandleGetTranscodeManifest) - r.Get("/transcode/{session_id}/segment/{name}", playbackHandler.HandleGetTranscodeSegment) + r.Get("/transcode/{session_id}/master.m3u8", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/playback/transcode/{session_id}/master.m3u8", playbackHandler.HandleGetTranscodeManifest)) + r.Get("/transcode/{session_id}/segment/{name}", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/playback/transcode/{session_id}/segment/{name}", playbackHandler.HandleGetTranscodeSegment)) // Playback realtime control socket — needs auth but not profile. r.Get("/sessions/{session_id}/control/ws", playbackHandler.HandleSessionWebSocket) @@ -2660,11 +2664,11 @@ func NewRouter(deps Dependencies) chi.Router { // Stream routes. if streamHandler != nil { - r.Get("/stream/{session_id}", streamHandler.HandleStream) - r.Head("/stream/{session_id}", streamHandler.HandleStream) - r.Get("/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle) - r.Head("/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle) - r.Get("/stream/{session_id}/subtitles/{track}/fonts", streamHandler.HandleSubtitleFonts) + r.Get("/stream/{session_id}", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/stream/{session_id}", streamHandler.HandleStream)) + r.Head("/stream/{session_id}", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/stream/{session_id}", streamHandler.HandleStream)) + r.Get("/stream/{session_id}/subtitles/{track}", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle)) + r.Head("/stream/{session_id}/subtitles/{track}", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle)) + r.Get("/stream/{session_id}/subtitles/{track}/fonts", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/stream/{session_id}/subtitles/{track}/fonts", streamHandler.HandleSubtitleFonts)) } // Download routes. @@ -2688,18 +2692,18 @@ func NewRouter(deps Dependencies) chi.Router { // GET+HEAD: background download stacks probe with HEAD // before issuing ranged GETs; http.ServeContent handles // HEAD natively. - r.Get("/{id}/file", downloadHandler.HandleDownloadFile) - r.Head("/{id}/file", downloadHandler.HandleDownloadFile) - r.Get("/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy) - r.Head("/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy) + r.Get("/{id}/file", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/downloads/{id}/file", downloadHandler.HandleDownloadFile)) + r.Head("/{id}/file", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/downloads/{id}/file", downloadHandler.HandleDownloadFile)) + r.Get("/{id}/file-proxy", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/downloads/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy)) + r.Head("/{id}/file-proxy", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/downloads/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy)) r.Get("/{id}/manifest", downloadHandler.HandleManifest) r.Get("/{id}/artwork/{kind}", downloadHandler.HandleArtwork) - r.Get("/{id}/subtitles/{ref}", downloadHandler.HandleSubtitle) + r.Get("/{id}/subtitles/{ref}", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/downloads/{id}/subtitles/{ref}", downloadHandler.HandleSubtitle)) }) - r.Get("/direct-download", downloadHandler.HandleDirectDownload) - r.Head("/direct-download", downloadHandler.HandleDirectDownload) - r.Get("/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy) - r.Head("/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy) + r.Get("/direct-download", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/direct-download", downloadHandler.HandleDirectDownload)) + r.Head("/direct-download", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/direct-download", downloadHandler.HandleDirectDownload)) + r.Get("/direct-download-proxy", observeNative(deps.StreamTelemetry, http.MethodGet, "/api/v1/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy)) + r.Head("/direct-download-proxy", observeNative(deps.StreamTelemetry, http.MethodHead, "/api/v1/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy)) // Recipe gallery catalog (no profile required — purely static metadata). recipeHandler := &handlers.RecipeHandler{} diff --git a/internal/api/router_http2_test.go b/internal/api/router_http2_test.go index 082406d64..fd89dccb8 100644 --- a/internal/api/router_http2_test.go +++ b/internal/api/router_http2_test.go @@ -10,6 +10,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) // TestMountedNativeRouterServesMediaOverHTTP2 is a does-not-regress check. @@ -24,7 +25,9 @@ func TestMountedNativeRouterServesMediaOverHTTP2(t *testing.T) { t.Fatalf("LoadFromDB: %v", err) } - routes := &socketRoutes{} + telemetryConfig := streamtelemetry.DefaultConfig("http2-test") + telemetryConfig.Enabled = true + routes := &socketRoutes{telemetry: streamtelemetry.NewRegistry(telemetryConfig, streamtelemetry.NewLocalStore(), nil)} root := chi.NewRouter() useBaseMiddleware(root, Dependencies{ Config: cfg, @@ -77,4 +80,7 @@ func TestMountedNativeRouterServesMediaOverHTTP2(t *testing.T) { if resp.StatusCode != http.StatusPartialContent || string(body) != "2345" { t.Fatalf("HTTP/2 Range = %d %q, want 206 %q", resp.StatusCode, body, "2345") } + if snapshot := routes.telemetry.Sweep(); len(snapshot.Sessions) != 1 || snapshot.Sessions[0].RequestCount != 2 { + t.Fatalf("HTTP/2 telemetry snapshot = %+v", snapshot) + } } diff --git a/internal/api/router_socket_test.go b/internal/api/router_socket_test.go index 4245a6e30..46c85dc67 100644 --- a/internal/api/router_socket_test.go +++ b/internal/api/router_socket_test.go @@ -19,6 +19,7 @@ import ( "github.com/Silo-Server/silo-server/internal/activitylog" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) const nativeSocketMediaETag = `"native-socket-media-v1"` @@ -30,23 +31,41 @@ func (socketActivityWriter) Close() error { return nil } type socketRoutes struct { readerFromSeen atomic.Bool + telemetry *streamtelemetry.Registry } func (h *socketRoutes) Mount(r chi.Router) { media := []byte("0123456789abcdefghijklmnopqrstuvwxyz") serveMedia := func(w http.ResponseWriter, req *http.Request) { + streamtelemetry.Attach(req.Context(), streamtelemetry.Attachment{Subject: streamtelemetry.UserSubject(7), + ProfileID: "socket-profile", SessionID: "socket-session", MediaFileID: 42, + PlayMethod: "direct", StartedAt: time.Unix(1_700_000_000, 0), StartedAtSource: streamtelemetry.StartedAtSourceSession}) _, ok := w.(io.ReaderFrom) h.readerFromSeen.Store(ok) w.Header().Set("Content-Type", "video/mp4") w.Header().Set("ETag", nativeSocketMediaETag) http.ServeContent(w, req, "movie.mp4", time.Unix(1_700_000_000, 0), bytes.NewReader(media)) } - r.Get("/api/v1/stream/socket-test", serveMedia) - r.Head("/api/v1/stream/socket-test", serveMedia) - r.Get("/api/v1/stream/socket-test/subtitles/1/fonts", func(w http.ResponseWriter, _ *http.Request) { + wrap := func(method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if h.telemetry == nil { + return handler + } + route := streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyNative, Method: method, Pattern: pattern, + Class: streamtelemetry.ClassPlayback, Role: streamtelemetry.RoleViewerEgress, CapRelevant: true, Enrolled: true, + Capture: func(r *http.Request) streamtelemetry.CaptureSet { + return streamtelemetry.CaptureSet{Method: r.Method, Pattern: pattern, ReceivedAt: time.Now()} + }} + return h.telemetry.Observe(route)(handler).ServeHTTP + } + r.Get("/api/v1/stream/socket-test", wrap(http.MethodGet, "/api/v1/stream/socket-test", serveMedia)) + r.Head("/api/v1/stream/socket-test", wrap(http.MethodHead, "/api/v1/stream/socket-test", serveMedia)) + r.Get("/api/v1/stream/socket-test/subtitles/1/fonts", wrap(http.MethodGet, "/api/v1/stream/socket-test/subtitles/1/fonts", func(w http.ResponseWriter, req *http.Request) { + streamtelemetry.Attach(req.Context(), streamtelemetry.Attachment{Subject: streamtelemetry.UserSubject(7), + ProfileID: "socket-profile", SessionID: "socket-session", MediaFileID: 42, + PlayMethod: "direct", StartedAt: time.Unix(1_700_000_000, 0), StartedAtSource: streamtelemetry.StartedAtSourceSession}) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"payload": strings.Repeat("compressible-json-", 128)}) - }) + })) } func TestMountedNativeRouterPreservesMediaHTTPAndCompression(t *testing.T) { @@ -54,7 +73,9 @@ func TestMountedNativeRouterPreservesMediaHTTPAndCompression(t *testing.T) { if err != nil { t.Fatalf("LoadFromDB: %v", err) } - routes := &socketRoutes{} + telemetryConfig := streamtelemetry.DefaultConfig("socket-test") + telemetryConfig.Enabled = true + routes := &socketRoutes{telemetry: streamtelemetry.NewRegistry(telemetryConfig, streamtelemetry.NewLocalStore(), nil)} // Drive the real middleware chain (useBaseMiddleware is what NewRouter // itself mounts) rather than NewRouter's full route tree: the media routes // are only registered when their handler dependencies are non-nil, which a @@ -110,6 +131,10 @@ func TestMountedNativeRouterPreservesMediaHTTPAndCompression(t *testing.T) { if body, err := io.ReadAll(zr); err != nil || !bytes.Contains(body, []byte("compressible-json")) { t.Fatalf("compressed JSON body invalid: body=%q err=%v", body, err) } + snapshot := routes.telemetry.Sweep() + if len(snapshot.Sessions) != 1 || snapshot.Sessions[0].RequestCount != 9 || snapshot.Sessions[0].BytesAccepted == 0 { + t.Fatalf("mounted telemetry snapshot = %+v", snapshot) + } } func assertNativeSocketResponse(t *testing.T, client *http.Client, method, url string, headers map[string]string, wantStatus int, wantBody string) { diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt new file mode 100644 index 000000000..26be420fb --- /dev/null +++ b/internal/api/testdata/media_routes.txt @@ -0,0 +1,358 @@ +# fixture 1 +GET /api/v1/health non-media +GET /api/v1/ready non-media +# fixture 2 +GET /api/v1/admin/access-groups non-media +POST /api/v1/admin/access-groups non-media +DELETE /api/v1/admin/access-groups/{id} non-media +GET /api/v1/admin/access-groups/{id} non-media +PUT /api/v1/admin/access-groups/{id} non-media +GET /api/v1/admin/api-keys non-media +POST /api/v1/admin/api-keys non-media +DELETE /api/v1/admin/api-keys/{id} non-media +PUT /api/v1/admin/api-keys/{id}/tier non-media +POST /api/v1/admin/catalog/export non-media +POST /api/v1/admin/catalog/export-jobs non-media +POST /api/v1/admin/catalog/export-jobs/{id}/publish non-media +POST /api/v1/admin/catalog/import non-media +POST /api/v1/admin/catalog/import-jobs non-media +GET /api/v1/admin/catalog/import-sources non-media +GET /api/v1/admin/catalog/local-import-sources non-media +GET /api/v1/admin/catalog/search/status non-media +PUT /api/v1/admin/collection-groups/{groupID}/collections/reorder non-media +DELETE /api/v1/admin/collection-groups/{id} non-media +PUT /api/v1/admin/collection-groups/{id} non-media +GET /api/v1/admin/collections/ non-media +POST /api/v1/admin/collections/ non-media +POST /api/v1/admin/collections/import/mdblist non-media +POST /api/v1/admin/collections/import/tmdb non-media +POST /api/v1/admin/collections/import/trakt non-media +PUT /api/v1/admin/collections/order non-media +POST /api/v1/admin/collections/preview non-media +GET /api/v1/admin/collections/template-bundles non-media +POST /api/v1/admin/collections/template-bundles/{bundleID}/apply non-media +POST /api/v1/admin/collections/template-bundles/{bundleID}/apply-job non-media +GET /api/v1/admin/collections/templates non-media +DELETE /api/v1/admin/collections/{id} non-media +PUT /api/v1/admin/collections/{id} non-media +DELETE /api/v1/admin/collections/{id}/image non-media +PUT /api/v1/admin/collections/{id}/items/order non-media +DELETE /api/v1/admin/collections/{id}/items/{item_id} non-media +PUT /api/v1/admin/collections/{id}/items/{item_id} non-media +POST /api/v1/admin/collections/{id}/sync non-media +GET /api/v1/admin/devices non-media +GET /api/v1/admin/devices/{user_id}/{device_id} non-media +GET /api/v1/admin/diagnostics/reports/ non-media +DELETE /api/v1/admin/diagnostics/reports/{id} non-media +GET /api/v1/admin/diagnostics/reports/{id} non-media +GET /api/v1/admin/diagnostics/reports/{id}/download non-media +POST /api/v1/admin/email/test non-media +POST /api/v1/admin/files/{fileId}/contribute non-media +GET /api/v1/admin/files/{fileId}/contributions non-media +GET /api/v1/admin/filesystem/browse non-media +GET /api/v1/admin/history-import-sources/ non-media +POST /api/v1/admin/history-import-sources/ non-media +DELETE /api/v1/admin/history-import-sources/{id} non-media +PUT /api/v1/admin/history-import-sources/{id} non-media +GET /api/v1/admin/history-imports/mappings non-media +POST /api/v1/admin/history-imports/mappings non-media +DELETE /api/v1/admin/history-imports/mappings/{id} non-media +PUT /api/v1/admin/history-imports/mappings/{id} non-media +POST /api/v1/admin/history-imports/mappings/{id}/run non-media +POST /api/v1/admin/history-imports/plex/login non-media +GET /api/v1/admin/history-imports/runs non-media +GET /api/v1/admin/history-imports/runs/{id} non-media +POST /api/v1/admin/history-imports/runs/{id}/cancel non-media +POST /api/v1/admin/history-imports/sources/{id}/bulk-run non-media +DELETE /api/v1/admin/history-imports/sources/{id}/token non-media +PUT /api/v1/admin/history-imports/sources/{id}/token non-media +GET /api/v1/admin/history-imports/sources/{id}/users non-media +GET /api/v1/admin/invitations/ non-media +POST /api/v1/admin/invitations/ non-media +DELETE /api/v1/admin/invitations/{id} non-media +POST /api/v1/admin/invitations/{id}/resend non-media +GET /api/v1/admin/invite-codes/ non-media +POST /api/v1/admin/invite-codes/ non-media +DELETE /api/v1/admin/invite-codes/{id} non-media +PUT /api/v1/admin/invite-codes/{id} non-media +POST /api/v1/admin/invite-codes/{id}/top-up non-media +GET /api/v1/admin/items/{id}/files non-media +POST /api/v1/admin/items/{id}/merge non-media +PATCH /api/v1/admin/items/{id}/metadata non-media +POST /api/v1/admin/items/{id}/metadata-translation non-media +GET /api/v1/admin/items/{id}/metadata-translation/jobs non-media +POST /api/v1/admin/items/{id}/metadata-translation/jobs/{job_id}/cancel non-media +POST /api/v1/admin/items/{id}/refresh-metadata non-media +POST /api/v1/admin/items/{id}/split non-media +PATCH /api/v1/admin/jellyfin-compat/settings non-media +GET /api/v1/admin/jellyfin-compat/status non-media +POST /api/v1/admin/jellyfin-compat/web/install non-media +POST /api/v1/admin/jellyfin-compat/web/remove non-media +POST /api/v1/admin/jellyfin-compat/web/update non-media +GET /api/v1/admin/jobs/ non-media +GET /api/v1/admin/jobs/{id} non-media +POST /api/v1/admin/jobs/{id}/cancel non-media +GET /api/v1/admin/libraries/{libraryID}/collection-groups/ non-media +POST /api/v1/admin/libraries/{libraryID}/collection-groups/ non-media +PUT /api/v1/admin/libraries/{libraryID}/collection-groups/reorder non-media +GET /api/v1/admin/literary-works/items/{content_id}/candidates non-media +POST /api/v1/admin/literary-works/link non-media +POST /api/v1/admin/literary-works/matches/confirm non-media +POST /api/v1/admin/literary-works/matches/ignore non-media +DELETE /api/v1/admin/literary-works/{work_id}/items/{content_id} non-media +GET /api/v1/admin/markers/files/{fileId}/history non-media +GET /api/v1/admin/markers/history non-media +GET /api/v1/admin/markers/items/{id}/history non-media +DELETE /api/v1/admin/notifications/push/relay non-media +POST /api/v1/admin/notifications/push/relay/register non-media +GET /api/v1/admin/playback-history non-media +GET /api/v1/admin/rate-limits/config non-media +PUT /api/v1/admin/rate-limits/config non-media +GET /api/v1/admin/request-integrations non-media +POST /api/v1/admin/request-integrations non-media +DELETE /api/v1/admin/request-integrations/{id} non-media +PUT /api/v1/admin/request-integrations/{id} non-media +POST /api/v1/admin/request-integrations/{id}/options non-media +GET /api/v1/admin/request-settings non-media +PUT /api/v1/admin/request-settings non-media +GET /api/v1/admin/request-users/{user_id}/limit non-media +PUT /api/v1/admin/request-users/{user_id}/limit non-media +GET /api/v1/admin/requests non-media +POST /api/v1/admin/requests/{id}/approve non-media +POST /api/v1/admin/requests/{id}/cancel non-media +POST /api/v1/admin/requests/{id}/decline non-media +POST /api/v1/admin/requests/{id}/retry non-media +GET /api/v1/admin/sections/ non-media +POST /api/v1/admin/sections/ non-media +POST /api/v1/admin/sections/bulk-create non-media +POST /api/v1/admin/sections/preview non-media +PUT /api/v1/admin/sections/reorder non-media +POST /api/v1/admin/sections/restore-defaults non-media +DELETE /api/v1/admin/sections/{id} non-media +PUT /api/v1/admin/sections/{id} non-media +POST /api/v1/admin/server/restart non-media +GET /api/v1/admin/server/status non-media +GET /api/v1/admin/sessions non-media +GET /api/v1/admin/sessions/capabilities non-media +POST /api/v1/admin/sessions/{session_id}/message non-media +POST /api/v1/admin/sessions/{session_id}/pause non-media +POST /api/v1/admin/sessions/{session_id}/resume non-media +POST /api/v1/admin/sessions/{session_id}/stop non-media +POST /api/v1/admin/sessions/{session_id}/terminate non-media +GET /api/v1/admin/settings non-media +PUT /api/v1/admin/settings non-media +POST /api/v1/admin/settings/check/{kind} non-media +GET /api/v1/admin/settings/effective non-media +GET /api/v1/admin/settings/sections non-media +PUT /api/v1/admin/settings/sections non-media +GET /api/v1/admin/settings/sensitive-status non-media +GET /api/v1/admin/settings/{key} non-media +PUT /api/v1/admin/settings/{key} non-media +GET /api/v1/admin/stats non-media +GET /api/v1/admin/subtitle-providers/ non-media +PUT /api/v1/admin/subtitle-providers/{provider}/ non-media +POST /api/v1/admin/subtitle-providers/{provider}/test non-media +GET /api/v1/admin/subtitles/ non-media +DELETE /api/v1/admin/subtitles/{id}/ non-media +PATCH /api/v1/admin/subtitles/{id}/ non-media +GET /api/v1/admin/subtitles/{id}/download non-media +GET /api/v1/admin/system/build non-media +GET /api/v1/admin/system/hw-accel non-media +GET /api/v1/admin/unmatched non-media +GET /api/v1/admin/users non-media +POST /api/v1/admin/users non-media +DELETE /api/v1/admin/users/{id} non-media +GET /api/v1/admin/users/{id} non-media +PUT /api/v1/admin/users/{id} non-media +POST /api/v1/admin/users/{id}/impersonate non-media +GET /api/v1/admin/users/{id}/profiles non-media +GET /api/v1/admin/users/{userId}/api-keys non-media +GET /api/v1/api-keys/ non-media +POST /api/v1/api-keys/ non-media +DELETE /api/v1/api-keys/{id} non-media +GET /api/v1/auth/device non-media +POST /api/v1/auth/device/approve non-media +GET /api/v1/auth/device/capability non-media +POST /api/v1/auth/device/deny non-media +POST /api/v1/auth/device/poll non-media +POST /api/v1/auth/device/start non-media +POST /api/v1/auth/impersonation/end non-media +POST /api/v1/auth/login non-media +POST /api/v1/auth/logout non-media +GET /api/v1/auth/me non-media +POST /api/v1/auth/plugin-launch non-media +GET /api/v1/auth/providers non-media +POST /api/v1/auth/refresh non-media +GET /api/v1/auth/sessions non-media +DELETE /api/v1/auth/sessions/{id} non-media +GET /api/v1/auth/setup non-media +POST /api/v1/auth/setup non-media +GET /api/v1/auth/signup non-media +POST /api/v1/auth/signup non-media +GET /api/v1/calendar non-media +GET /api/v1/catalog non-media +GET /api/v1/catalog/audiobook-groups non-media +GET /api/v1/catalog/filters non-media +GET /api/v1/catalog/filters/search non-media +GET /api/v1/catalog/items/{id} non-media +GET /api/v1/catalog/items/{id}/episodes non-media +GET /api/v1/catalog/items/{id}/manga-files non-media +GET /api/v1/catalog/items/{id}/versions non-media +POST /api/v1/catalog/query non-media +GET /api/v1/catalog/series/{id}/seasons non-media +GET /api/v1/catalog/series/{id}/seasons/{num} non-media +GET /api/v1/catalog/series/{id}/seasons/{num}/episodes non-media +GET /api/v1/compat/connect-info non-media +POST /api/v1/diagnostics/reports non-media +POST /api/v1/diagnostics/reports/uploads/ non-media +DELETE /api/v1/diagnostics/reports/uploads/{upload_id} non-media +PUT /api/v1/diagnostics/reports/uploads/{upload_id}/chunks/{chunk_index} non-media +POST /api/v1/diagnostics/reports/uploads/{upload_id}/complete non-media +GET /api/v1/diagnostics/status non-media +GET /api/v1/direct-download media transfer viewer_egress false true +HEAD /api/v1/direct-download media transfer viewer_egress false true +GET /api/v1/direct-download-proxy media transfer viewer_egress false true +HEAD /api/v1/direct-download-proxy media transfer viewer_egress false true +GET /api/v1/downloads/ non-media +POST /api/v1/downloads/ non-media +GET /api/v1/downloads/batches/{batch_id}/manifests non-media +GET /api/v1/downloads/capability non-media +GET /api/v1/downloads/subscriptions non-media +POST /api/v1/downloads/subscriptions non-media +POST /api/v1/downloads/subscriptions/sync non-media +DELETE /api/v1/downloads/subscriptions/{id} non-media +GET /api/v1/downloads/subscriptions/{id} non-media +PATCH /api/v1/downloads/subscriptions/{id} non-media +DELETE /api/v1/downloads/{id} non-media +PATCH /api/v1/downloads/{id} non-media +GET /api/v1/downloads/{id}/artwork/{kind} non-media +GET /api/v1/downloads/{id}/file media transfer viewer_egress false true +HEAD /api/v1/downloads/{id}/file media transfer viewer_egress false true +GET /api/v1/downloads/{id}/file-proxy media transfer viewer_egress false true +HEAD /api/v1/downloads/{id}/file-proxy media transfer viewer_egress false true +GET /api/v1/downloads/{id}/manifest non-media +GET /api/v1/downloads/{id}/subtitles/{ref} media transfer viewer_egress false true +GET /api/v1/ebooks/capability non-media +GET /api/v1/ebooks/{content_id}/annotations non-media +POST /api/v1/ebooks/{content_id}/annotations non-media +DELETE /api/v1/ebooks/{content_id}/annotations/{annotation_id} non-media +PATCH /api/v1/ebooks/{content_id}/annotations/{annotation_id} non-media +GET /api/v1/ebooks/{content_id}/files/{file_id}/read media transfer viewer_egress false true +HEAD /api/v1/ebooks/{content_id}/files/{file_id}/read media transfer viewer_egress false true +GET /api/v1/ebooks/{content_id}/progress non-media +PUT /api/v1/ebooks/{content_id}/progress non-media +GET /api/v1/ebooks/{content_id}/reader-config non-media +PUT /api/v1/ebooks/{content_id}/reader-config non-media +GET /api/v1/health non-media +POST /api/v1/history-imports/emby-connect/login non-media +POST /api/v1/history-imports/plex/auth/check non-media +POST /api/v1/history-imports/plex/auth/pin non-media +GET /api/v1/history-imports/runs non-media +POST /api/v1/history-imports/runs non-media +GET /api/v1/history-imports/runs/{id} non-media +GET /api/v1/history-imports/sources non-media +GET /api/v1/home/layout non-media +GET /api/v1/home/sections non-media +GET /api/v1/home/sections/{id}/items non-media +GET /api/v1/invitations/{token}/ non-media +POST /api/v1/invitations/{token}/accept non-media +GET /api/v1/items/trailers/capability non-media +POST /api/v1/items/{id}/translate-description non-media +GET /api/v1/libraries/ non-media +POST /api/v1/libraries/ non-media +GET /api/v1/libraries/metadata-match-queue non-media +GET /api/v1/libraries/provider-defaults non-media +PUT /api/v1/libraries/reorder non-media +GET /api/v1/libraries/roots non-media +DELETE /api/v1/libraries/roots/override non-media +PUT /api/v1/libraries/roots/override non-media +GET /api/v1/libraries/skipped-roots non-media +GET /api/v1/libraries/stale-ids non-media +POST /api/v1/libraries/stale-ids/{contentID}/rematch non-media +GET /api/v1/libraries/unmatched-items non-media +DELETE /api/v1/libraries/{id} non-media +PUT /api/v1/libraries/{id} non-media +POST /api/v1/libraries/{id}/check-mount non-media +POST /api/v1/libraries/{id}/confirm-empty-root-cleanup non-media +GET /api/v1/libraries/{id}/metadata-match-queue non-media +POST /api/v1/libraries/{id}/metadata-match-queue/cancel non-media +POST /api/v1/libraries/{id}/metadata-match-queue/retry non-media +DELETE /api/v1/libraries/{id}/poster non-media +PUT /api/v1/libraries/{id}/poster non-media +GET /api/v1/libraries/{id}/providers non-media +PUT /api/v1/libraries/{id}/providers non-media +POST /api/v1/libraries/{id}/refresh-metadata non-media +GET /api/v1/library/{id}/collections non-media +GET /api/v1/library/{id}/collections/{collection_id}/items non-media +GET /api/v1/library/{id}/layout non-media +GET /api/v1/library/{id}/sections non-media +GET /api/v1/library/{id}/sections/{sectionId}/items non-media +GET /api/v1/library/{id}/user-collections non-media +GET /api/v1/markers/files/{fileId} non-media +PUT /api/v1/markers/files/{fileId} non-media +DELETE /api/v1/markers/files/{fileId}/{segment} non-media +GET /api/v1/markers/items/{id} non-media +PUT /api/v1/markers/items/{id} non-media +GET /api/v1/metadata/ai/status non-media +GET /api/v1/playback/capability non-media +POST /api/v1/playback/route-events non-media +GET /api/v1/playback/sessions/{session_id}/control/ws non-media +POST /api/v1/playback/start non-media +GET /api/v1/playback/transcode/{session_id}/master.m3u8 media manifest viewer_egress true true +GET /api/v1/playback/transcode/{session_id}/segment/{name} media playback viewer_egress true true +DELETE /api/v1/playback/{session_id} non-media +POST /api/v1/playback/{session_id}/progress non-media +POST /api/v1/playback/{session_id}/replan non-media +GET /api/v1/profile/sections/ non-media +PUT /api/v1/profile/sections/ non-media +GET /api/v1/profile/sections/flags non-media +DELETE /api/v1/profile/sections/reset non-media +GET /api/v1/profile/sections/settings non-media +GET /api/v1/ready non-media +GET /api/v1/recommendations/because-watched/{item_id} non-media +GET /api/v1/recommendations/discover non-media +GET /api/v1/recommendations/for-you/main non-media +GET /api/v1/recommendations/for-you/rows non-media +GET /api/v1/recommendations/popular non-media +GET /api/v1/recommendations/recently-added non-media +GET /api/v1/recommendations/section/{kind} non-media +GET /api/v1/recommendations/section/{kind}/{key} non-media +GET /api/v1/recommendations/similar-users non-media +GET /api/v1/recommendations/similar/{item_id} non-media +GET /api/v1/recommendations/taste-profile non-media +POST /api/v1/recommendations/taste-seed non-media +GET /api/v1/recommendations/taste-seed/items non-media +GET /api/v1/recommendations/watch-tonight non-media +GET /api/v1/recommendations/watch-tonight/cards non-media +POST /api/v1/requests/ non-media +GET /api/v1/requests/detail/{media_type}/{tmdb_id} non-media +GET /api/v1/requests/discover non-media +GET /api/v1/requests/discover/browse/genre/{slug} non-media +GET /api/v1/requests/discover/browse/network/{slug} non-media +GET /api/v1/requests/discover/browse/studio/{slug} non-media +GET /api/v1/requests/discover/genres non-media +GET /api/v1/requests/discover/networks non-media +GET /api/v1/requests/discover/studios non-media +GET /api/v1/requests/discover/{section} non-media +GET /api/v1/requests/mine non-media +GET /api/v1/requests/search non-media +GET /api/v1/requests/status non-media +GET /api/v1/requests/{id} non-media +POST /api/v1/requests/{id}/cancel non-media +POST /api/v1/scan non-media +POST /api/v1/scan/cancel non-media +GET /api/v1/sections/recipes non-media +GET /api/v1/sections/recipes/{type}/candidates non-media +GET /api/v1/stream/{session_id} media playback viewer_egress true true +HEAD /api/v1/stream/{session_id} media playback viewer_egress true true +GET /api/v1/stream/{session_id}/subtitles/{track} media playback viewer_egress true true +HEAD /api/v1/stream/{session_id}/subtitles/{track} media playback viewer_egress true true +GET /api/v1/stream/{session_id}/subtitles/{track}/fonts media playback viewer_egress true true +GET /api/v1/subtitles/providers/status non-media +GET /api/v1/theme/admin-css non-media +GET /api/v1/theme/catalog non-media +POST /api/v1/theme/catalog/refresh non-media +GET /api/v1/theme/download non-media +GET /api/v1/user/libraries non-media +GET /api/v1/watch/{id} non-media +GET /api/v1/works/{work_id} non-media diff --git a/internal/audiobooks/abs/handler.go b/internal/audiobooks/abs/handler.go index 2a1d2e378..0df981d6c 100644 --- a/internal/audiobooks/abs/handler.go +++ b/internal/audiobooks/abs/handler.go @@ -361,6 +361,7 @@ func New(deps Dependencies) *Handler { // here so stage-by-stage handlers land in the right places without needing to // revisit Mount later. func (h *Handler) Mount(parent chi.Router) { + declareABSMediaRoutes() parent.Group(func(r chi.Router) { r.Use(h.accessLog) h.mountRoutes(r) diff --git a/internal/audiobooks/abs/media_routes.go b/internal/audiobooks/abs/media_routes.go new file mode 100644 index 000000000..79b9df489 --- /dev/null +++ b/internal/audiobooks/abs/media_routes.go @@ -0,0 +1,35 @@ +package abs + +import ( + "net/http" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var absMediaRoutes = func() []streamtelemetry.MediaRoute { + const absAPIPrefix = "/abs/api" + const apiPrefix = "/api" + routes := []streamtelemetry.MediaRoute{ + absRoute(http.MethodGet, "/public/session/{sid}/track/{idx}", streamtelemetry.ClassPlayback, true, "session_id"), + absRoute(http.MethodHead, "/public/session/{sid}/track/{idx}", streamtelemetry.ClassPlayback, true, "session_id"), + absRoute(http.MethodGet, "/abs/public/session/{sid}/track/{idx}", streamtelemetry.ClassPlayback, true, "session_id"), + absRoute(http.MethodHead, "/abs/public/session/{sid}/track/{idx}", streamtelemetry.ClassPlayback, true, "session_id"), + absRoute(http.MethodGet, "/feed/{slug}/file/{ino}", streamtelemetry.ClassTransfer, false, "feed_owner"), + } + for _, prefix := range []string{apiPrefix, absAPIPrefix} { + routes = append(routes, + absRoute(http.MethodGet, prefix+"/items/{libraryItemId}/file/{ino}", streamtelemetry.ClassTransfer, false, "abs_user"), + absRoute(http.MethodGet, prefix+"/items/{libraryItemId}/file/{ino}/download", streamtelemetry.ClassTransfer, false, "abs_user"), + absRoute(http.MethodGet, prefix+"/items/{id}/ebook/{fileid}", streamtelemetry.ClassTransfer, false, "abs_user"), + ) + } + return routes +}() + +func absRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool, key string) streamtelemetry.MediaRoute { + return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyABS, Method: method, Pattern: pattern, + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: key, + CapRelevant: capRelevant, Enrolled: false} +} + +func declareABSMediaRoutes() { streamtelemetry.DeclareRoutes(absMediaRoutes...) } diff --git a/internal/audiobooks/abs/media_routes_test.go b/internal/audiobooks/abs/media_routes_test.go new file mode 100644 index 000000000..8e30de6d8 --- /dev/null +++ b/internal/audiobooks/abs/media_routes_test.go @@ -0,0 +1,47 @@ +package abs + +import ( + "flag" + "os" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var updateRouteManifest = flag.Bool("update-route-manifest", false, "update checked-in route manifest") + +func TestMediaRouteManifest(t *testing.T) { + declareABSMediaRoutes() + makeRouter := func() chi.Routes { + router := chi.NewRouter() + New(Dependencies{MediaStore: noopMediaStore{}}).Mount(router) + return router + } + actual, err := streamtelemetry.BuildRouteManifest([]chi.Routes{makeRouter(), makeRouter()}, absMediaRoutes) + if err != nil { + t.Fatal(err) + } + const path = "testdata/media_routes.txt" + if *updateRouteManifest { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(actual), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != actual { + t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") + } + for _, route := range absMediaRoutes { + if route.Enrolled { + t.Fatalf("ABS route enrolled: %s %s", route.Method, route.Pattern) + } + } +} diff --git a/internal/audiobooks/abs/testdata/media_routes.txt b/internal/audiobooks/abs/testdata/media_routes.txt new file mode 100644 index 000000000..48b7d7401 --- /dev/null +++ b/internal/audiobooks/abs/testdata/media_routes.txt @@ -0,0 +1,368 @@ +# fixture 1 +POST /abs/api/auth/logout non-media +POST /abs/api/auth/refresh non-media +POST /abs/api/authorize non-media +GET /abs/api/authors/{id} non-media +GET /abs/api/authors/{id}/image non-media +GET /abs/api/collections non-media +POST /abs/api/collections non-media +DELETE /abs/api/collections/{id} non-media +GET /abs/api/collections/{id} non-media +PATCH /abs/api/collections/{id} non-media +DELETE /abs/api/collections/{id}/book/{bookId} non-media +POST /abs/api/collections/{id}/book/{bookId} non-media +POST /abs/api/emails/send-ebook-to-device non-media +GET /abs/api/feeds non-media +POST /abs/api/feeds/item/{itemId}/open non-media +POST /abs/api/feeds/{id}/close non-media +GET /abs/api/healthcheck non-media +GET /abs/api/init non-media +GET /abs/api/items/{id} non-media +GET /abs/api/items/{id}/cover non-media +GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +PATCH /abs/api/items/{id}/ebook/{fileid}/status non-media +GET /abs/api/items/{id}/similar non-media +GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false +GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +POST /abs/api/items/{libraryItemId}/play non-media +POST /abs/api/items/{libraryItemId}/play/{episodeId} non-media +GET /abs/api/libraries non-media +GET /abs/api/libraries/{libraryId} non-media +GET /abs/api/libraries/{libraryId}/authors non-media +GET /abs/api/libraries/{libraryId}/collections non-media +GET /abs/api/libraries/{libraryId}/items non-media +GET /abs/api/libraries/{libraryId}/personalized non-media +GET /abs/api/libraries/{libraryId}/playlists non-media +GET /abs/api/libraries/{libraryId}/recent-episodes non-media +GET /abs/api/libraries/{libraryId}/search non-media +GET /abs/api/libraries/{libraryId}/series non-media +POST /abs/api/login non-media +POST /abs/api/logout non-media +GET /abs/api/me non-media +GET /abs/api/me/ereader-devices non-media +PATCH /abs/api/me/item/{itemId}/bookmark non-media +POST /abs/api/me/item/{itemId}/bookmark non-media +DELETE /abs/api/me/item/{itemId}/bookmark/{time} non-media +GET /abs/api/me/items-in-progress non-media +GET /abs/api/me/listening-sessions non-media +GET /abs/api/me/listening-sessions/{sid} non-media +GET /abs/api/me/listening-stats non-media +GET /abs/api/me/progress non-media +GET /abs/api/me/progress/{itemId}/readd-to-continue-listening non-media +GET /abs/api/me/progress/{itemId}/remove-from-continue-listening non-media +DELETE /abs/api/me/progress/{libraryItemId} non-media +GET /abs/api/me/progress/{libraryItemId} non-media +PATCH /abs/api/me/progress/{libraryItemId} non-media +POST /abs/api/me/progress/{libraryItemId} non-media +PATCH /abs/api/me/progress/{libraryItemId}/{episodeId} non-media +GET /abs/api/me/smart-collections non-media +POST /abs/api/me/smart-collections non-media +DELETE /abs/api/me/smart-collections/{id} non-media +GET /abs/api/me/smart-collections/{id} non-media +PATCH /abs/api/me/smart-collections/{id} non-media +GET /abs/api/me/smart-collections/{id}/items non-media +GET /abs/api/me/stats/year/{year} non-media +GET /abs/api/ping non-media +GET /abs/api/playlists non-media +POST /abs/api/playlists non-media +DELETE /abs/api/playlists/{id} non-media +GET /abs/api/playlists/{id} non-media +PATCH /abs/api/playlists/{id} non-media +POST /abs/api/playlists/{id}/batch/add non-media +POST /abs/api/playlists/{id}/batch/remove non-media +POST /abs/api/playlists/{id}/item non-media +DELETE /abs/api/playlists/{id}/item/{libraryItemId} non-media +DELETE /abs/api/playlists/{id}/item/{libraryItemId}/{episodeId} non-media +POST /abs/api/podcasts/feed non-media +GET /abs/api/search/podcast non-media +GET /abs/api/series/{id} non-media +POST /abs/api/session/local non-media +POST /abs/api/session/local-all non-media +PATCH /abs/api/session/{sid} non-media +POST /abs/api/session/{sid}/close non-media +POST /abs/api/session/{sid}/sync non-media +GET /abs/api/status non-media +GET /abs/auth-settings non-media +GET /abs/healthcheck non-media +GET /abs/init non-media +GET /abs/ping non-media +GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /api/auth-settings non-media +POST /api/auth/refresh non-media +POST /api/authorize non-media +GET /api/authors/{id} non-media +GET /api/authors/{id}/image non-media +GET /api/collections non-media +POST /api/collections non-media +DELETE /api/collections/{id} non-media +GET /api/collections/{id} non-media +PATCH /api/collections/{id} non-media +DELETE /api/collections/{id}/book/{bookId} non-media +POST /api/collections/{id}/book/{bookId} non-media +POST /api/emails/send-ebook-to-device non-media +GET /api/feeds non-media +POST /api/feeds/item/{itemId}/open non-media +POST /api/feeds/{id}/close non-media +GET /api/healthcheck non-media +GET /api/init non-media +GET /api/items/{id} non-media +GET /api/items/{id}/cover non-media +GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +PATCH /api/items/{id}/ebook/{fileid}/status non-media +GET /api/items/{id}/similar non-media +GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false +GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +POST /api/items/{libraryItemId}/play non-media +POST /api/items/{libraryItemId}/play/{episodeId} non-media +GET /api/libraries non-media +GET /api/libraries/{libraryId} non-media +GET /api/libraries/{libraryId}/authors non-media +GET /api/libraries/{libraryId}/collections non-media +GET /api/libraries/{libraryId}/items non-media +GET /api/libraries/{libraryId}/personalized non-media +GET /api/libraries/{libraryId}/playlists non-media +GET /api/libraries/{libraryId}/recent-episodes non-media +GET /api/libraries/{libraryId}/search non-media +GET /api/libraries/{libraryId}/series non-media +POST /api/login non-media +POST /api/logout non-media +GET /api/me non-media +GET /api/me/ereader-devices non-media +PATCH /api/me/item/{itemId}/bookmark non-media +POST /api/me/item/{itemId}/bookmark non-media +DELETE /api/me/item/{itemId}/bookmark/{time} non-media +GET /api/me/items-in-progress non-media +GET /api/me/listening-sessions non-media +GET /api/me/listening-sessions/{sid} non-media +GET /api/me/listening-stats non-media +GET /api/me/progress non-media +GET /api/me/progress/{itemId}/readd-to-continue-listening non-media +GET /api/me/progress/{itemId}/remove-from-continue-listening non-media +DELETE /api/me/progress/{libraryItemId} non-media +GET /api/me/progress/{libraryItemId} non-media +PATCH /api/me/progress/{libraryItemId} non-media +POST /api/me/progress/{libraryItemId} non-media +PATCH /api/me/progress/{libraryItemId}/{episodeId} non-media +GET /api/me/smart-collections non-media +POST /api/me/smart-collections non-media +DELETE /api/me/smart-collections/{id} non-media +GET /api/me/smart-collections/{id} non-media +PATCH /api/me/smart-collections/{id} non-media +GET /api/me/smart-collections/{id}/items non-media +GET /api/me/stats/year/{year} non-media +GET /api/ping non-media +GET /api/playlists non-media +POST /api/playlists non-media +DELETE /api/playlists/{id} non-media +GET /api/playlists/{id} non-media +PATCH /api/playlists/{id} non-media +POST /api/playlists/{id}/batch/add non-media +POST /api/playlists/{id}/batch/remove non-media +POST /api/playlists/{id}/item non-media +DELETE /api/playlists/{id}/item/{libraryItemId} non-media +DELETE /api/playlists/{id}/item/{libraryItemId}/{episodeId} non-media +POST /api/podcasts/feed non-media +GET /api/search/podcast non-media +GET /api/series/{id} non-media +POST /api/session/local non-media +POST /api/session/local-all non-media +PATCH /api/session/{sid} non-media +POST /api/session/{sid}/close non-media +POST /api/session/{sid}/sync non-media +POST /auth/refresh non-media +GET /feed/{slug} non-media +GET /feed/{slug}.xml non-media +GET /feed/{slug}/file/{ino} media transfer viewer_egress false false +GET /healthcheck non-media +GET /init non-media +POST /login non-media +POST /logout non-media +GET /ping non-media +GET /public/session/{sid}/track/{idx} media playback viewer_egress true false +HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /status non-media +# fixture 2 +POST /abs/api/auth/logout non-media +POST /abs/api/auth/refresh non-media +POST /abs/api/authorize non-media +GET /abs/api/authors/{id} non-media +GET /abs/api/authors/{id}/image non-media +GET /abs/api/collections non-media +POST /abs/api/collections non-media +DELETE /abs/api/collections/{id} non-media +GET /abs/api/collections/{id} non-media +PATCH /abs/api/collections/{id} non-media +DELETE /abs/api/collections/{id}/book/{bookId} non-media +POST /abs/api/collections/{id}/book/{bookId} non-media +POST /abs/api/emails/send-ebook-to-device non-media +GET /abs/api/feeds non-media +POST /abs/api/feeds/item/{itemId}/open non-media +POST /abs/api/feeds/{id}/close non-media +GET /abs/api/healthcheck non-media +GET /abs/api/init non-media +GET /abs/api/items/{id} non-media +GET /abs/api/items/{id}/cover non-media +GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +PATCH /abs/api/items/{id}/ebook/{fileid}/status non-media +GET /abs/api/items/{id}/similar non-media +GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false +GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +POST /abs/api/items/{libraryItemId}/play non-media +POST /abs/api/items/{libraryItemId}/play/{episodeId} non-media +GET /abs/api/libraries non-media +GET /abs/api/libraries/{libraryId} non-media +GET /abs/api/libraries/{libraryId}/authors non-media +GET /abs/api/libraries/{libraryId}/collections non-media +GET /abs/api/libraries/{libraryId}/items non-media +GET /abs/api/libraries/{libraryId}/personalized non-media +GET /abs/api/libraries/{libraryId}/playlists non-media +GET /abs/api/libraries/{libraryId}/recent-episodes non-media +GET /abs/api/libraries/{libraryId}/search non-media +GET /abs/api/libraries/{libraryId}/series non-media +POST /abs/api/login non-media +POST /abs/api/logout non-media +GET /abs/api/me non-media +GET /abs/api/me/ereader-devices non-media +PATCH /abs/api/me/item/{itemId}/bookmark non-media +POST /abs/api/me/item/{itemId}/bookmark non-media +DELETE /abs/api/me/item/{itemId}/bookmark/{time} non-media +GET /abs/api/me/items-in-progress non-media +GET /abs/api/me/listening-sessions non-media +GET /abs/api/me/listening-sessions/{sid} non-media +GET /abs/api/me/listening-stats non-media +GET /abs/api/me/progress non-media +GET /abs/api/me/progress/{itemId}/readd-to-continue-listening non-media +GET /abs/api/me/progress/{itemId}/remove-from-continue-listening non-media +DELETE /abs/api/me/progress/{libraryItemId} non-media +GET /abs/api/me/progress/{libraryItemId} non-media +PATCH /abs/api/me/progress/{libraryItemId} non-media +POST /abs/api/me/progress/{libraryItemId} non-media +PATCH /abs/api/me/progress/{libraryItemId}/{episodeId} non-media +GET /abs/api/me/smart-collections non-media +POST /abs/api/me/smart-collections non-media +DELETE /abs/api/me/smart-collections/{id} non-media +GET /abs/api/me/smart-collections/{id} non-media +PATCH /abs/api/me/smart-collections/{id} non-media +GET /abs/api/me/smart-collections/{id}/items non-media +GET /abs/api/me/stats/year/{year} non-media +GET /abs/api/ping non-media +GET /abs/api/playlists non-media +POST /abs/api/playlists non-media +DELETE /abs/api/playlists/{id} non-media +GET /abs/api/playlists/{id} non-media +PATCH /abs/api/playlists/{id} non-media +POST /abs/api/playlists/{id}/batch/add non-media +POST /abs/api/playlists/{id}/batch/remove non-media +POST /abs/api/playlists/{id}/item non-media +DELETE /abs/api/playlists/{id}/item/{libraryItemId} non-media +DELETE /abs/api/playlists/{id}/item/{libraryItemId}/{episodeId} non-media +POST /abs/api/podcasts/feed non-media +GET /abs/api/search/podcast non-media +GET /abs/api/series/{id} non-media +POST /abs/api/session/local non-media +POST /abs/api/session/local-all non-media +PATCH /abs/api/session/{sid} non-media +POST /abs/api/session/{sid}/close non-media +POST /abs/api/session/{sid}/sync non-media +GET /abs/api/status non-media +GET /abs/auth-settings non-media +GET /abs/healthcheck non-media +GET /abs/init non-media +GET /abs/ping non-media +GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /api/auth-settings non-media +POST /api/auth/refresh non-media +POST /api/authorize non-media +GET /api/authors/{id} non-media +GET /api/authors/{id}/image non-media +GET /api/collections non-media +POST /api/collections non-media +DELETE /api/collections/{id} non-media +GET /api/collections/{id} non-media +PATCH /api/collections/{id} non-media +DELETE /api/collections/{id}/book/{bookId} non-media +POST /api/collections/{id}/book/{bookId} non-media +POST /api/emails/send-ebook-to-device non-media +GET /api/feeds non-media +POST /api/feeds/item/{itemId}/open non-media +POST /api/feeds/{id}/close non-media +GET /api/healthcheck non-media +GET /api/init non-media +GET /api/items/{id} non-media +GET /api/items/{id}/cover non-media +GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +PATCH /api/items/{id}/ebook/{fileid}/status non-media +GET /api/items/{id}/similar non-media +GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false +GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +POST /api/items/{libraryItemId}/play non-media +POST /api/items/{libraryItemId}/play/{episodeId} non-media +GET /api/libraries non-media +GET /api/libraries/{libraryId} non-media +GET /api/libraries/{libraryId}/authors non-media +GET /api/libraries/{libraryId}/collections non-media +GET /api/libraries/{libraryId}/items non-media +GET /api/libraries/{libraryId}/personalized non-media +GET /api/libraries/{libraryId}/playlists non-media +GET /api/libraries/{libraryId}/recent-episodes non-media +GET /api/libraries/{libraryId}/search non-media +GET /api/libraries/{libraryId}/series non-media +POST /api/login non-media +POST /api/logout non-media +GET /api/me non-media +GET /api/me/ereader-devices non-media +PATCH /api/me/item/{itemId}/bookmark non-media +POST /api/me/item/{itemId}/bookmark non-media +DELETE /api/me/item/{itemId}/bookmark/{time} non-media +GET /api/me/items-in-progress non-media +GET /api/me/listening-sessions non-media +GET /api/me/listening-sessions/{sid} non-media +GET /api/me/listening-stats non-media +GET /api/me/progress non-media +GET /api/me/progress/{itemId}/readd-to-continue-listening non-media +GET /api/me/progress/{itemId}/remove-from-continue-listening non-media +DELETE /api/me/progress/{libraryItemId} non-media +GET /api/me/progress/{libraryItemId} non-media +PATCH /api/me/progress/{libraryItemId} non-media +POST /api/me/progress/{libraryItemId} non-media +PATCH /api/me/progress/{libraryItemId}/{episodeId} non-media +GET /api/me/smart-collections non-media +POST /api/me/smart-collections non-media +DELETE /api/me/smart-collections/{id} non-media +GET /api/me/smart-collections/{id} non-media +PATCH /api/me/smart-collections/{id} non-media +GET /api/me/smart-collections/{id}/items non-media +GET /api/me/stats/year/{year} non-media +GET /api/ping non-media +GET /api/playlists non-media +POST /api/playlists non-media +DELETE /api/playlists/{id} non-media +GET /api/playlists/{id} non-media +PATCH /api/playlists/{id} non-media +POST /api/playlists/{id}/batch/add non-media +POST /api/playlists/{id}/batch/remove non-media +POST /api/playlists/{id}/item non-media +DELETE /api/playlists/{id}/item/{libraryItemId} non-media +DELETE /api/playlists/{id}/item/{libraryItemId}/{episodeId} non-media +POST /api/podcasts/feed non-media +GET /api/search/podcast non-media +GET /api/series/{id} non-media +POST /api/session/local non-media +POST /api/session/local-all non-media +PATCH /api/session/{sid} non-media +POST /api/session/{sid}/close non-media +POST /api/session/{sid}/sync non-media +POST /auth/refresh non-media +GET /feed/{slug} non-media +GET /feed/{slug}.xml non-media +GET /feed/{slug}/file/{ino} media transfer viewer_egress false false +GET /healthcheck non-media +GET /init non-media +POST /login non-media +POST /logout non-media +GET /ping non-media +GET /public/session/{sid}/track/{idx} media playback viewer_egress true false +HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /status non-media diff --git a/internal/downloads/offline.go b/internal/downloads/offline.go index 28b8c58cc..10231fbc3 100644 --- a/internal/downloads/offline.go +++ b/internal/downloads/offline.go @@ -176,6 +176,7 @@ func (s *Service) ServeSubtitle(ctx context.Context, w http.ResponseWriter, _ *h if err := s.itemAccess.EnsureAccessible(ctx, dl.ContentID, filter); err != nil { return err } + notifyServeAuthorized(ctx, FileTarget{DownloadID: dl.ID, MediaFileID: dl.MediaFileID}) kind, value, err := parseSubtitleRef(ref) if err != nil { diff --git a/internal/downloads/serve_observer.go b/internal/downloads/serve_observer.go new file mode 100644 index 000000000..f0431f54b --- /dev/null +++ b/internal/downloads/serve_observer.go @@ -0,0 +1,25 @@ +package downloads + +import "context" + +type serveAuthorizedContextKey struct{} + +// WithServeAuthorized registers a request-scoped callback invoked after a file +// target is authorized and before response bytes are served. It does not alter +// authorization or serving behavior. +func WithServeAuthorized(ctx context.Context, callback func(FileTarget)) context.Context { + if ctx == nil || callback == nil { + return ctx + } + return context.WithValue(ctx, serveAuthorizedContextKey{}, callback) +} + +func notifyServeAuthorized(ctx context.Context, target FileTarget) { + if ctx == nil { + return + } + callback, _ := ctx.Value(serveAuthorizedContextKey{}).(func(FileTarget)) + if callback != nil { + callback(target) + } +} diff --git a/internal/downloads/service.go b/internal/downloads/service.go index 61c21c1cf..1d1bd43ec 100644 --- a/internal/downloads/service.go +++ b/internal/downloads/service.go @@ -870,6 +870,7 @@ func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *htt if err != nil { return err } + notifyServeAuthorized(ctx, *target) return s.serveLocalFile(ctx, w, r, target.Path, userID) } @@ -909,6 +910,7 @@ func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http. if err != nil { return err } + notifyServeAuthorized(ctx, *target) return s.serveFileTarget(ctx, w, r, target, userID) } @@ -931,7 +933,6 @@ func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http. if dl.Status == StatusPreparing { return fmt.Errorf("download is preparing: %w", ErrDownloadNotActive) } - // Atomically transition queued → downloading for original rows. Artifact // (remux/transcode) rows are already ready by the time bytes are served. if dl.Format == FormatOriginal && dl.Status == StatusQueued { @@ -1089,6 +1090,7 @@ func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter, if err != nil { return err } + notifyServeAuthorized(ctx, *target) return s.serveFileTarget(ctx, w, r, target, userID) } diff --git a/internal/httpstream/rolling_deadline.go b/internal/httpstream/rolling_deadline.go index 399e97d02..79c3e2fb3 100644 --- a/internal/httpstream/rolling_deadline.go +++ b/internal/httpstream/rolling_deadline.go @@ -166,10 +166,21 @@ func (s *RollingDeadlineWriter) BytesWritten() int64 { // Outcome classifies the first write failure, or a canceled request when no // write failure was surfaced by the transport. func (s *RollingDeadlineWriter) Outcome(ctx context.Context) StreamOutcome { - if isTimeoutError(s.firstWriteErr) { + var ctxErr error + if ctx != nil { + ctxErr = ctx.Err() + } + return ClassifyOutcome(s.firstWriteErr, ctxErr) +} + +// ClassifyOutcome classifies a streaming response from its first write error +// and request-context error. It is shared by every streaming writer so stalled +// connections have one definition throughout the server. +func ClassifyOutcome(firstWriteErr, ctxErr error) StreamOutcome { + if isTimeoutError(firstWriteErr) { return OutcomeStalledReap } - if s.firstWriteErr != nil || (ctx != nil && ctx.Err() != nil) { + if firstWriteErr != nil || ctxErr != nil { return OutcomeClientGone } return OutcomeCompleted diff --git a/internal/httpstream/rolling_deadline_test.go b/internal/httpstream/rolling_deadline_test.go index d64c548da..0b056fdaa 100644 --- a/internal/httpstream/rolling_deadline_test.go +++ b/internal/httpstream/rolling_deadline_test.go @@ -15,6 +15,27 @@ import ( "time" ) +func TestClassifyOutcome(t *testing.T) { + tests := []struct { + name string + firstWriteErr error + contextErr error + want StreamOutcome + }{ + {name: "completed", want: OutcomeCompleted}, + {name: "stalled write", firstWriteErr: os.ErrDeadlineExceeded, want: OutcomeStalledReap}, + {name: "write failure", firstWriteErr: io.ErrClosedPipe, want: OutcomeClientGone}, + {name: "canceled context", contextErr: context.Canceled, want: OutcomeClientGone}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := ClassifyOutcome(test.firstWriteErr, test.contextErr); got != test.want { + t.Fatalf("ClassifyOutcome() = %q, want %q", got, test.want) + } + }) + } +} + // TestStreamSurvivesServerWriteTimeout is the regression test for the 120s // stream-truncation bug: a response that keeps making progress must outlive // the server's absolute WriteTimeout when wrapped. diff --git a/internal/jellycompat/media_routes.go b/internal/jellycompat/media_routes.go new file mode 100644 index 000000000..e83196c97 --- /dev/null +++ b/internal/jellycompat/media_routes.go @@ -0,0 +1,30 @@ +package jellycompat + +import ( + "net/http" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var jellycompatMediaRoutes = []streamtelemetry.MediaRoute{ + compatRoute(http.MethodGet, "/Playback/BitrateTest", streamtelemetry.ClassTransfer, false), + compatRoute(http.MethodGet, "/Items/{id}/Download", streamtelemetry.ClassTransfer, false), + compatRoute(http.MethodHead, "/Items/{id}/Download", streamtelemetry.ClassTransfer, false), + compatRoute(http.MethodGet, "/Videos/{id}/stream", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodHead, "/Videos/{id}/stream", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodGet, "/Videos/{id}/stream.{container}", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodHead, "/Videos/{id}/stream.{container}", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodGet, "/Videos/{id}/master.m3u8", streamtelemetry.ClassManifest, true), + compatRoute(http.MethodGet, "/Videos/{id}/hls/{playlistId}/stream.m3u8", streamtelemetry.ClassManifest, true), + compatRoute(http.MethodGet, "/Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer}", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodGet, "/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat}", streamtelemetry.ClassPlayback, true), + compatRoute(http.MethodGet, "/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat}", streamtelemetry.ClassPlayback, true), +} + +func compatRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { + return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyJellycompat, Method: method, Pattern: pattern, + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "compat_play_session", + CapRelevant: capRelevant, Enrolled: false} +} + +func declareJellycompatMediaRoutes() { streamtelemetry.DeclareRoutes(jellycompatMediaRoutes...) } diff --git a/internal/jellycompat/media_routes_test.go b/internal/jellycompat/media_routes_test.go new file mode 100644 index 000000000..3c8407e0b --- /dev/null +++ b/internal/jellycompat/media_routes_test.go @@ -0,0 +1,49 @@ +package jellycompat + +import ( + "flag" + "os" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var updateRouteManifest = flag.Bool("update-route-manifest", false, "update checked-in route manifest") + +func TestMediaRouteManifest(t *testing.T) { + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatal(err) + } + declareJellycompatMediaRoutes() + minimal := NewRouter(Dependencies{Config: cfg}) + maximal := NewRouter(Dependencies{Config: cfg}) + actual, err := streamtelemetry.BuildRouteManifest([]chi.Routes{minimal, maximal}, jellycompatMediaRoutes) + if err != nil { + t.Fatal(err) + } + const path = "testdata/media_routes.txt" + if *updateRouteManifest { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(actual), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != actual { + t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") + } + for _, route := range jellycompatMediaRoutes { + if route.Enrolled { + t.Fatalf("jellycompat route enrolled: %s %s", route.Method, route.Pattern) + } + } +} diff --git a/internal/jellycompat/router.go b/internal/jellycompat/router.go index 2937e0ae0..d81e8c855 100644 --- a/internal/jellycompat/router.go +++ b/internal/jellycompat/router.go @@ -24,6 +24,7 @@ import ( // NewRouter builds the Jellyfin-compatibility router. func NewRouter(deps Dependencies) chi.Router { + declareJellycompatMediaRoutes() deps = withDefaults(deps) r := chi.NewRouter() diff --git a/internal/jellycompat/testdata/media_routes.txt b/internal/jellycompat/testdata/media_routes.txt new file mode 100644 index 000000000..7ce1bd7c5 --- /dev/null +++ b/internal/jellycompat/testdata/media_routes.txt @@ -0,0 +1,226 @@ +# fixture 1 +GET / non-media +HEAD / non-media +GET /Artists non-media +GET /Branding/Configuration non-media +POST /ClientLog/Document non-media +GET /DisplayPreferences/{displayPreferencesId} non-media +POST /DisplayPreferences/{displayPreferencesId} non-media +GET /Episode/{id}/IntroTimestamps non-media +GET /Episode/{id}/Timestamps non-media +GET /Genres non-media +GET /Genres/{name} non-media +GET /Items non-media +GET /Items/Filters non-media +GET /Items/Filters2 non-media +GET /Items/Latest non-media +GET /Items/Suggestions non-media +GET /Items/{id} non-media +GET /Items/{id}/Download media transfer viewer_egress false false +HEAD /Items/{id}/Download media transfer viewer_egress false false +GET /Items/{id}/Images/{imageType} non-media +GET /Items/{id}/Images/{imageType}/{index} non-media +GET /Items/{id}/Intros non-media +GET /Items/{id}/LocalTrailers non-media +GET /Items/{id}/PlaybackInfo non-media +POST /Items/{id}/PlaybackInfo non-media +GET /Items/{id}/Similar non-media +GET /Items/{id}/SpecialFeatures non-media +GET /Items/{id}/ThemeMedia non-media +GET /Items/{id}/ThemeSongs non-media +GET /Library/VirtualFolders non-media +GET /MediaSegments/{id} non-media +GET /Movies/Recommendations non-media +GET /Movies/{id}/Similar non-media +GET /Persons non-media +GET /Playback/BitrateTest media transfer viewer_egress false false +GET /QuickConnect/Enabled non-media +GET /Search/Hints non-media +GET /Sessions non-media +POST /Sessions/Capabilities non-media +POST /Sessions/Capabilities/Full non-media +POST /Sessions/Logout non-media +POST /Sessions/Playing non-media +POST /Sessions/Playing/Progress non-media +POST /Sessions/Playing/Stopped non-media +GET /Shows/NextUp non-media +GET /Shows/Upcoming non-media +GET /Shows/{id}/Episodes non-media +GET /Shows/{id}/Seasons non-media +GET /Shows/{id}/Similar non-media +GET /Studios non-media +GET /System/Endpoint non-media +GET /System/Info non-media +GET /System/Info/Public non-media +HEAD /System/Info/Public non-media +GET /System/Ping non-media +HEAD /System/Ping non-media +DELETE /UserFavoriteItems/{itemId} non-media +POST /UserFavoriteItems/{itemId} non-media +GET /UserImage non-media +HEAD /UserImage non-media +GET /UserItems/Resume non-media +GET /UserItems/{itemId}/UserData non-media +DELETE /UserPlayedItems/{itemId} non-media +POST /UserPlayedItems/{itemId} non-media +GET /UserViews non-media +GET /UserViews/GroupingOptions non-media +GET /Users non-media +POST /Users/AuthenticateByName non-media +GET /Users/Me non-media +GET /Users/Public non-media +GET /Users/{id} non-media +GET /Users/{id}/Images/Primary non-media +HEAD /Users/{id}/Images/Primary non-media +GET /Users/{id}/Items non-media +GET /Users/{id}/Items/Latest non-media +DELETE /Users/{userId}/FavoriteItems/{itemId} non-media +POST /Users/{userId}/FavoriteItems/{itemId} non-media +GET /Users/{userId}/Items/Resume non-media +GET /Users/{userId}/Items/{id} non-media +GET /Users/{userId}/Items/{id}/Intros non-media +GET /Users/{userId}/Items/{id}/LocalTrailers non-media +GET /Users/{userId}/Items/{id}/PlaybackInfo non-media +POST /Users/{userId}/Items/{id}/PlaybackInfo non-media +GET /Users/{userId}/Items/{id}/SpecialFeatures non-media +GET /Users/{userId}/Items/{id}/ThemeMedia non-media +GET /Users/{userId}/Items/{id}/ThemeSongs non-media +GET /Users/{userId}/Items/{itemId}/UserData non-media +POST /Users/{userId}/Items/{itemId}/UserData non-media +DELETE /Users/{userId}/PlayedItems/{itemId} non-media +POST /Users/{userId}/PlayedItems/{itemId} non-media +GET /Users/{userId}/Views non-media +DELETE /Videos/ActiveEncodings non-media +GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true false +GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true false +GET /Videos/{id}/master.m3u8 media manifest viewer_egress true false +GET /Videos/{id}/stream media playback viewer_egress true false +HEAD /Videos/{id}/stream media playback viewer_egress true false +GET /Videos/{id}/stream.{container} media playback viewer_egress true false +HEAD /Videos/{id}/stream.{container} media playback viewer_egress true false +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /socket non-media +GET /web non-media +CONNECT /web/* non-media +DELETE /web/* non-media +GET /web/* non-media +HEAD /web/* non-media +OPTIONS /web/* non-media +PATCH /web/* non-media +POST /web/* non-media +PUT /web/* non-media +TRACE /web/* non-media +# fixture 2 +GET / non-media +HEAD / non-media +GET /Artists non-media +GET /Branding/Configuration non-media +POST /ClientLog/Document non-media +GET /DisplayPreferences/{displayPreferencesId} non-media +POST /DisplayPreferences/{displayPreferencesId} non-media +GET /Episode/{id}/IntroTimestamps non-media +GET /Episode/{id}/Timestamps non-media +GET /Genres non-media +GET /Genres/{name} non-media +GET /Items non-media +GET /Items/Filters non-media +GET /Items/Filters2 non-media +GET /Items/Latest non-media +GET /Items/Suggestions non-media +GET /Items/{id} non-media +GET /Items/{id}/Download media transfer viewer_egress false false +HEAD /Items/{id}/Download media transfer viewer_egress false false +GET /Items/{id}/Images/{imageType} non-media +GET /Items/{id}/Images/{imageType}/{index} non-media +GET /Items/{id}/Intros non-media +GET /Items/{id}/LocalTrailers non-media +GET /Items/{id}/PlaybackInfo non-media +POST /Items/{id}/PlaybackInfo non-media +GET /Items/{id}/Similar non-media +GET /Items/{id}/SpecialFeatures non-media +GET /Items/{id}/ThemeMedia non-media +GET /Items/{id}/ThemeSongs non-media +GET /Library/VirtualFolders non-media +GET /MediaSegments/{id} non-media +GET /Movies/Recommendations non-media +GET /Movies/{id}/Similar non-media +GET /Persons non-media +GET /Playback/BitrateTest media transfer viewer_egress false false +GET /QuickConnect/Enabled non-media +GET /Search/Hints non-media +GET /Sessions non-media +POST /Sessions/Capabilities non-media +POST /Sessions/Capabilities/Full non-media +POST /Sessions/Logout non-media +POST /Sessions/Playing non-media +POST /Sessions/Playing/Progress non-media +POST /Sessions/Playing/Stopped non-media +GET /Shows/NextUp non-media +GET /Shows/Upcoming non-media +GET /Shows/{id}/Episodes non-media +GET /Shows/{id}/Seasons non-media +GET /Shows/{id}/Similar non-media +GET /Studios non-media +GET /System/Endpoint non-media +GET /System/Info non-media +GET /System/Info/Public non-media +HEAD /System/Info/Public non-media +GET /System/Ping non-media +HEAD /System/Ping non-media +DELETE /UserFavoriteItems/{itemId} non-media +POST /UserFavoriteItems/{itemId} non-media +GET /UserImage non-media +HEAD /UserImage non-media +GET /UserItems/Resume non-media +GET /UserItems/{itemId}/UserData non-media +DELETE /UserPlayedItems/{itemId} non-media +POST /UserPlayedItems/{itemId} non-media +GET /UserViews non-media +GET /UserViews/GroupingOptions non-media +GET /Users non-media +POST /Users/AuthenticateByName non-media +GET /Users/Me non-media +GET /Users/Public non-media +GET /Users/{id} non-media +GET /Users/{id}/Images/Primary non-media +HEAD /Users/{id}/Images/Primary non-media +GET /Users/{id}/Items non-media +GET /Users/{id}/Items/Latest non-media +DELETE /Users/{userId}/FavoriteItems/{itemId} non-media +POST /Users/{userId}/FavoriteItems/{itemId} non-media +GET /Users/{userId}/Items/Resume non-media +GET /Users/{userId}/Items/{id} non-media +GET /Users/{userId}/Items/{id}/Intros non-media +GET /Users/{userId}/Items/{id}/LocalTrailers non-media +GET /Users/{userId}/Items/{id}/PlaybackInfo non-media +POST /Users/{userId}/Items/{id}/PlaybackInfo non-media +GET /Users/{userId}/Items/{id}/SpecialFeatures non-media +GET /Users/{userId}/Items/{id}/ThemeMedia non-media +GET /Users/{userId}/Items/{id}/ThemeSongs non-media +GET /Users/{userId}/Items/{itemId}/UserData non-media +POST /Users/{userId}/Items/{itemId}/UserData non-media +DELETE /Users/{userId}/PlayedItems/{itemId} non-media +POST /Users/{userId}/PlayedItems/{itemId} non-media +GET /Users/{userId}/Views non-media +DELETE /Videos/ActiveEncodings non-media +GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true false +GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true false +GET /Videos/{id}/master.m3u8 media manifest viewer_egress true false +GET /Videos/{id}/stream media playback viewer_egress true false +HEAD /Videos/{id}/stream media playback viewer_egress true false +GET /Videos/{id}/stream.{container} media playback viewer_egress true false +HEAD /Videos/{id}/stream.{container} media playback viewer_egress true false +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /socket non-media +GET /web non-media +CONNECT /web/* non-media +DELETE /web/* non-media +GET /web/* non-media +HEAD /web/* non-media +OPTIONS /web/* non-media +PATCH /web/* non-media +POST /web/* non-media +PUT /web/* non-media +TRACE /web/* non-media diff --git a/internal/proxy/media_routes.go b/internal/proxy/media_routes.go new file mode 100644 index 000000000..64fa1f263 --- /dev/null +++ b/internal/proxy/media_routes.go @@ -0,0 +1,29 @@ +package proxy + +import ( + "net/http" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var proxyMediaRoutes = []streamtelemetry.MediaRoute{ + proxyRoute(http.MethodGet, "/stream/direct/{token}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodHead, "/stream/direct/{token}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodGet, "/stream/remux/{token}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodHead, "/stream/remux/{token}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodGet, "/stream/transcode/{token}/master.m3u8", streamtelemetry.ClassManifest, true), + proxyRoute(http.MethodHead, "/stream/transcode/{token}/master.m3u8", streamtelemetry.ClassManifest, true), + proxyRoute(http.MethodGet, "/stream/transcode/{token}/segment/{name}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodGet, "/stream/subtitles/{token}/{track}", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodGet, "/stream/subtitles/{token}/{track}/fonts", streamtelemetry.ClassPlayback, true), + proxyRoute(http.MethodGet, "/downloads/file/{token}", streamtelemetry.ClassTransfer, false), + proxyRoute(http.MethodHead, "/downloads/file/{token}", streamtelemetry.ClassTransfer, false), +} + +func proxyRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { + return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyProxy, Method: method, Pattern: pattern, + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "verified_stream_token", + CapRelevant: capRelevant, Enrolled: false} +} + +func declareProxyMediaRoutes() { streamtelemetry.DeclareRoutes(proxyMediaRoutes...) } diff --git a/internal/proxy/media_routes_test.go b/internal/proxy/media_routes_test.go new file mode 100644 index 000000000..ca238a35b --- /dev/null +++ b/internal/proxy/media_routes_test.go @@ -0,0 +1,51 @@ +package proxy + +import ( + "flag" + "os" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var updateRouteManifest = flag.Bool("update-route-manifest", false, "update checked-in route manifest") + +func TestMediaRouteManifest(t *testing.T) { + declareProxyMediaRoutes() + makeRouter := func() chi.Routes { + return NewServer(nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}), nodesessions.NewTracker(nil, "", "", "")).Handler().(chi.Routes) + } + assertMediaManifest(t, []chi.Routes{makeRouter(), makeRouter()}, proxyMediaRoutes, "testdata/media_routes.txt") +} + +func assertMediaManifest(t *testing.T, fixtures []chi.Routes, declared []streamtelemetry.MediaRoute, path string) { + t.Helper() + actual, err := streamtelemetry.BuildRouteManifest(fixtures, declared) + if err != nil { + t.Fatal(err) + } + if *updateRouteManifest { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(actual), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != actual { + t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") + } + for _, route := range declared { + if route.Enrolled { + t.Fatalf("non-native route enrolled: %s %s", route.Method, route.Pattern) + } + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 7baf192ae..0f210ad50 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -101,6 +101,7 @@ func newStreamTransport() *http.Transport { // Handler returns the chi.Router with all proxy routes mounted. func (s *Server) Handler() http.Handler { + declareProxyMediaRoutes() r := chi.NewRouter() if s.clientIP != nil { r.Use(clientip.Middleware(s.clientIP)) diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt new file mode 100644 index 000000000..c069fab55 --- /dev/null +++ b/internal/proxy/testdata/media_routes.txt @@ -0,0 +1,32 @@ +# fixture 1 +POST /admin/force-reload non-media +GET /api/v1/health non-media +GET /downloads/file/{token} media transfer viewer_egress false false +HEAD /downloads/file/{token} media transfer viewer_egress false false +GET /hw-capabilities non-media +GET /status non-media +GET /stream/direct/{token} media playback viewer_egress true false +HEAD /stream/direct/{token} media playback viewer_egress true false +GET /stream/remux/{token} media playback viewer_egress true false +HEAD /stream/remux/{token} media playback viewer_egress true false +GET /stream/subtitles/{token}/{track} media playback viewer_egress true false +GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true false +GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false +HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false +GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true false +# fixture 2 +POST /admin/force-reload non-media +GET /api/v1/health non-media +GET /downloads/file/{token} media transfer viewer_egress false false +HEAD /downloads/file/{token} media transfer viewer_egress false false +GET /hw-capabilities non-media +GET /status non-media +GET /stream/direct/{token} media playback viewer_egress true false +HEAD /stream/direct/{token} media playback viewer_egress true false +GET /stream/remux/{token} media playback viewer_egress true false +HEAD /stream/remux/{token} media playback viewer_egress true false +GET /stream/subtitles/{token}/{track} media playback viewer_egress true false +GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true false +GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false +HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false +GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true false diff --git a/internal/streamtelemetry/benchmark_test.go b/internal/streamtelemetry/benchmark_test.go new file mode 100644 index 000000000..cbe233950 --- /dev/null +++ b/internal/streamtelemetry/benchmark_test.go @@ -0,0 +1,96 @@ +package streamtelemetry + +import ( + "bytes" + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +const benchmarkBodySize = 8 << 20 + +func benchmarkRegistry(enabled bool) *Registry { + cfg := DefaultConfig("benchmark") + cfg.Enabled = enabled + cfg.MaxObservationsPerSession = 4096 + return NewRegistry(cfg, NewLocalStore(), nil) +} + +func benchmarkHandler(registry *Registry, progressive bool) http.Handler { + route := MediaRoute{Family: FamilyNative, Method: http.MethodGet, Pattern: "/media/{id}", + Class: ClassPlayback, Role: RoleViewerEgress, CapRelevant: true, Enrolled: true, + Capture: func(r *http.Request) CaptureSet { return CaptureSet{Method: r.Method, Pattern: "/media/{id}"} }} + body := bytes.Repeat([]byte{'x'}, benchmarkBodySize) + return registry.Observe(route)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("benchmark-session")) + if progressive { + for offset := 0; offset < len(body); offset += 32 << 10 { + end := min(offset+(32<<10), len(body)) + if _, err := w.Write(body[offset:end]); err != nil { + return + } + } + return + } + _, _ = io.Copy(w, bytes.NewReader(body)) + })) +} + +func runHTTPBenchmark(b *testing.B, enabled, progressive, http2 bool, collector bool) { + registry := benchmarkRegistry(enabled) + if collector { + registry.Start(b.Context()) + } + server := httptest.NewUnstartedServer(benchmarkHandler(registry, progressive)) + server.EnableHTTP2 = http2 + if http2 { + server.StartTLS() + } else { + server.Start() + } + b.Cleanup(server.Close) + client := server.Client() + if transport, ok := client.Transport.(*http.Transport); ok { + transport.DisableCompression = true + if http2 { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } //nolint:gosec // benchmark server + } + b.Cleanup(client.CloseIdleConnections) + b.SetBytes(benchmarkBodySize) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + response, err := client.Get(server.URL + "/media/1") + if err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, response.Body); err != nil { + b.Fatal(err) + } + _ = response.Body.Close() + } +} + +func BenchmarkDirectPlay(b *testing.B) { + b.Run("disabled", func(b *testing.B) { runHTTPBenchmark(b, false, false, false, false) }) + b.Run("enabled", func(b *testing.B) { runHTTPBenchmark(b, true, false, false, false) }) +} + +func BenchmarkRemuxWrite(b *testing.B) { + b.Run("disabled", func(b *testing.B) { runHTTPBenchmark(b, false, true, false, false) }) + b.Run("enabled", func(b *testing.B) { runHTTPBenchmark(b, true, true, false, false) }) +} + +func BenchmarkHLSSegmentRPS(b *testing.B) { + b.Run("disabled", func(b *testing.B) { runHTTPBenchmark(b, false, false, false, false) }) + b.Run("enabled", func(b *testing.B) { runHTTPBenchmark(b, true, false, false, false) }) + b.Run("enabled_with_collector", func(b *testing.B) { runHTTPBenchmark(b, true, false, false, true) }) +} + +func BenchmarkHTTP2Write(b *testing.B) { + b.Run("disabled", func(b *testing.B) { runHTTPBenchmark(b, false, true, true, false) }) + b.Run("enabled", func(b *testing.B) { runHTTPBenchmark(b, true, true, true, false) }) +} diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go new file mode 100644 index 000000000..9356720fa --- /dev/null +++ b/internal/streamtelemetry/config.go @@ -0,0 +1,103 @@ +package streamtelemetry + +import ( + "log/slog" + "os" + "strconv" + "strings" + "time" +) + +const ( + enabledEnv = "SILO_STREAM_TELEMETRY_ENABLED" + sweepIntervalEnv = "SILO_STREAM_TELEMETRY_SWEEP_INTERVAL" + retentionEnv = "SILO_STREAM_TELEMETRY_RETENTION" + maxSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_SESSIONS" + maxTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_TRANSFERS" + maxObservationsEnv = "SILO_STREAM_TELEMETRY_MAX_OBSERVATIONS" +) + +type Config struct { + Enabled bool + NodeID string + PublisherID string + + SweepInterval time.Duration + Retention time.Duration + + MaxSessions int64 + MaxTransfers int64 + MaxObservations int64 + MaxObservationsPerSession int + MaxViewerIPsPerSession int + MaxIdentityConflictsPerSession int + MaxDeviceIDsPerSession int + MaxClientVariantsPerSession int + MaxMediaFileIDsPerSession int + MaxPlayMethodsPerSession int + MaxTokenIssuedAtPerSession int + MaxRoutesPerSession int +} + +func DefaultConfig(nodeID string) Config { + return Config{ + NodeID: nodeID, SweepInterval: time.Second, Retention: 5 * time.Minute, + MaxSessions: 10_000, MaxTransfers: 10_000, MaxObservations: 50_000, + MaxObservationsPerSession: 64, MaxViewerIPsPerSession: 32, + MaxIdentityConflictsPerSession: 16, MaxDeviceIDsPerSession: 32, + MaxClientVariantsPerSession: 16, MaxMediaFileIDsPerSession: 32, + MaxPlayMethodsPerSession: 16, MaxTokenIssuedAtPerSession: 32, + MaxRoutesPerSession: 32, + } +} + +// ConfigFromEnv returns a safe configuration. Invalid telemetry settings are +// ignored while disabled; while enabled they disable telemetry and are logged. +func ConfigFromEnv(nodeID string) Config { + cfg := DefaultConfig(nodeID) + cfg.Enabled = envEnabled(os.Getenv(enabledEnv)) + invalid := make([]string, 0) + parseDuration := func(name string, dst *time.Duration) { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed <= 0 { + invalid = append(invalid, name) + return + } + *dst = parsed + } + parsePositive := func(name string, dst *int64) { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 { + invalid = append(invalid, name) + return + } + *dst = parsed + } + parseDuration(sweepIntervalEnv, &cfg.SweepInterval) + parseDuration(retentionEnv, &cfg.Retention) + parsePositive(maxSessionsEnv, &cfg.MaxSessions) + parsePositive(maxTransfersEnv, &cfg.MaxTransfers) + parsePositive(maxObservationsEnv, &cfg.MaxObservations) + if len(invalid) > 0 { + if cfg.Enabled { + cfg.Enabled = false + slog.Error("stream telemetry disabled because configuration is invalid", "variables", strings.Join(invalid, ",")) + } else { + slog.Warn("ignoring invalid disabled stream telemetry configuration", "variables", strings.Join(invalid, ",")) + } + } + return cfg +} + +func envEnabled(value string) bool { + value = strings.TrimSpace(strings.ToLower(value)) + return value == "1" || value == "true" || value == "yes" || value == "on" +} diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go new file mode 100644 index 000000000..8916676e5 --- /dev/null +++ b/internal/streamtelemetry/config_test.go @@ -0,0 +1,50 @@ +package streamtelemetry + +import ( + "testing" + "time" +) + +func TestConfigFromEnvValidation(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + clearConfigEnv(t) + cfg := ConfigFromEnv("node") + if cfg.Enabled || cfg.SweepInterval != time.Second || cfg.Retention != 5*time.Minute || cfg.MaxObservations != 50_000 { + t.Fatalf("defaults = %+v", cfg) + } + }) + t.Run("valid enabled overrides", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(sweepIntervalEnv, "250ms") + t.Setenv(retentionEnv, "6m") + t.Setenv(maxSessionsEnv, "12") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.SweepInterval != 250*time.Millisecond || cfg.Retention != 6*time.Minute || cfg.MaxSessions != 12 { + t.Fatalf("overrides = %+v", cfg) + } + }) + t.Run("invalid enabled disables", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(sweepIntervalEnv, "0s") + if cfg := ConfigFromEnv("node"); cfg.Enabled { + t.Fatalf("invalid config remained enabled: %+v", cfg) + } + }) + t.Run("invalid disabled is ignored", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(maxTransfersEnv, "not-a-number") + cfg := ConfigFromEnv("node") + if cfg.Enabled || cfg.MaxTransfers != 10_000 { + t.Fatalf("disabled invalid config = %+v", cfg) + } + }) +} + +func clearConfigEnv(t *testing.T) { + t.Helper() + for _, name := range []string{enabledEnv, sweepIntervalEnv, retentionEnv, maxSessionsEnv, maxTransfersEnv, maxObservationsEnv} { + t.Setenv(name, "") + } +} diff --git a/internal/streamtelemetry/doc.go b/internal/streamtelemetry/doc.go new file mode 100644 index 000000000..6e5c63ac2 --- /dev/null +++ b/internal/streamtelemetry/doc.go @@ -0,0 +1,17 @@ +// Package streamtelemetry records local, observation-only telemetry for media +// responses. +// +// Observe starts a provisional per-request Observation. A handler promotes it +// with Attach only after it has loaded and authorized the canonical playback +// session or transfer owner. A request that never attaches cannot create logical +// activity; its accepted bytes are instead reported as unattributed. Released +// observations fold their final byte count into a retained LogicalSession or +// Transfer, so short requests that fit between collector sweeps are not lost. +// +// BytesAccepted means response body bytes accepted by the writer at the point +// where Observe is enrolled. It is wire bytes on bulk routes that bypass outer +// compression, and pre-compression bytes on compressible subtitle/font routes. +// +// P0b is deliberately local and observational. This package performs no +// admission, throttling, cutting, persistence, or distributed publication. +package streamtelemetry diff --git a/internal/streamtelemetry/identity.go b/internal/streamtelemetry/identity.go new file mode 100644 index 000000000..2293c2c2d --- /dev/null +++ b/internal/streamtelemetry/identity.go @@ -0,0 +1,77 @@ +package streamtelemetry + +import ( + "strconv" + "time" +) + +type SubjectKind string + +const ( + SubjectUser SubjectKind = "user" + SubjectABSUser SubjectKind = "abs_user" + SubjectIP SubjectKind = "ip" +) + +type Subject struct { + Kind SubjectKind + ID string +} + +func UserSubject(id int) Subject { + return Subject{Kind: SubjectUser, ID: strconv.Itoa(id)} +} + +type StartedAtSource string + +const ( + StartedAtSourceClaim StartedAtSource = "claim" + StartedAtSourceSession StartedAtSource = "session" + StartedAtSourceIssuedAt StartedAtSource = "issued_at" + StartedAtSourceFirstSeen StartedAtSource = "first_seen" +) + +type TokenIssuedAtSource string + +const ( + TokenIssuedAtSourceNone TokenIssuedAtSource = "none" + TokenIssuedAtSourceVerified TokenIssuedAtSource = "verified" +) + +type ClientVariant struct { + Name string + Version string + Build string + Channel string +} + +type CaptureSet struct { + Method string + Pattern string + ViewerIP string + DeviceID string + Client ClientVariant + UserAgent string + ReceivedAt time.Time + TokenIssuedAt time.Time + TokenIssuedFrom TokenIssuedAtSource +} + +type Attachment struct { + Subject Subject + ProfileID string + SessionID string + MediaFileID int + PlayMethod string + StartedAt time.Time + StartedAtSource StartedAtSource + TokenIssuedAt time.Time + TokenIssuedAtSource TokenIssuedAtSource +} + +type IdentityConflict struct { + Field string + Existing string + Offered string + ObservedAt time.Time +} diff --git a/internal/streamtelemetry/manifest.go b/internal/streamtelemetry/manifest.go new file mode 100644 index 000000000..e8493a357 --- /dev/null +++ b/internal/streamtelemetry/manifest.go @@ -0,0 +1,79 @@ +package streamtelemetry + +import ( + "errors" + "fmt" + "net/http" + "sort" + "strings" + + "github.com/go-chi/chi/v5" +) + +type WalkedRoute struct { + Method string + Pattern string +} + +func WalkRoutes(router chi.Routes) ([]WalkedRoute, error) { + var routes []WalkedRoute + err := chi.Walk(router, func(method, pattern string, _ http.Handler, _ ...func(http.Handler) http.Handler) error { + routes = append(routes, WalkedRoute{Method: method, Pattern: pattern}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(routes, func(i, j int) bool { + if routes[i].Pattern == routes[j].Pattern { + return routes[i].Method < routes[j].Method + } + return routes[i].Pattern < routes[j].Pattern + }) + return routes, nil +} + +func FormatManifest(routes []WalkedRoute, media []MediaRoute) string { + declared := make(map[string]MediaRoute, len(media)) + for _, route := range media { + declared[route.Method+" "+route.Pattern] = route + } + var b strings.Builder + for _, route := range routes { + key := route.Method + " " + route.Pattern + if mediaRoute, ok := declared[key]; ok { + fmt.Fprintf(&b, "%s\tmedia\t%s\t%s\t%t\t%t\n", key, mediaRoute.Class, mediaRoute.Role, mediaRoute.CapRelevant, mediaRoute.Enrolled) + } else { + fmt.Fprintf(&b, "%s\tnon-media\n", key) + } + } + return b.String() +} + +// BuildRouteManifest walks minimal and maximal fixture routers, renders the +// complete declared-or-non-media classification, and verifies that their union +// covers every declared media route. +func BuildRouteManifest(routers []chi.Routes, media []MediaRoute) (string, error) { + if len(routers) != 2 { + return "", errors.New("route manifest requires minimal and maximal fixtures") + } + seen := make(map[string]struct{}) + var b strings.Builder + for index, router := range routers { + routes, err := WalkRoutes(router) + if err != nil { + return "", err + } + fmt.Fprintf(&b, "# fixture %d\n", index+1) + b.WriteString(FormatManifest(routes, media)) + for _, route := range routes { + seen[route.Method+" "+route.Pattern] = struct{}{} + } + } + for _, route := range media { + if _, ok := seen[route.Method+" "+route.Pattern]; !ok { + return "", fmt.Errorf("declared media route was not walked: %s %s", route.Method, route.Pattern) + } + } + return b.String(), nil +} diff --git a/internal/streamtelemetry/observation.go b/internal/streamtelemetry/observation.go new file mode 100644 index 000000000..764cc85a6 --- /dev/null +++ b/internal/streamtelemetry/observation.go @@ -0,0 +1,90 @@ +package streamtelemetry + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/google/uuid" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type observationContextKey struct{} + +type Observation struct { + id string + registry *Registry + route MediaRoute + Capture CaptureSet + + bytesAccepted atomic.Int64 + cut atomic.Bool + + mu sync.Mutex + attachment *Attachment + target observationTarget + firstWriteErr error + released bool + countingOnly bool + reserved bool +} + +type observationTarget struct { + session *logicalSession + transfer *transfer +} + +func (o *Observation) AddBytes(n int64) { + if o != nil && n > 0 { + o.bytesAccepted.Add(n) + } +} + +func (o *Observation) BytesAccepted() int64 { + if o == nil { + return 0 + } + return o.bytesAccepted.Load() +} + +func (o *Observation) recordWriteError(err error) { + if o == nil || err == nil { + return + } + o.mu.Lock() + if o.firstWriteErr == nil { + o.firstWriteErr = err + } + o.mu.Unlock() +} + +func (o *Observation) outcome(ctxErr error, completed bool) httpstream.StreamOutcome { + if !completed { + return OutcomeUnknown + } + o.mu.Lock() + err := o.firstWriteErr + o.mu.Unlock() + return httpstream.ClassifyOutcome(err, ctxErr) +} + +func Attach(ctx context.Context, attachment Attachment) { + if ctx == nil { + return + } + obs, _ := ctx.Value(observationContextKey{}).(*Observation) + if obs == nil || obs.registry == nil { + return + } + obs.registry.attach(obs, attachment) +} + +func newObservation(registry *Registry, route MediaRoute, capture CaptureSet) *Observation { + return &Observation{ + id: uuid.NewString(), + registry: registry, + route: route, + Capture: capture, + } +} diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go new file mode 100644 index 000000000..8d3c756b3 --- /dev/null +++ b/internal/streamtelemetry/registry.go @@ -0,0 +1,408 @@ +package streamtelemetry + +import ( + "context" + "hash/maphash" + "log/slog" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +const shardCount = 32 + +var now = time.Now + +type sessionShard struct { + sync.RWMutex + sessions map[string]*logicalSession +} + +type Registry struct { + cfg Config + store SnapshotStore + logger *slog.Logger + seed maphash.Seed + shards [shardCount]sessionShard + + transfersMu sync.RWMutex + transfers map[string]*transfer + + sessionReservations atomic.Int64 + transferReservations atomic.Int64 + observationReservations atomic.Int64 + droppedObservations atomic.Int64 + droppedBytes atomic.Int64 + unattributedObservations atomic.Int64 + unattributedBytes atomic.Int64 + truncated atomic.Bool + lastWarnUnixNano atomic.Int64 + lastPublishWarnUnixNano atomic.Int64 +} + +func NewRegistry(cfg Config, store SnapshotStore, logger *slog.Logger) *Registry { + if cfg.PublisherID == "" { + cfg.PublisherID = uuid.NewString() + } + if store == nil { + store = NewLocalStore() + } + if logger == nil { + logger = slog.Default() + } + r := &Registry{cfg: cfg, store: store, logger: logger, seed: maphash.MakeSeed(), transfers: make(map[string]*transfer)} + for i := range r.shards { + r.shards[i].sessions = make(map[string]*logicalSession) + } + return r +} + +func (r *Registry) Enabled() bool { return r != nil && r.cfg.Enabled } + +func (r *Registry) Store() SnapshotStore { + if r == nil { + return nil + } + return r.store +} + +func reserve(counter *atomic.Int64, max int64) bool { + for { + current := counter.Load() + if current >= max { + return false + } + if counter.CompareAndSwap(current, current+1) { + return true + } + } +} + +func (r *Registry) begin(route MediaRoute, capture CaptureSet) *Observation { + obs := newObservation(r, route, capture) + if reserve(&r.observationReservations, r.cfg.MaxObservations) { + obs.reserved = true + } else { + obs.countingOnly = true + r.drop("observation capacity exhausted") + } + return obs +} + +func (r *Registry) attach(obs *Observation, attachment Attachment) { + obs.mu.Lock() + defer obs.mu.Unlock() + if obs.released || obs.countingOnly { + return + } + observedAt := obs.Capture.ReceivedAt + if observedAt.IsZero() { + observedAt = now() + } + if obs.attachment != nil { + if obs.target.session != nil { + s := obs.target.session + s.mu.Lock() + s.recordConflicts(attachment, observedAt, r.cfg.MaxIdentityConflictsPerSession) + s.mu.Unlock() + } + return + } + if attachment.TokenIssuedAt.IsZero() && !obs.Capture.TokenIssuedAt.IsZero() { + attachment.TokenIssuedAt = obs.Capture.TokenIssuedAt + attachment.TokenIssuedAtSource = obs.Capture.TokenIssuedFrom + } + if attachment.TokenIssuedAtSource == "" { + attachment.TokenIssuedAtSource = TokenIssuedAtSourceNone + } + if obs.route.Class == ClassTransfer { + if !reserve(&r.transferReservations, r.cfg.MaxTransfers) { + obs.countingOnly = true + r.drop("transfer capacity exhausted") + return + } + t := &transfer{id: obs.id, subject: attachment.Subject, profileID: attachment.ProfileID, + mediaFileID: attachment.MediaFileID, openObservations: 1, requestCount: 1, + route: obs.route, capture: obs.Capture, observation: obs, + outcomes: make(map[httpstream.StreamOutcome]int64)} + r.transfersMu.Lock() + r.transfers[t.id] = t + r.transfersMu.Unlock() + obs.attachment = &attachment + obs.target.transfer = t + return + } + if attachment.SessionID == "" { + obs.countingOnly = true + r.drop("attachment has no canonical session id") + return + } + shard := r.shard(attachment.SessionID) + shard.Lock() + s := shard.sessions[attachment.SessionID] + if s == nil { + if !reserve(&r.sessionReservations, r.cfg.MaxSessions) { + shard.Unlock() + obs.countingOnly = true + r.drop("session capacity exhausted") + return + } + s = newLogicalSession(attachment, r.cfg, observedAt) + shard.sessions[attachment.SessionID] = s + } + s.mu.Lock() + if len(s.observations) >= r.cfg.MaxObservationsPerSession { + s.mu.Unlock() + shard.Unlock() + obs.countingOnly = true + r.drop("per-session observation capacity exhausted") + return + } + s.recordConflicts(attachment, observedAt, r.cfg.MaxIdentityConflictsPerSession) + key := routeID(obs.Capture.Method, obs.Capture.Pattern) + activity := s.routes[key] + if activity == nil { + if len(s.routes) >= r.cfg.MaxRoutesPerSession { + s.routesOverflowed = true + } else { + activity = &routeActivity{Method: obs.Capture.Method, Pattern: obs.Capture.Pattern, + Role: obs.route.Role, Class: obs.route.Class, CapRelevant: obs.route.CapRelevant} + s.routes[key] = activity + } + } + s.observations[obs.id] = obs + s.openObservations++ + s.requestCount++ + if activity != nil { + activity.Open++ + activity.Requests++ + } + if obs.Capture.ViewerIP != "" { + s.viewerIPs.add(obs.Capture.ViewerIP) + } + if obs.Capture.DeviceID != "" { + s.deviceIDs.add(obs.Capture.DeviceID) + } + if obs.Capture.Client != (ClientVariant{}) { + s.clientVariants.add(obs.Capture.Client) + } + if obs.Capture.UserAgent != "" { + s.userAgents.add(obs.Capture.UserAgent) + } + s.tokenIssuedSources[attachment.TokenIssuedAtSource]++ + if !attachment.TokenIssuedAt.IsZero() { + s.tokenIssuedAts.add(attachment.TokenIssuedAt.UnixNano()) + } + s.mu.Unlock() + shard.Unlock() + obs.attachment = &attachment + obs.target.session = s +} + +func (r *Registry) release(obs *Observation, outcome httpstream.StreamOutcome) { + obs.mu.Lock() + if obs.released { + obs.mu.Unlock() + return + } + obs.released = true + target := obs.target + attached := obs.attachment != nil + countingOnly := obs.countingOnly + obs.mu.Unlock() + bytes := obs.BytesAccepted() + if countingOnly { + r.droppedBytes.Add(bytes) + } else if !attached { + r.unattributedObservations.Add(1) + r.unattributedBytes.Add(bytes) + } else if target.transfer != nil { + t := target.transfer + t.mu.Lock() + t.bytesFolded += bytes + t.openObservations-- + t.lastObservationEnd = now() + t.outcomes[outcome]++ + t.observation = nil + t.mu.Unlock() + } else if target.session != nil { + s := target.session + s.mu.Lock() + delete(s.observations, obs.id) + s.bytesFolded += bytes + s.openObservations-- + s.lastObservationEnd = now() + s.outcomes[outcome]++ + if activity := s.routes[routeID(obs.Capture.Method, obs.Capture.Pattern)]; activity != nil { + activity.Open-- + activity.BytesFolded += bytes + activity.LastObservationEnd = s.lastObservationEnd + } + s.mu.Unlock() + } + if obs.reserved { + r.observationReservations.Add(-1) + } +} + +func (r *Registry) drop(reason string) { + r.truncated.Store(true) + r.droppedObservations.Add(1) + r.warnRateLimited(reason, &r.lastWarnUnixNano) +} + +func (r *Registry) warnRateLimited(message string, stamp *atomic.Int64, attrs ...any) { + n := now().UnixNano() + for { + previous := stamp.Load() + if previous != 0 && n-previous < int64(time.Minute) { + return + } + if stamp.CompareAndSwap(previous, n) { + attrs = append([]any{"component", "stream_telemetry"}, attrs...) + attrs = append([]any{"reason", message}, attrs...) + r.logger.Warn("stream telemetry warning", attrs...) + return + } + } +} + +func (r *Registry) shard(id string) *sessionShard { + var h maphash.Hash + h.SetSeed(r.seed) + h.WriteString(id) + return &r.shards[h.Sum64()%shardCount] +} + +func (r *Registry) SetRealtimeConnection(sessionID string, connected bool) { + if r == nil || !r.cfg.Enabled || sessionID == "" { + return + } + shard := r.shard(sessionID) + shard.RLock() + s := shard.sessions[sessionID] + if s != nil { + s.mu.Lock() + s.realtimeAlive = connected + s.mu.Unlock() + } + shard.RUnlock() +} + +func (r *Registry) Start(ctx context.Context) { + if r == nil || !r.cfg.Enabled { + return + } + go func() { + ticker := time.NewTicker(r.cfg.SweepInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case sweepStart := <-ticker.C: + snapshot := r.sweep(sweepStart) + if err := r.store.Publish(ctx, snapshot); err != nil { + r.warnRateLimited("failed to publish stream telemetry snapshot", &r.lastPublishWarnUnixNano, "error", err) + } + } + } + }() +} + +func (r *Registry) Sweep() Snapshot { return r.sweep(now()) } + +func (r *Registry) sweep(sweepStart time.Time) Snapshot { + for i := range r.shards { + shard := &r.shards[i] + shard.Lock() + for id, s := range shard.sessions { + s.mu.Lock() + total := s.bytesFolded + routeTotals := make(map[string]int64, len(s.routes)) + for key, activity := range s.routes { + routeTotals[key] = activity.BytesFolded + } + for _, obs := range s.observations { + bytes := obs.BytesAccepted() + total += bytes + key := routeID(obs.Capture.Method, obs.Capture.Pattern) + if _, tracked := s.routes[key]; tracked { + routeTotals[key] += bytes + } + } + if total > s.lastSweptBytes { + s.lastByteAccepted = sweepStart + } + s.lastSweptBytes = total + for key, totalForRoute := range routeTotals { + activity := s.routes[key] + if totalForRoute > activity.LastSweptBytes { + activity.LastByteAccepted = sweepStart + } + activity.LastSweptBytes = totalForRoute + } + prune := s.openObservations == 0 && !s.lastObservationEnd.IsZero() && sweepStart.Sub(s.lastObservationEnd) >= r.cfg.Retention + s.mu.Unlock() + if prune { + delete(shard.sessions, id) + r.sessionReservations.Add(-1) + } + } + shard.Unlock() + } + r.transfersMu.Lock() + for id, t := range r.transfers { + t.mu.Lock() + total := t.bytesFolded + if t.observation != nil { + total += t.observation.BytesAccepted() + } + if total > t.lastSweptBytes { + t.lastByteAccepted = sweepStart + } + t.lastSweptBytes = total + prune := t.openObservations == 0 && !t.lastObservationEnd.IsZero() && sweepStart.Sub(t.lastObservationEnd) >= r.cfg.Retention + t.mu.Unlock() + if prune { + delete(r.transfers, id) + r.transferReservations.Add(-1) + } + } + r.transfersMu.Unlock() + return r.SnapshotAt(sweepStart) +} + +func (r *Registry) Snapshot() Snapshot { return r.SnapshotAt(now()) } + +func (r *Registry) SnapshotAt(capturedAt time.Time) Snapshot { + view := Snapshot{PublisherID: r.cfg.PublisherID, NodeID: r.cfg.NodeID, CapturedAt: capturedAt, + Truncated: r.truncated.Load(), DroppedObservations: r.droppedObservations.Load(), + DroppedBytes: r.droppedBytes.Load(), UnattributedObservations: r.unattributedObservations.Load(), + UnattributedBytes: r.unattributedBytes.Load()} + for i := range r.shards { + shard := &r.shards[i] + shard.RLock() + for _, s := range shard.sessions { + s.mu.Lock() + view.Sessions = append(view.Sessions, sessionViewOf(s)) + s.mu.Unlock() + } + shard.RUnlock() + } + r.transfersMu.RLock() + for _, t := range r.transfers { + t.mu.Lock() + view.Transfers = append(view.Transfers, transferViewOf(t)) + t.mu.Unlock() + } + r.transfersMu.RUnlock() + sort.Slice(view.Sessions, func(i, j int) bool { return view.Sessions[i].SessionID < view.Sessions[j].SessionID }) + sort.Slice(view.Transfers, func(i, j int) bool { return view.Transfers[i].ID < view.Transfers[j].ID }) + return cloneSnapshot(view) +} diff --git a/internal/streamtelemetry/registry_test.go b/internal/streamtelemetry/registry_test.go new file mode 100644 index 000000000..6403c8f96 --- /dev/null +++ b/internal/streamtelemetry/registry_test.go @@ -0,0 +1,275 @@ +package streamtelemetry + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +func testConfig() Config { + cfg := DefaultConfig("test-node") + cfg.Enabled = true + cfg.PublisherID = "test-publisher" + cfg.Retention = time.Millisecond + return cfg +} + +func testRoute(class Class) MediaRoute { + return MediaRoute{Family: FamilyNative, Method: http.MethodGet, Pattern: "/media/{id}", + Class: class, Role: RoleViewerEgress, CapRelevant: class != ClassTransfer, Enrolled: true} +} + +func testAttachment(id string) Attachment { + return Attachment{Subject: UserSubject(7), ProfileID: "profile", SessionID: id, MediaFileID: 42, + PlayMethod: "direct", StartedAt: time.Unix(100, 0), StartedAtSource: StartedAtSourceSession, + TokenIssuedAtSource: TokenIssuedAtSourceNone} +} + +func TestProvisionalObservationDoesNotCreateLogicalActivity(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), slog.New(slog.DiscardHandler)) + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte("denied")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("provisional request created logical activity: %+v", snapshot) + } + if snapshot.UnattributedObservations != 1 || snapshot.UnattributedBytes != 6 { + t.Fatalf("unattributed counters = %d/%d", snapshot.UnattributedObservations, snapshot.UnattributedBytes) + } +} + +func TestReleaseFoldsShortObservationAndCollectorAdvancesByteClock(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("session-1")) + _, _ = w.Write([]byte("payload")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + before := registry.Snapshot() + if len(before.Sessions) != 1 || before.Sessions[0].OpenObservations != 0 { + t.Fatalf("released session = %+v", before.Sessions) + } + swept := registry.Sweep() + if swept.Sessions[0].BytesAccepted != 7 || swept.Sessions[0].LastByteAccepted.IsZero() { + t.Fatalf("swept session = %+v", swept.Sessions[0]) + } + if got := swept.Sessions[0].Routes[0].BytesAccepted; got != 7 { + t.Fatalf("route bytes = %d", got) + } +} + +func TestReleaseConcurrentWithSweepDoesNotLoseOrDoubleCount(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, testAttachment("session-race")) + obs.AddBytes(4096) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); registry.release(obs, httpstreamOutcomeCompleted) }() + go func() { defer wg.Done(); _ = registry.Sweep() }() + wg.Wait() + snapshot := registry.Sweep() + if got := snapshot.Sessions[0].BytesAccepted; got != 4096 { + t.Fatalf("bytes after concurrent release/sweep = %d", got) + } +} + +const httpstreamOutcomeCompleted = "completed" + +func TestExactObservationBoundServesThroughAndCountsDrops(t *testing.T) { + cfg := testConfig() + cfg.MaxObservations = 2 + registry := NewRegistry(cfg, NewLocalStore(), nil) + one := registry.begin(testRoute(ClassPlayback), CaptureSet{}) + two := registry.begin(testRoute(ClassPlayback), CaptureSet{}) + three := registry.begin(testRoute(ClassPlayback), CaptureSet{}) + three.AddBytes(9) + registry.release(three, OutcomeUnknown) + if !three.countingOnly || registry.observationReservations.Load() != 2 { + t.Fatalf("bound was not exact: counting=%t reservations=%d", three.countingOnly, registry.observationReservations.Load()) + } + registry.release(one, OutcomeUnknown) + registry.release(two, OutcomeUnknown) + snapshot := registry.Snapshot() + if !snapshot.Truncated || snapshot.DroppedObservations != 1 || snapshot.DroppedBytes != 9 { + t.Fatalf("drop counters = %+v", snapshot) + } +} + +func TestStartedAtImprovementAndIdentityConflict(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Unix(200, 0)}) + first := testAttachment("session-conflict") + first.StartedAt = time.Time{} + first.StartedAtSource = "" + registry.attach(obs, first) + offered := first + offered.Subject = UserSubject(8) + offered.StartedAt = time.Unix(50, 0) + offered.StartedAtSource = StartedAtSourceClaim + registry.attach(obs, offered) + registry.release(obs, httpstreamOutcomeCompleted) + snapshot := registry.Sweep() + session := snapshot.Sessions[0] + if !session.HasIdentityConflict || session.Subject != UserSubject(7) { + t.Fatalf("conflict did not preserve identity: %+v", session) + } + if session.StartedAtSource != StartedAtSourceClaim || !session.StartedAt.Equal(offered.StartedAt) || session.StartedAtDegraded { + t.Fatalf("started-at authority was not improved: %+v", session) + } +} + +func TestMidPlaybackReplanUpdatesStateWithoutIdentityConflict(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + first := testAttachment("session-replan") + first.MediaFileID = 100 + first.PlayMethod = "direct" + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, first) + registry.release(obs, httpstreamOutcomeCompleted) + + replanned := first + replanned.MediaFileID = 200 + replanned.PlayMethod = "transcode" + obs = registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, replanned) + registry.release(obs, httpstreamOutcomeCompleted) + + session := registry.Sweep().Sessions[0] + if session.HasIdentityConflict || len(session.IdentityConflicts) != 0 { + t.Fatalf("replan recorded identity conflict: %+v", session.IdentityConflicts) + } + if session.MediaFileID != 200 || session.PlayMethod != "transcode" { + t.Fatalf("current replan state = media %d, method %q", session.MediaFileID, session.PlayMethod) + } + if len(session.MediaFileIDs) != 2 || session.MediaFileIDs[0] != 100 || session.MediaFileIDs[1] != 200 { + t.Fatalf("observed media files = %v", session.MediaFileIDs) + } + if len(session.PlayMethods) != 2 || session.PlayMethods[0] != "direct" || session.PlayMethods[1] != "transcode" { + t.Fatalf("observed play methods = %v", session.PlayMethods) + } + + changedOwner := replanned + changedOwner.Subject = UserSubject(8) + obs = registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, changedOwner) + registry.release(obs, httpstreamOutcomeCompleted) + if session = registry.Sweep().Sessions[0]; !session.HasIdentityConflict { + t.Fatal("changed subject did not record an identity conflict") + } +} + +func TestUnknownAttachmentFieldsDoNotDisagreeWithSession(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + first := testAttachment("session-partial") + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, first) + registry.release(obs, httpstreamOutcomeCompleted) + + partial := Attachment{SessionID: first.SessionID} + obs = registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, partial) + registry.release(obs, httpstreamOutcomeCompleted) + + session := registry.Sweep().Sessions[0] + if session.HasIdentityConflict || len(session.IdentityConflicts) != 0 { + t.Fatalf("unknown fields recorded disagreement: %+v", session.IdentityConflicts) + } + if session.MediaFileID != first.MediaFileID || session.PlayMethod != first.PlayMethod { + t.Fatalf("unknown fields replaced current state: media %d, method %q", session.MediaFileID, session.PlayMethod) + } +} + +func TestPruneReleasesReservations(t *testing.T) { + cfg := testConfig() + registry := NewRegistry(cfg, NewLocalStore(), nil) + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, testAttachment("prune")) + registry.release(obs, httpstreamOutcomeCompleted) + registry.sweep(time.Now().Add(2 * cfg.Retention)) + if registry.sessionReservations.Load() != 0 || registry.observationReservations.Load() != 0 { + t.Fatalf("reservations leaked: sessions=%d observations=%d", registry.sessionReservations.Load(), registry.observationReservations.Load()) + } +} + +func TestRouteBoundDropsNewestRouteWithoutDroppingObservation(t *testing.T) { + cfg := testConfig() + cfg.MaxRoutesPerSession = 1 + registry := NewRegistry(cfg, NewLocalStore(), nil) + for index, pattern := range []string{"/one", "/two"} { + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: pattern, ReceivedAt: time.Now()}) + registry.attach(obs, testAttachment("route-bound")) + obs.AddBytes(int64(index + 1)) + registry.release(obs, httpstreamOutcomeCompleted) + } + snapshot := registry.Sweep() + session := snapshot.Sessions[0] + if !session.RoutesOverflowed || len(session.Routes) != 1 || session.BytesAccepted != 3 || snapshot.DroppedObservations != 0 { + t.Fatalf("route overflow = %+v", snapshot) + } +} + +func TestSetRealtimeConnectionIgnoresUnknownAndIsIdempotent(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + registry.SetRealtimeConnection("missing", true) + if len(registry.Snapshot().Sessions) != 0 { + t.Fatal("realtime update created a session") + } + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, testAttachment("known")) + registry.SetRealtimeConnection("known", true) + registry.SetRealtimeConnection("known", true) + if !registry.Snapshot().Sessions[0].RealtimeConnectionAlive { + t.Fatal("realtime connection not recorded") + } + registry.release(obs, httpstreamOutcomeCompleted) +} + +type failingStore struct{ published atomic.Int64 } + +func (s *failingStore) Publish(context.Context, Snapshot) error { + s.published.Add(1) + return errors.New("publish failed") +} +func (s *failingStore) Load(context.Context) (Snapshot, error) { return Snapshot{}, nil } + +func TestStartContinuesAfterPublishError(t *testing.T) { + cfg := testConfig() + cfg.SweepInterval = time.Millisecond + store := &failingStore{} + registry := NewRegistry(cfg, store, slog.New(slog.DiscardHandler)) + ctx, cancel := context.WithCancel(context.Background()) + registry.Start(ctx) + time.Sleep(8 * time.Millisecond) + cancel() + if store.published.Load() < 2 { + t.Fatalf("collector stopped after publish error: %d publishes", store.published.Load()) + } +} + +func TestLocalStoreDeepCopies(t *testing.T) { + store := NewLocalStore() + source := Snapshot{Sessions: []SessionView{{ViewerIPs: []string{"one"}, Routes: []RouteActivityView{{Pattern: "/one"}}, Outcomes: map[httpstream.StreamOutcome]int64{"completed": 1}}}} + if err := store.Publish(context.Background(), source); err != nil { + t.Fatal(err) + } + source.Sessions[0].ViewerIPs[0] = "mutated-source" + loaded, _ := store.Load(context.Background()) + loaded.Sessions[0].ViewerIPs[0] = "mutated-load" + loaded.Sessions[0].Routes[0].Pattern = "/mutated" + loadedAgain, _ := store.Load(context.Background()) + if loadedAgain.Sessions[0].ViewerIPs[0] != "one" || loadedAgain.Sessions[0].Routes[0].Pattern != "/one" { + t.Fatalf("store returned aliased snapshot: %+v", loadedAgain) + } +} diff --git a/internal/streamtelemetry/route.go b/internal/streamtelemetry/route.go new file mode 100644 index 000000000..2d9873e58 --- /dev/null +++ b/internal/streamtelemetry/route.go @@ -0,0 +1,132 @@ +package streamtelemetry + +import ( + "fmt" + "net" + "net/http" + "reflect" + "sort" + "sync" + + "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/go-chi/chi/v5" +) + +type Family string +type Class string +type Role string + +const ( + FamilyNative Family = "native" + FamilyJellycompat Family = "jellycompat" + FamilyProxy Family = "proxy" + FamilyABS Family = "abs" + FamilyTranscodeNode Family = "transcode_node" + + ClassPlayback Class = "playback" + ClassManifest Class = "manifest" + ClassTransfer Class = "transfer" + + RoleViewerEgress Role = "viewer_egress" + RoleInternalRelay Role = "internal_relay" + RoleProducer Role = "producer" +) + +type MediaRoute struct { + Family Family + Method string + Pattern string + Class Class + Role Role + CanonicalSessionKey string + CapRelevant bool + Enrolled bool + Capture func(*http.Request) CaptureSet +} + +type routeKey struct { + family Family + method string + pattern string +} + +var declarations = struct { + sync.RWMutex + routes map[routeKey]MediaRoute +}{routes: make(map[routeKey]MediaRoute)} + +func DeclareRoutes(routes ...MediaRoute) { + declarations.Lock() + defer declarations.Unlock() + for _, route := range routes { + key := routeKey{route.Family, route.Method, route.Pattern} + if existing, ok := declarations.routes[key]; ok { + if !sameDeclaration(existing, route) { + panic(fmt.Sprintf("conflicting media route declaration: %s %s %s", route.Family, route.Method, route.Pattern)) + } + continue + } + declarations.routes[key] = route + } +} + +func DeclaredRoutes(family Family) []MediaRoute { + declarations.RLock() + defer declarations.RUnlock() + routes := make([]MediaRoute, 0) + for _, route := range declarations.routes { + if route.Family == family { + routes = append(routes, route) + } + } + sort.Slice(routes, func(i, j int) bool { + if routes[i].Pattern == routes[j].Pattern { + return routes[i].Method < routes[j].Method + } + return routes[i].Pattern < routes[j].Pattern + }) + return routes +} + +func sameDeclaration(a, b MediaRoute) bool { + return a.Family == b.Family && a.Method == b.Method && a.Pattern == b.Pattern && + a.Class == b.Class && a.Role == b.Role && + a.CanonicalSessionKey == b.CanonicalSessionKey && + a.CapRelevant == b.CapRelevant && a.Enrolled == b.Enrolled && + capturePointer(a.Capture) == capturePointer(b.Capture) +} + +func capturePointer(capture func(*http.Request) CaptureSet) uintptr { + if capture == nil { + return 0 + } + return reflect.ValueOf(capture).Pointer() +} + +func genericCapture(r *http.Request) CaptureSet { + pattern := "" + if routeContext := chi.RouteContext(r.Context()); routeContext != nil { + pattern = routeContext.RoutePattern() + } + return CaptureSet{ + Method: r.Method, + Pattern: pattern, + ViewerIP: viewerIP(r), + UserAgent: r.UserAgent(), + ReceivedAt: now(), + } +} + +func viewerIP(r *http.Request) string { + if r == nil { + return "" + } + if ip := clientip.FromContext(r.Context()); ip != "" { + return ip + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + return host + } + return r.RemoteAddr +} diff --git a/internal/streamtelemetry/route_test.go b/internal/streamtelemetry/route_test.go new file mode 100644 index 000000000..6a86257a4 --- /dev/null +++ b/internal/streamtelemetry/route_test.go @@ -0,0 +1,17 @@ +package streamtelemetry + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestGenericCaptureIgnoresUnresolvedForwardedFor(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/media", nil) + request.RemoteAddr = "192.0.2.10:4321" + request.Header.Set("X-Forwarded-For", "8.8.8.8") + + if got := genericCapture(request).ViewerIP; got != "192.0.2.10" { + t.Fatalf("viewer IP = %q, want socket peer", got) + } +} diff --git a/internal/streamtelemetry/session.go b/internal/streamtelemetry/session.go new file mode 100644 index 000000000..88cf8a295 --- /dev/null +++ b/internal/streamtelemetry/session.go @@ -0,0 +1,190 @@ +package streamtelemetry + +import ( + "sync" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type routeActivity struct { + Method string + Pattern string + Role Role + Class Class + CapRelevant bool + Open int + Requests int64 + BytesFolded int64 + LastSweptBytes int64 + LastByteAccepted time.Time + LastObservationEnd time.Time +} + +type boundedSet[T comparable] struct { + values map[T]struct{} + max int + overflowed bool +} + +func newBoundedSet[T comparable](max int) boundedSet[T] { + return boundedSet[T]{values: make(map[T]struct{}), max: max} +} + +func (s *boundedSet[T]) add(value T) { + if _, ok := s.values[value]; ok { + return + } + if len(s.values) >= s.max { + s.overflowed = true + return + } + s.values[value] = struct{}{} +} + +type logicalSession struct { + mu sync.Mutex + + subject Subject + profileID string + sessionID string + mediaFileID int + playMethod string + startedAt time.Time + startedAtSource StartedAtSource + startedDegraded bool + + bytesFolded int64 + lastSweptBytes int64 + lastByteAccepted time.Time + lastObservationEnd time.Time + openObservations int + realtimeAlive bool + requestCount int64 + hasIdentityConflict bool + identityConflicts []IdentityConflict + identityOverflowed bool + routes map[string]*routeActivity + routesOverflowed bool + observations map[string]*Observation + viewerIPs boundedSet[string] + deviceIDs boundedSet[string] + clientVariants boundedSet[ClientVariant] + userAgents boundedSet[string] + mediaFileIDs boundedSet[int] + playMethods boundedSet[string] + tokenIssuedAts boundedSet[int64] + tokenIssuedSources map[TokenIssuedAtSource]int64 + outcomes map[httpstream.StreamOutcome]int64 +} + +type transfer struct { + mu sync.Mutex + + id string + subject Subject + profileID string + mediaFileID int + bytesFolded int64 + lastSweptBytes int64 + lastByteAccepted time.Time + lastObservationEnd time.Time + openObservations int + requestCount int64 + route MediaRoute + capture CaptureSet + observation *Observation + outcomes map[httpstream.StreamOutcome]int64 +} + +func newLogicalSession(a Attachment, cfg Config, observedAt time.Time) *logicalSession { + startedAt, source, degraded := normalizeStartedAt(a.StartedAt, a.StartedAtSource, observedAt) + session := &logicalSession{ + subject: a.Subject, profileID: a.ProfileID, sessionID: a.SessionID, + mediaFileID: a.MediaFileID, playMethod: a.PlayMethod, + startedAt: startedAt, startedAtSource: source, startedDegraded: degraded, + routes: make(map[string]*routeActivity), observations: make(map[string]*Observation), + viewerIPs: newBoundedSet[string](cfg.MaxViewerIPsPerSession), + deviceIDs: newBoundedSet[string](cfg.MaxDeviceIDsPerSession), + clientVariants: newBoundedSet[ClientVariant](cfg.MaxClientVariantsPerSession), + userAgents: newBoundedSet[string](cfg.MaxClientVariantsPerSession), + mediaFileIDs: newBoundedSet[int](cfg.MaxMediaFileIDsPerSession), + playMethods: newBoundedSet[string](cfg.MaxPlayMethodsPerSession), + tokenIssuedAts: newBoundedSet[int64](cfg.MaxTokenIssuedAtPerSession), + tokenIssuedSources: make(map[TokenIssuedAtSource]int64), + outcomes: make(map[httpstream.StreamOutcome]int64), + } + if a.MediaFileID != 0 { + session.mediaFileIDs.add(a.MediaFileID) + } + if a.PlayMethod != "" { + session.playMethods.add(a.PlayMethod) + } + return session +} + +func normalizeStartedAt(value time.Time, source StartedAtSource, observedAt time.Time) (time.Time, StartedAtSource, bool) { + if value.IsZero() || startedAtRank(source) == 0 { + return observedAt, StartedAtSourceFirstSeen, true + } + return value, source, source == StartedAtSourceIssuedAt || source == StartedAtSourceFirstSeen +} + +func startedAtRank(source StartedAtSource) int { + switch source { + case StartedAtSourceClaim: + return 4 + case StartedAtSourceSession: + return 3 + case StartedAtSourceIssuedAt: + return 2 + case StartedAtSourceFirstSeen: + return 1 + default: + return 0 + } +} + +func routeID(method, pattern string) string { return method + "\x00" + pattern } + +func (s *logicalSession) recordConflicts(a Attachment, observedAt time.Time, max int) { + checks := []struct{ field, existing, offered string }{ + {"subject.kind", string(s.subject.Kind), string(a.Subject.Kind)}, + {"subject.id", s.subject.ID, a.Subject.ID}, + {"profile_id", s.profileID, a.ProfileID}, + } + for _, check := range checks { + if check.existing == "" || check.offered == "" || check.existing == check.offered { + continue + } + s.hasIdentityConflict = true + if len(s.identityConflicts) >= max { + s.identityOverflowed = true + continue + } + s.identityConflicts = append(s.identityConflicts, IdentityConflict{ + Field: check.field, Existing: check.existing, Offered: check.offered, ObservedAt: observedAt, + }) + } + if a.MediaFileID != 0 { + s.mediaFileID = a.MediaFileID + s.mediaFileIDs.add(a.MediaFileID) + } + if a.PlayMethod != "" { + s.playMethod = a.PlayMethod + s.playMethods.add(a.PlayMethod) + } + if rank := startedAtRank(a.StartedAtSource); !a.StartedAt.IsZero() && rank > startedAtRank(s.startedAtSource) { + old := s.startedAt.Format(time.RFC3339Nano) + s.startedAt = a.StartedAt + s.startedAtSource = a.StartedAtSource + s.startedDegraded = a.StartedAtSource == StartedAtSourceIssuedAt || a.StartedAtSource == StartedAtSourceFirstSeen + if len(s.identityConflicts) < max { + s.identityConflicts = append(s.identityConflicts, IdentityConflict{ + Field: "started_at_replaced", Existing: old, Offered: a.StartedAt.Format(time.RFC3339Nano), ObservedAt: observedAt, + }) + } else { + s.identityOverflowed = true + } + } +} diff --git a/internal/streamtelemetry/store.go b/internal/streamtelemetry/store.go new file mode 100644 index 000000000..4d7d3a187 --- /dev/null +++ b/internal/streamtelemetry/store.go @@ -0,0 +1,32 @@ +package streamtelemetry + +import ( + "context" + "sync" +) + +type SnapshotStore interface { + Publish(context.Context, Snapshot) error + Load(context.Context) (Snapshot, error) +} + +type LocalStore struct { + mu sync.RWMutex + snapshot Snapshot +} + +func NewLocalStore() *LocalStore { return &LocalStore{} } + +func (s *LocalStore) Publish(_ context.Context, snapshot Snapshot) error { + s.mu.Lock() + s.snapshot = cloneSnapshot(snapshot) + s.mu.Unlock() + return nil +} + +func (s *LocalStore) Load(_ context.Context) (Snapshot, error) { + s.mu.RLock() + snapshot := cloneSnapshot(s.snapshot) + s.mu.RUnlock() + return snapshot, nil +} diff --git a/internal/streamtelemetry/view.go b/internal/streamtelemetry/view.go new file mode 100644 index 000000000..6941be265 --- /dev/null +++ b/internal/streamtelemetry/view.go @@ -0,0 +1,206 @@ +package streamtelemetry + +import ( + "sort" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type RouteActivityView struct { + Method string + Pattern string + Role Role + Class Class + CapRelevant bool + Open int + Requests int64 + BytesAccepted int64 + LastByteAccepted time.Time + LastObservationEnd time.Time +} + +type SessionView struct { + Subject Subject + ProfileID string + SessionID string + MediaFileID int + PlayMethod string + MediaFileIDs []int + MediaFileIDsOverflowed bool + PlayMethods []string + PlayMethodsOverflowed bool + StartedAt time.Time + StartedAtSource StartedAtSource + StartedAtDegraded bool + BytesAccepted int64 // pre-compression at the enrollment point; see package documentation. + LastByteAccepted time.Time + LastObservationEnd time.Time + OpenObservations int + RealtimeConnectionAlive bool + RequestCount int64 + Routes []RouteActivityView + RoutesOverflowed bool + ViewerIPs []string + ViewerIPsOverflowed bool + DeviceIDs []string + DeviceIDsOverflowed bool + ClientVariants []ClientVariant + ClientVariantsOverflowed bool + UserAgents []string + UserAgentsOverflowed bool + TokenIssuedAts []time.Time + TokenIssuedAtsOverflowed bool + TokenIssuedAtSources map[TokenIssuedAtSource]int64 + Outcomes map[httpstream.StreamOutcome]int64 + HasIdentityConflict bool + IdentityConflicts []IdentityConflict + IdentityConflictsOverflowed bool +} + +type TransferView struct { + ID string + Subject Subject + ProfileID string + MediaFileID int + Method string + Pattern string + Role Role + BytesAccepted int64 + LastByteAccepted time.Time + LastObservationEnd time.Time + OpenObservations int + RequestCount int64 + ViewerIP string + DeviceID string + Client ClientVariant + UserAgent string + Outcomes map[httpstream.StreamOutcome]int64 +} + +type Snapshot struct { + PublisherID string + NodeID string + CapturedAt time.Time + Sessions []SessionView + Transfers []TransferView + Truncated bool + DroppedObservations int64 + DroppedBytes int64 + UnattributedObservations int64 + UnattributedBytes int64 +} + +func sessionViewOf(s *logicalSession) SessionView { + view := SessionView{ + Subject: s.subject, ProfileID: s.profileID, SessionID: s.sessionID, + MediaFileID: s.mediaFileID, PlayMethod: s.playMethod, + MediaFileIDsOverflowed: s.mediaFileIDs.overflowed, + PlayMethodsOverflowed: s.playMethods.overflowed, + StartedAt: s.startedAt, StartedAtSource: s.startedAtSource, StartedAtDegraded: s.startedDegraded, + BytesAccepted: s.lastSweptBytes, LastByteAccepted: s.lastByteAccepted, + LastObservationEnd: s.lastObservationEnd, OpenObservations: s.openObservations, + RealtimeConnectionAlive: s.realtimeAlive, RequestCount: s.requestCount, + RoutesOverflowed: s.routesOverflowed, ViewerIPsOverflowed: s.viewerIPs.overflowed, + DeviceIDsOverflowed: s.deviceIDs.overflowed, ClientVariantsOverflowed: s.clientVariants.overflowed, + UserAgentsOverflowed: s.userAgents.overflowed, TokenIssuedAtsOverflowed: s.tokenIssuedAts.overflowed, + TokenIssuedAtSources: cloneTokenSources(s.tokenIssuedSources), + Outcomes: cloneOutcomes(s.outcomes), HasIdentityConflict: s.hasIdentityConflict, + IdentityConflicts: append([]IdentityConflict(nil), s.identityConflicts...), + IdentityConflictsOverflowed: s.identityOverflowed, + } + for _, route := range s.routes { + view.Routes = append(view.Routes, RouteActivityView{Method: route.Method, Pattern: route.Pattern, + Role: route.Role, Class: route.Class, CapRelevant: route.CapRelevant, Open: route.Open, + Requests: route.Requests, BytesAccepted: route.LastSweptBytes, + LastByteAccepted: route.LastByteAccepted, LastObservationEnd: route.LastObservationEnd}) + } + for value := range s.viewerIPs.values { + view.ViewerIPs = append(view.ViewerIPs, value) + } + for value := range s.deviceIDs.values { + view.DeviceIDs = append(view.DeviceIDs, value) + } + for value := range s.clientVariants.values { + view.ClientVariants = append(view.ClientVariants, value) + } + for value := range s.userAgents.values { + view.UserAgents = append(view.UserAgents, value) + } + for value := range s.mediaFileIDs.values { + view.MediaFileIDs = append(view.MediaFileIDs, value) + } + for value := range s.playMethods.values { + view.PlayMethods = append(view.PlayMethods, value) + } + for value := range s.tokenIssuedAts.values { + view.TokenIssuedAts = append(view.TokenIssuedAts, time.Unix(0, value)) + } + sort.Slice(view.Routes, func(i, j int) bool { + return view.Routes[i].Method+view.Routes[i].Pattern < view.Routes[j].Method+view.Routes[j].Pattern + }) + sort.Strings(view.ViewerIPs) + sort.Strings(view.DeviceIDs) + sort.Strings(view.UserAgents) + sort.Ints(view.MediaFileIDs) + sort.Strings(view.PlayMethods) + sort.Slice(view.ClientVariants, func(i, j int) bool { + return clientVariantKey(view.ClientVariants[i]) < clientVariantKey(view.ClientVariants[j]) + }) + sort.Slice(view.TokenIssuedAts, func(i, j int) bool { return view.TokenIssuedAts[i].Before(view.TokenIssuedAts[j]) }) + return view +} + +func transferViewOf(t *transfer) TransferView { + return TransferView{ID: t.id, Subject: t.subject, ProfileID: t.profileID, MediaFileID: t.mediaFileID, + Method: t.capture.Method, Pattern: t.capture.Pattern, Role: t.route.Role, + BytesAccepted: t.lastSweptBytes, LastByteAccepted: t.lastByteAccepted, + LastObservationEnd: t.lastObservationEnd, OpenObservations: t.openObservations, + RequestCount: t.requestCount, ViewerIP: t.capture.ViewerIP, DeviceID: t.capture.DeviceID, + Client: t.capture.Client, UserAgent: t.capture.UserAgent, Outcomes: cloneOutcomes(t.outcomes)} +} + +func cloneSnapshot(source Snapshot) Snapshot { + destination := source + destination.Sessions = make([]SessionView, len(source.Sessions)) + for i := range source.Sessions { + destination.Sessions[i] = source.Sessions[i] + destination.Sessions[i].Routes = append([]RouteActivityView(nil), source.Sessions[i].Routes...) + destination.Sessions[i].ViewerIPs = append([]string(nil), source.Sessions[i].ViewerIPs...) + destination.Sessions[i].DeviceIDs = append([]string(nil), source.Sessions[i].DeviceIDs...) + destination.Sessions[i].ClientVariants = append([]ClientVariant(nil), source.Sessions[i].ClientVariants...) + destination.Sessions[i].UserAgents = append([]string(nil), source.Sessions[i].UserAgents...) + destination.Sessions[i].MediaFileIDs = append([]int(nil), source.Sessions[i].MediaFileIDs...) + destination.Sessions[i].PlayMethods = append([]string(nil), source.Sessions[i].PlayMethods...) + destination.Sessions[i].TokenIssuedAts = append([]time.Time(nil), source.Sessions[i].TokenIssuedAts...) + destination.Sessions[i].IdentityConflicts = append([]IdentityConflict(nil), source.Sessions[i].IdentityConflicts...) + destination.Sessions[i].Outcomes = cloneOutcomes(source.Sessions[i].Outcomes) + destination.Sessions[i].TokenIssuedAtSources = cloneTokenSources(source.Sessions[i].TokenIssuedAtSources) + } + destination.Transfers = make([]TransferView, len(source.Transfers)) + for i := range source.Transfers { + destination.Transfers[i] = source.Transfers[i] + destination.Transfers[i].Outcomes = cloneOutcomes(source.Transfers[i].Outcomes) + } + return destination +} + +func cloneTokenSources(source map[TokenIssuedAtSource]int64) map[TokenIssuedAtSource]int64 { + destination := make(map[TokenIssuedAtSource]int64, len(source)) + for key, value := range source { + destination[key] = value + } + return destination +} + +func cloneOutcomes(source map[httpstream.StreamOutcome]int64) map[httpstream.StreamOutcome]int64 { + destination := make(map[httpstream.StreamOutcome]int64, len(source)) + for key, value := range source { + destination[key] = value + } + return destination +} + +func clientVariantKey(value ClientVariant) string { + return value.Name + "\x00" + value.Version + "\x00" + value.Build + "\x00" + value.Channel +} diff --git a/internal/streamtelemetry/writer.go b/internal/streamtelemetry/writer.go new file mode 100644 index 000000000..db41c5f97 --- /dev/null +++ b/internal/streamtelemetry/writer.go @@ -0,0 +1,116 @@ +package streamtelemetry + +import ( + "bufio" + "context" + "io" + "net" + "net/http" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +const OutcomeUnknown httpstream.StreamOutcome = "unknown" + +func (r *Registry) Observe(route MediaRoute) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + if r == nil || !r.cfg.Enabled || !route.Enrolled { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + var capture CaptureSet + if route.Capture != nil { + capture = route.Capture(request) + if capture.Method == "" { + capture.Method = request.Method + } + if capture.Pattern == "" { + capture.Pattern = route.Pattern + } + if capture.ReceivedAt.IsZero() { + capture.ReceivedAt = now() + } + } else { + capture = genericCapture(request) + } + obs := r.begin(route, capture) + observed := &observedWriter{w: w, observation: obs, bodyEligible: request.Method != http.MethodHead} + request = request.WithContext(context.WithValue(request.Context(), observationContextKey{}, obs)) + completed := false + defer func() { + r.release(obs, obs.outcome(request.Context().Err(), completed)) + }() + next.ServeHTTP(observed, request) + completed = true + }) + } +} + +type observedWriter struct { + w http.ResponseWriter + observation *Observation + bodyEligible bool + statusCode int +} + +func (w *observedWriter) Header() http.Header { return w.w.Header() } + +func (w *observedWriter) WriteHeader(statusCode int) { + if w.statusCode == 0 { + w.statusCode = statusCode + } + w.w.WriteHeader(statusCode) +} + +func (w *observedWriter) Write(p []byte) (int, error) { + if w.observation.cut.Load() { + return 0, context.Canceled + } + if w.statusCode == 0 { + w.statusCode = http.StatusOK + } + n, err := w.w.Write(p) + if w.bodyEligible { + w.observation.AddBytes(int64(n)) + } + w.observation.recordWriteError(err) + return n, err +} + +func (w *observedWriter) ReadFrom(reader io.Reader) (int64, error) { + if w.observation.cut.Load() { + return 0, context.Canceled + } + readerFrom, ok := httpstream.ReaderFromOf(w.w) + if !ok { + return io.Copy(httpstream.WriterOnly(w), reader) + } + if w.statusCode == 0 { + w.statusCode = http.StatusOK + } + return httpstream.CopyChunked(readerFrom, reader, httpstream.ReadFromChunkDefault, func(n int64, err error) { + if w.bodyEligible { + w.observation.AddBytes(n) + } + w.observation.recordWriteError(err) + }) +} + +func (w *observedWriter) Unwrap() http.ResponseWriter { return w.w } +func (w *observedWriter) Flush() { _ = http.NewResponseController(w.w).Flush() } + +func (w *observedWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := w.w.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return hijacker.Hijack() +} + +func (w *observedWriter) Push(target string, options *http.PushOptions) error { + pusher, ok := w.w.(http.Pusher) + if !ok { + return http.ErrNotSupported + } + return pusher.Push(target, options) +} diff --git a/internal/streamtelemetry/writer_test.go b/internal/streamtelemetry/writer_test.go new file mode 100644 index 000000000..fea5ff7e3 --- /dev/null +++ b/internal/streamtelemetry/writer_test.go @@ -0,0 +1,143 @@ +package streamtelemetry + +import ( + "bufio" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type readerFromRecorder struct { + *httptest.ResponseRecorder + readFrom bool +} + +func (w *readerFromRecorder) ReadFrom(r io.Reader) (int64, error) { + w.readFrom = true + return io.Copy(w.ResponseRecorder, r) +} + +func TestObservedWriterPreservesReadFromAndCountsAfterWrite(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{}) + underlying := &readerFromRecorder{ResponseRecorder: httptest.NewRecorder()} + w := &observedWriter{w: underlying, observation: obs, bodyEligible: true} + n, err := w.ReadFrom(strings.NewReader("abcdef")) + if err != nil || n != 6 || !underlying.readFrom || obs.BytesAccepted() != 6 { + t.Fatalf("ReadFrom = %d, %v, fast=%t, bytes=%d", n, err, underlying.readFrom, obs.BytesAccepted()) + } + registry.release(obs, OutcomeUnknown) +} + +func TestObservedWriterHEADCountsZero(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + handler := registry.Observe(MediaRoute{Family: FamilyNative, Method: http.MethodHead, Pattern: "/head", Class: ClassPlayback, Role: RoleViewerEgress, Enrolled: true})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("head")) + _, _ = w.Write([]byte("not-on-wire")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodHead, "/head", nil)) + if got := registry.Sweep().Sessions[0].BytesAccepted; got != 0 { + t.Fatalf("HEAD bytes = %d", got) + } +} + +func TestObserveDoesNotRunGenericCaptureWhenRouteCaptureExists(t *testing.T) { + originalNow := now + defer func() { now = originalNow }() + nowCalls := 0 + now = func() time.Time { + nowCalls++ + return time.Unix(123, 0) + } + captureCalls := 0 + route := testRoute(ClassPlayback) + route.Capture = func(*http.Request) CaptureSet { + captureCalls++ + return CaptureSet{Method: http.MethodGet, Pattern: route.Pattern, ReceivedAt: time.Unix(100, 0)} + } + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + handler := registry.Observe(route)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + if captureCalls != 1 || nowCalls != 0 { + t.Fatalf("capture calls = %d, generic timestamp calls = %d", captureCalls, nowCalls) + } +} + +func TestObservedWriterPanicReleasesUnknownAndPropagates(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("panic")) + panic("boom") + })) + defer func() { + if recover() == nil { + t.Fatal("panic did not propagate") + } + snapshot := registry.Sweep() + if snapshot.Sessions[0].Outcomes[OutcomeUnknown] != 1 { + t.Fatalf("outcomes = %+v", snapshot.Sessions[0].Outcomes) + } + }() + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) +} + +type optionalWriter struct{ *httptest.ResponseRecorder } + +func (w *optionalWriter) Flush() {} +func (w *optionalWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { return nil, nil, nil } +func (w *optionalWriter) Push(string, *http.PushOptions) error { return nil } + +func TestObservedWriterPreservesOptionalInterfaces(t *testing.T) { + obs := newObservation(nil, MediaRoute{}, CaptureSet{}) + w := &observedWriter{w: &optionalWriter{httptest.NewRecorder()}, observation: obs, bodyEligible: true} + if _, _, err := w.Hijack(); err != nil { + t.Fatal(err) + } + if err := w.Push("/asset", nil); err != nil { + t.Fatal(err) + } + w.Flush() +} + +type failingResponseWriter struct{ err error } + +func (w *failingResponseWriter) Header() http.Header { return make(http.Header) } +func (w *failingResponseWriter) WriteHeader(int) {} +func (w *failingResponseWriter) Write([]byte) (int, error) { return 0, w.err } + +type timeoutWriteError struct{} + +func (timeoutWriteError) Error() string { return "write deadline exceeded" } +func (timeoutWriteError) Timeout() bool { return true } +func (timeoutWriteError) Temporary() bool { return false } + +func TestObservedWriterClassifiesTransportFailuresOnRelease(t *testing.T) { + tests := []struct { + name string + err error + want httpstream.StreamOutcome + }{ + {name: "stalled reap", err: timeoutWriteError{}, want: httpstream.OutcomeStalledReap}, + {name: "client gone", err: io.ErrClosedPipe, want: httpstream.OutcomeClientGone}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), nil) + obs := registry.begin(testRoute(ClassPlayback), CaptureSet{Method: http.MethodGet, Pattern: "/media/{id}", ReceivedAt: time.Now()}) + registry.attach(obs, testAttachment("failure")) + writer := &observedWriter{w: &failingResponseWriter{err: test.err}, observation: obs, bodyEligible: true} + _, _ = writer.Write([]byte("body")) + registry.release(obs, obs.outcome(nil, true)) + session := registry.Sweep().Sessions[0] + if session.Outcomes[test.want] != 1 { + t.Fatalf("outcomes = %+v", session.Outcomes) + } + }) + } +} diff --git a/internal/transcodenode/media_routes.go b/internal/transcodenode/media_routes.go new file mode 100644 index 000000000..572d9edcb --- /dev/null +++ b/internal/transcodenode/media_routes.go @@ -0,0 +1,22 @@ +package transcodenode + +import ( + "net/http" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var transcodeNodeMediaRoutes = []streamtelemetry.MediaRoute{ + nodeRoute(http.MethodGet, "/downloads/artifacts/{artifact_id}", streamtelemetry.ClassTransfer), + nodeRoute(http.MethodHead, "/downloads/artifacts/{artifact_id}", streamtelemetry.ClassTransfer), + nodeRoute(http.MethodGet, "/transcode/{session_id}/master.m3u8", streamtelemetry.ClassManifest), + nodeRoute(http.MethodGet, "/transcode/{session_id}/segment/{name}", streamtelemetry.ClassPlayback), +} + +func nodeRoute(method, pattern string, class streamtelemetry.Class) streamtelemetry.MediaRoute { + return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyTranscodeNode, Method: method, Pattern: pattern, + Class: class, Role: streamtelemetry.RoleInternalRelay, CanonicalSessionKey: "transport_session_id", + CapRelevant: false, Enrolled: false} +} + +func declareTranscodeNodeMediaRoutes() { streamtelemetry.DeclareRoutes(transcodeNodeMediaRoutes...) } diff --git a/internal/transcodenode/media_routes_test.go b/internal/transcodenode/media_routes_test.go new file mode 100644 index 000000000..b832f073f --- /dev/null +++ b/internal/transcodenode/media_routes_test.go @@ -0,0 +1,49 @@ +package transcodenode + +import ( + "flag" + "os" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +var updateRouteManifest = flag.Bool("update-route-manifest", false, "update checked-in route manifest") + +func TestMediaRouteManifest(t *testing.T) { + declareTranscodeNodeMediaRoutes() + makeRouter := func() chi.Routes { + // Handler's idle reaper is guarded by reaperOnce, so repeated fixtures + // start only the single process-wide goroutine existing tests already use. + return NewServer(nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}), nodesessions.NewTracker(nil, "", "", "")).Handler().(chi.Routes) + } + actual, err := streamtelemetry.BuildRouteManifest([]chi.Routes{makeRouter(), makeRouter()}, transcodeNodeMediaRoutes) + if err != nil { + t.Fatal(err) + } + const path = "testdata/media_routes.txt" + if *updateRouteManifest { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(actual), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != actual { + t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") + } + for _, route := range transcodeNodeMediaRoutes { + if route.Enrolled { + t.Fatalf("transcode-node route enrolled: %s %s", route.Method, route.Pattern) + } + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 651fcd53f..f7c2031ed 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -447,6 +447,7 @@ func (s *Server) SetInputPathAuthorizer(authorizer InputPathAuthorizer) { // Handler returns the chi.Router with all transcode routes. func (s *Server) Handler() http.Handler { + declareTranscodeNodeMediaRoutes() s.startIdleReaper() r := chi.NewRouter() r.Get("/api/v1/health", s.handleHealth) diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt new file mode 100644 index 000000000..5f7a8c77a --- /dev/null +++ b/internal/transcodenode/testdata/media_routes.txt @@ -0,0 +1,28 @@ +# fixture 1 +POST /admin/force-reload non-media +GET /api/v1/health non-media +POST /chapter-thumbnails/extract non-media +DELETE /downloads/artifacts/{artifact_id} non-media +GET /downloads/artifacts/{artifact_id} media transfer internal_relay false false +HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false false +POST /downloads/prepare non-media +GET /hw-capabilities non-media +GET /status non-media +POST /transcode/start non-media +DELETE /transcode/{session_id} non-media +GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false false +GET /transcode/{session_id}/segment/{name} media playback internal_relay false false +# fixture 2 +POST /admin/force-reload non-media +GET /api/v1/health non-media +POST /chapter-thumbnails/extract non-media +DELETE /downloads/artifacts/{artifact_id} non-media +GET /downloads/artifacts/{artifact_id} media transfer internal_relay false false +HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false false +POST /downloads/prepare non-media +GET /hw-capabilities non-media +GET /status non-media +POST /transcode/start non-media +DELETE /transcode/{session_id} non-media +GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false false +GET /transcode/{session_id}/segment/{name} media playback internal_relay false false From 7a49cca1dcd96dbd4a7e844b6cd72d61d5c76d5d Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:33:41 +0000 Subject: [PATCH 06/44] feat(streamtelemetry): publish snapshots to redis and merge a global view P0c of the stream telemetry and enforcement design. Adds the distributed read-only view: publisher sequencing, a Redis snapshot transport, freshness, and the merged GlobalMonitoringView with its complete/degraded flag. Still observation only, behind SILO_STREAM_TELEMETRY_DISTRIBUTED (default off). No election, no fence token, no sanctions, no admin endpoint, no /api/v1 change and no PostgreSQL write. Each process carries a random publisher id, a process epoch and a sequence incremented exactly once per published snapshot. Snapshots land in one Redis hash per publisher instance, so a stalled or oversized publisher cannot make every session vanish atomically the way a per-node blob would. Publishing is one MULTI/EXEC rather than a Lua script: Lua's unpack exceeds its C-stack limit past roughly eight thousand elements, which is inside the existing ten thousand session cap, and chunking the script would forfeit the atomicity it existed for. A concurrent HGETALL still observes one side of the update. Membership is the heartbeat itself. Publishers score themselves into a sorted-set roster, pruned by ZREMRANGEBYSCORE with a two-TTL margin so one clock cannot evict another. A heartbeat that is fresh while its snapshot is stale means the publisher stalled, so the view is degraded and names it; a heartbeat past the membership TTL means the process is gone, so it is dropped and the view is whole again. This matters because session byte totals are monotonic and consumers derive rates by subtraction: silently dropping a publisher makes a merged sum move backwards. BuildGlobalView is pure, taking the roster, decoded snapshots, errors, a build time and every bound as input, so the merge contract is unit tested without Redis. Viewer bytes sum only viewer-egress route activity and never the all-roles SessionView total, relay bytes stay separate for correlation, viewer addresses union, open observations sum, and identity is contributed only by publishers that authenticated the request. A populated disagreement over subject, profile or media file records every value with its publishers and leaves the scalar empty rather than picking an edge; play method gets no merged scalar at all, because no available timestamp can prove which publisher's value is later. Completeness additionally requires that no publisher truncated, that the reader hit no cap and that nothing failed to decode, and the view names the reasons it is incomplete. Wire values are versioned JSON with explicit field tags, Unix nanosecond times that preserve the zero time, and decode-time rejection of negative counters. The design asked for compact binary; JSON is a deliberate deviation, isolated behind the store interface, taken because a hand-written binary codec for a struct with this many maps and slices was the likeliest source of defects in a change whose whole value is a correct merge. Encoding cost and size are benchmarked and recorded. --- cmd/silo/main.go | 27 +- internal/streamtelemetry/codec.go | 310 +++++++++ internal/streamtelemetry/codec_bench_test.go | 58 ++ internal/streamtelemetry/codec_test.go | 133 ++++ internal/streamtelemetry/config.go | 114 +++- internal/streamtelemetry/config_test.go | 88 ++- internal/streamtelemetry/doc.go | 8 +- internal/streamtelemetry/global.go | 634 +++++++++++++++++++ internal/streamtelemetry/global_test.go | 217 +++++++ internal/streamtelemetry/registry.go | 74 ++- internal/streamtelemetry/registry_test.go | 152 +++++ internal/streamtelemetry/session.go | 2 +- internal/streamtelemetry/store.go | 55 ++ internal/streamtelemetry/store_redis.go | 392 ++++++++++++ internal/streamtelemetry/store_redis_test.go | 339 ++++++++++ internal/streamtelemetry/view.go | 2 + 16 files changed, 2561 insertions(+), 44 deletions(-) create mode 100644 internal/streamtelemetry/codec.go create mode 100644 internal/streamtelemetry/codec_bench_test.go create mode 100644 internal/streamtelemetry/codec_test.go create mode 100644 internal/streamtelemetry/global.go create mode 100644 internal/streamtelemetry/global_test.go create mode 100644 internal/streamtelemetry/store_redis.go create mode 100644 internal/streamtelemetry/store_redis_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 1b65a8a77..02861fd01 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -692,11 +692,6 @@ func main() { appCtx, appCancel := context.WithCancel(ctx) defer appCancel() var streamTelemetryRegistry *streamtelemetry.Registry - if mode == "" || mode == "integrated" || mode == "api" { - streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) - streamTelemetryRegistry = streamtelemetry.NewRegistry(streamTelemetryConfig, streamtelemetry.NewLocalStore(), slog.Default()) - streamTelemetryRegistry.Start(appCtx) - } restartReqCh := make(chan struct{}, 1) var restartRequested atomic.Bool @@ -850,6 +845,25 @@ func main() { defer func() { _ = apiRedisClient.Close() }() } + if mode == "" || mode == "integrated" || mode == "api" { + streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) + store := streamtelemetry.GlobalSnapshotStore(streamtelemetry.NewLocalStore()) + if streamTelemetryConfig.Enabled && streamTelemetryConfig.Distributed { + if apiRedisClient != nil { + store = streamtelemetry.NewRedisStore(apiRedisClient, streamTelemetryConfig, slog.Default()) + pingCtx, pingCancel := context.WithTimeout(appCtx, 2*time.Second) + if pingErr := apiRedisClient.Ping(pingCtx).Err(); pingErr != nil { + slog.Error("stream telemetry distributed mode cannot reach redis; publisher will retry each sweep", "address", apiRedisClient.Options().Addr, "error", pingErr) + } + pingCancel() + } else { + slog.Error("stream telemetry distributed mode requested but redis is not configured; using local store") + } + } + streamTelemetryRegistry = streamtelemetry.NewRegistry(streamTelemetryConfig, store, slog.Default()) + streamTelemetryRegistry.Start(appCtx) + } + // Assigned below once the trusted-proxy config is seeded; captured by the // OnServerSettingUpdated closure, which only runs on admin requests after // startup completes. @@ -2836,6 +2850,9 @@ func main() { slog.Error("abs compat shutdown error", "error", shutdownErr) } } + if stopErr := streamTelemetryRegistry.Stop(shutdownCtx); stopErr != nil { + slog.Error("stream telemetry shutdown error", "error", stopErr) + } // 2. Clean up stale sessions. if sessionCleaner != nil { diff --git a/internal/streamtelemetry/codec.go b/internal/streamtelemetry/codec.go new file mode 100644 index 000000000..9f5376add --- /dev/null +++ b/internal/streamtelemetry/codec.go @@ -0,0 +1,310 @@ +package streamtelemetry + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +const ( + codecVersion = 1 + maxWireSlice = 100_000 + maxWireMap = 4_096 +) + +type errUnsupportedCodecVersion struct{ Version int } + +func (e errUnsupportedCodecVersion) Error() string { + return fmt.Sprintf("unsupported stream telemetry codec version %d", e.Version) +} + +type wireSubject struct { + Kind SubjectKind `json:"k"` + ID string `json:"id"` +} + +type wireClientVariant struct { + Name string `json:"n"` + Version string `json:"v"` + Build string `json:"b"` + Channel string `json:"c"` +} + +type wireRouteActivity struct { + Method string `json:"m"` + Pattern string `json:"p"` + Role Role `json:"r"` + Class Class `json:"c"` + CapRelevant bool `json:"cr"` + Open int `json:"o"` + Requests int64 `json:"rq"` + BytesAccepted int64 `json:"b"` + LastByteAccepted int64 `json:"lb"` + LastObservationEnd int64 `json:"le"` +} + +type wireIdentityConflict struct { + Field string `json:"f"` + Existing string `json:"e"` + Offered string `json:"o"` + ObservedAt int64 `json:"at"` +} + +type wireSession struct { + V int `json:"v"` + Subject wireSubject `json:"sub"` + ProfileID string `json:"pid"` + SessionID string `json:"sid"` + MediaFileID int `json:"mfid"` + PlayMethod string `json:"pm"` + MediaFileIDs []int `json:"mfids"` + MediaFileIDsOverflowed bool `json:"mfido"` + PlayMethods []string `json:"pms"` + PlayMethodsOverflowed bool `json:"pmo"` + StartedAt int64 `json:"st"` + StartedAtSource StartedAtSource `json:"sts"` + StartedAtDegraded bool `json:"std"` + BytesAccepted int64 `json:"ba"` + LastByteAccepted int64 `json:"lb"` + LastObservationEnd int64 `json:"le"` + OpenObservations int `json:"oo"` + RealtimeConnectionAlive bool `json:"rt"` + RequestCount int64 `json:"rc"` + Routes []wireRouteActivity `json:"routes"` + RoutesOverflowed bool `json:"ro"` + ViewerIPs []string `json:"ips"` + ViewerIPsOverflowed bool `json:"ipso"` + DeviceIDs []string `json:"dids"` + DeviceIDsOverflowed bool `json:"didso"` + ClientVariants []wireClientVariant `json:"clients"` + ClientVariantsOverflowed bool `json:"clientso"` + UserAgents []string `json:"uas"` + UserAgentsOverflowed bool `json:"uaso"` + TokenIssuedAts []int64 `json:"tiats"` + TokenIssuedAtsOverflowed bool `json:"tiatso"` + TokenIssuedAtSources map[TokenIssuedAtSource]int64 `json:"tis"` + Outcomes map[httpstream.StreamOutcome]int64 `json:"out"` + HasIdentityConflict bool `json:"hic"` + IdentityConflicts []wireIdentityConflict `json:"ics"` + IdentityConflictsOverflowed bool `json:"icso"` +} + +type wireTransfer struct { + V int `json:"v"` + ID string `json:"id"` + Subject wireSubject `json:"sub"` + ProfileID string `json:"pid"` + MediaFileID int `json:"mfid"` + Method string `json:"m"` + Pattern string `json:"p"` + Role Role `json:"r"` + BytesAccepted int64 `json:"ba"` + LastByteAccepted int64 `json:"lb"` + LastObservationEnd int64 `json:"le"` + OpenObservations int `json:"oo"` + RequestCount int64 `json:"rc"` + ViewerIP string `json:"ip"` + DeviceID string `json:"did"` + Client wireClientVariant `json:"client"` + UserAgent string `json:"ua"` + Outcomes map[httpstream.StreamOutcome]int64 `json:"out"` +} + +type publisherMeta struct { + V int `json:"v"` + PublisherID string `json:"pid"` + NodeID string `json:"nid"` + Epoch int64 `json:"ep"` + Sequence uint64 `json:"sq"` + CapturedAtUnixNano int64 `json:"cap"` + Truncated bool `json:"tr"` + DroppedObservations int64 `json:"do"` + DroppedBytes int64 `json:"db"` + UnattributedObservations int64 `json:"uo"` + UnattributedBytes int64 `json:"ub"` + SessionCount int `json:"sc"` + TransferCount int `json:"tc"` +} + +func timeToUnixNano(value time.Time) int64 { + if value.IsZero() { + return 0 + } + return value.UnixNano() +} + +func timeFromUnixNano(value int64) time.Time { + if value == 0 { + return time.Time{} + } + return time.Unix(0, value) +} + +func checkVersion(data []byte) error { + var header struct { + V int `json:"v"` + } + if err := json.Unmarshal(data, &header); err != nil { + return err + } + if header.V != codecVersion { + return errUnsupportedCodecVersion{Version: header.V} + } + return nil +} + +func encodeSession(value SessionView) ([]byte, error) { + w := wireSession{V: codecVersion, Subject: wireSubject{Kind: value.Subject.Kind, ID: value.Subject.ID}, ProfileID: value.ProfileID, + SessionID: value.SessionID, MediaFileID: value.MediaFileID, PlayMethod: value.PlayMethod, + MediaFileIDs: value.MediaFileIDs, MediaFileIDsOverflowed: value.MediaFileIDsOverflowed, + PlayMethods: value.PlayMethods, PlayMethodsOverflowed: value.PlayMethodsOverflowed, + StartedAt: timeToUnixNano(value.StartedAt), StartedAtSource: value.StartedAtSource, StartedAtDegraded: value.StartedAtDegraded, + BytesAccepted: value.BytesAccepted, LastByteAccepted: timeToUnixNano(value.LastByteAccepted), LastObservationEnd: timeToUnixNano(value.LastObservationEnd), + OpenObservations: value.OpenObservations, RealtimeConnectionAlive: value.RealtimeConnectionAlive, RequestCount: value.RequestCount, + RoutesOverflowed: value.RoutesOverflowed, ViewerIPs: value.ViewerIPs, ViewerIPsOverflowed: value.ViewerIPsOverflowed, + DeviceIDs: value.DeviceIDs, DeviceIDsOverflowed: value.DeviceIDsOverflowed, UserAgents: value.UserAgents, UserAgentsOverflowed: value.UserAgentsOverflowed, + TokenIssuedAtsOverflowed: value.TokenIssuedAtsOverflowed, + TokenIssuedAtSources: value.TokenIssuedAtSources, Outcomes: value.Outcomes, HasIdentityConflict: value.HasIdentityConflict, + IdentityConflictsOverflowed: value.IdentityConflictsOverflowed, ClientVariantsOverflowed: value.ClientVariantsOverflowed} + for _, route := range value.Routes { + w.Routes = append(w.Routes, wireRouteActivity{Method: route.Method, Pattern: route.Pattern, Role: route.Role, Class: route.Class, CapRelevant: route.CapRelevant, Open: route.Open, Requests: route.Requests, BytesAccepted: route.BytesAccepted, LastByteAccepted: timeToUnixNano(route.LastByteAccepted), LastObservationEnd: timeToUnixNano(route.LastObservationEnd)}) + } + for _, client := range value.ClientVariants { + w.ClientVariants = append(w.ClientVariants, wireClientVariant(client)) + } + for _, issued := range value.TokenIssuedAts { + w.TokenIssuedAts = append(w.TokenIssuedAts, timeToUnixNano(issued)) + } + for _, conflict := range value.IdentityConflicts { + w.IdentityConflicts = append(w.IdentityConflicts, wireIdentityConflict{Field: conflict.Field, Existing: conflict.Existing, Offered: conflict.Offered, ObservedAt: timeToUnixNano(conflict.ObservedAt)}) + } + return json.Marshal(w) +} + +func decodeSession(data []byte) (SessionView, error) { + if err := checkVersion(data); err != nil { + return SessionView{}, err + } + var w wireSession + if err := json.Unmarshal(data, &w); err != nil { + return SessionView{}, err + } + if err := validateSessionWire(w); err != nil { + return SessionView{}, err + } + v := SessionView{Subject: Subject{Kind: w.Subject.Kind, ID: w.Subject.ID}, ProfileID: w.ProfileID, SessionID: w.SessionID, + MediaFileID: w.MediaFileID, PlayMethod: w.PlayMethod, MediaFileIDs: w.MediaFileIDs, MediaFileIDsOverflowed: w.MediaFileIDsOverflowed, + PlayMethods: w.PlayMethods, PlayMethodsOverflowed: w.PlayMethodsOverflowed, StartedAt: timeFromUnixNano(w.StartedAt), StartedAtSource: w.StartedAtSource, + StartedAtDegraded: w.StartedAtDegraded, BytesAccepted: w.BytesAccepted, LastByteAccepted: timeFromUnixNano(w.LastByteAccepted), LastObservationEnd: timeFromUnixNano(w.LastObservationEnd), + OpenObservations: w.OpenObservations, RealtimeConnectionAlive: w.RealtimeConnectionAlive, RequestCount: w.RequestCount, RoutesOverflowed: w.RoutesOverflowed, + ViewerIPs: w.ViewerIPs, ViewerIPsOverflowed: w.ViewerIPsOverflowed, DeviceIDs: w.DeviceIDs, DeviceIDsOverflowed: w.DeviceIDsOverflowed, + UserAgents: w.UserAgents, UserAgentsOverflowed: w.UserAgentsOverflowed, TokenIssuedAtsOverflowed: w.TokenIssuedAtsOverflowed, + TokenIssuedAtSources: w.TokenIssuedAtSources, Outcomes: w.Outcomes, HasIdentityConflict: w.HasIdentityConflict, + IdentityConflictsOverflowed: w.IdentityConflictsOverflowed, ClientVariantsOverflowed: w.ClientVariantsOverflowed} + for _, route := range w.Routes { + v.Routes = append(v.Routes, RouteActivityView{Method: route.Method, Pattern: route.Pattern, Role: route.Role, Class: route.Class, CapRelevant: route.CapRelevant, Open: route.Open, Requests: route.Requests, BytesAccepted: route.BytesAccepted, LastByteAccepted: timeFromUnixNano(route.LastByteAccepted), LastObservationEnd: timeFromUnixNano(route.LastObservationEnd)}) + } + for _, client := range w.ClientVariants { + v.ClientVariants = append(v.ClientVariants, ClientVariant(client)) + } + for _, issued := range w.TokenIssuedAts { + v.TokenIssuedAts = append(v.TokenIssuedAts, timeFromUnixNano(issued)) + } + for _, conflict := range w.IdentityConflicts { + v.IdentityConflicts = append(v.IdentityConflicts, IdentityConflict{Field: conflict.Field, Existing: conflict.Existing, Offered: conflict.Offered, ObservedAt: timeFromUnixNano(conflict.ObservedAt)}) + } + return v, nil +} + +func validateSessionWire(w wireSession) error { + if w.MediaFileID < 0 || w.BytesAccepted < 0 || w.OpenObservations < 0 || w.RequestCount < 0 { + return errors.New("negative session counter") + } + lengths := []int{len(w.MediaFileIDs), len(w.PlayMethods), len(w.Routes), len(w.ViewerIPs), len(w.DeviceIDs), len(w.ClientVariants), len(w.UserAgents), len(w.TokenIssuedAts), len(w.IdentityConflicts)} + for _, length := range lengths { + if length > maxWireSlice { + return errors.New("session collection too large") + } + } + if len(w.TokenIssuedAtSources) > maxWireMap || len(w.Outcomes) > maxWireMap { + return errors.New("session map too large") + } + for _, value := range w.MediaFileIDs { + if value < 0 { + return errors.New("negative media file id") + } + } + for _, route := range w.Routes { + if route.Open < 0 || route.Requests < 0 || route.BytesAccepted < 0 { + return errors.New("negative route counter") + } + } + for _, value := range w.TokenIssuedAtSources { + if value < 0 { + return errors.New("negative token source counter") + } + } + for _, value := range w.Outcomes { + if value < 0 { + return errors.New("negative outcome counter") + } + } + return nil +} + +func encodeTransfer(value TransferView) ([]byte, error) { + w := wireTransfer{V: codecVersion, ID: value.ID, Subject: wireSubject{Kind: value.Subject.Kind, ID: value.Subject.ID}, ProfileID: value.ProfileID, MediaFileID: value.MediaFileID, + Method: value.Method, Pattern: value.Pattern, Role: value.Role, BytesAccepted: value.BytesAccepted, LastByteAccepted: timeToUnixNano(value.LastByteAccepted), LastObservationEnd: timeToUnixNano(value.LastObservationEnd), + OpenObservations: value.OpenObservations, RequestCount: value.RequestCount, ViewerIP: value.ViewerIP, DeviceID: value.DeviceID, + Client: wireClientVariant(value.Client), UserAgent: value.UserAgent, Outcomes: value.Outcomes} + return json.Marshal(w) +} + +func decodeTransfer(data []byte) (TransferView, error) { + if err := checkVersion(data); err != nil { + return TransferView{}, err + } + var w wireTransfer + if err := json.Unmarshal(data, &w); err != nil { + return TransferView{}, err + } + if w.MediaFileID < 0 || w.BytesAccepted < 0 || w.OpenObservations < 0 || w.RequestCount < 0 { + return TransferView{}, errors.New("negative transfer counter") + } + if len(w.Outcomes) > maxWireMap { + return TransferView{}, errors.New("transfer map too large") + } + for _, value := range w.Outcomes { + if value < 0 { + return TransferView{}, errors.New("negative outcome counter") + } + } + return TransferView{ID: w.ID, Subject: Subject{Kind: w.Subject.Kind, ID: w.Subject.ID}, ProfileID: w.ProfileID, MediaFileID: w.MediaFileID, Method: w.Method, Pattern: w.Pattern, Role: w.Role, + BytesAccepted: w.BytesAccepted, LastByteAccepted: timeFromUnixNano(w.LastByteAccepted), LastObservationEnd: timeFromUnixNano(w.LastObservationEnd), OpenObservations: w.OpenObservations, + RequestCount: w.RequestCount, ViewerIP: w.ViewerIP, DeviceID: w.DeviceID, Client: ClientVariant(w.Client), UserAgent: w.UserAgent, Outcomes: w.Outcomes}, nil +} + +func encodeMeta(value publisherMeta) ([]byte, error) { + value.V = codecVersion + return json.Marshal(value) +} + +func decodeMeta(data []byte) (publisherMeta, error) { + if err := checkVersion(data); err != nil { + return publisherMeta{}, err + } + var value publisherMeta + if err := json.Unmarshal(data, &value); err != nil { + return publisherMeta{}, err + } + if value.DroppedObservations < 0 || value.DroppedBytes < 0 || value.UnattributedObservations < 0 || value.UnattributedBytes < 0 || value.SessionCount < 0 || value.TransferCount < 0 { + return publisherMeta{}, errors.New("negative publisher metadata counter") + } + if value.SessionCount > maxWireSlice || value.TransferCount > maxWireSlice { + return publisherMeta{}, errors.New("publisher count too large") + } + return value, nil +} diff --git a/internal/streamtelemetry/codec_bench_test.go b/internal/streamtelemetry/codec_bench_test.go new file mode 100644 index 000000000..ddae5828e --- /dev/null +++ b/internal/streamtelemetry/codec_bench_test.go @@ -0,0 +1,58 @@ +package streamtelemetry + +import ( + "fmt" + "testing" + "time" +) + +var benchmarkTime = time.Unix(1_700_000_000, 0) + +func benchmarkSessionID(index int) string { return fmt.Sprintf("session-%05d", index) } + +func BenchmarkCodec(b *testing.B) { + session := populatedSessionView() + encoded, err := encodeSession(session) + if err != nil { + b.Fatal(err) + } + b.Run("encode_session", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(encoded))) + for b.Loop() { + if _, err := encodeSession(session); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(encoded)), "encoded_bytes") + }) + b.Run("decode_session", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(encoded))) + for b.Loop() { + if _, err := decodeSession(encoded); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(encoded)), "encoded_bytes") + }) +} + +func BenchmarkBuildGlobalView(b *testing.B) { + const sessionCount = 50_000 + cfg := DefaultConfig("node") + snapshot := Snapshot{PublisherID: "publisher", NodeID: "node", PublisherEpoch: 1, Sequence: 1, CapturedAt: benchmarkTime} + snapshot.Sessions = make([]SessionView, sessionCount) + for i := range snapshot.Sessions { + snapshot.Sessions[i] = SessionView{SessionID: benchmarkSessionID(i), Routes: []RouteActivityView{{Role: RoleViewerEgress, BytesAccepted: 1}}} + } + set := PublisherSet{Members: []Member{{PublisherID: "publisher", LastHeartbeat: benchmarkTime}}, Snapshots: []Snapshot{snapshot}} + params := ViewParams{Freshness: cfg.Freshness, MembershipTTL: cfg.MembershipTTL, MaxMergedSessions: sessionCount, MaxMergedTransfers: cfg.MaxMergedTransfers, + MaxViewerIPsPerSession: cfg.MaxViewerIPsPerSession, MaxDeviceIDsPerSession: cfg.MaxDeviceIDsPerSession, MaxClientVariantsPerSession: cfg.MaxClientVariantsPerSession, + MaxUserAgentsPerSession: cfg.MaxClientVariantsPerSession, MaxMediaFileIDsPerSession: cfg.MaxMediaFileIDsPerSession, MaxPlayMethodsPerSession: cfg.MaxPlayMethodsPerSession, + MaxTokenIssuedAtPerSession: cfg.MaxTokenIssuedAtPerSession, MaxRoutesPerSession: cfg.MaxRoutesPerSession, MaxIdentityConflictsPerSession: cfg.MaxIdentityConflictsPerSession} + b.ReportAllocs() + for b.Loop() { + _ = BuildGlobalView(set, benchmarkTime, params) + } +} diff --git a/internal/streamtelemetry/codec_test.go b/internal/streamtelemetry/codec_test.go new file mode 100644 index 000000000..db2b65746 --- /dev/null +++ b/internal/streamtelemetry/codec_test.go @@ -0,0 +1,133 @@ +package streamtelemetry + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +func populatedSessionView() SessionView { + t1 := time.Unix(1_700_000_000, 123) + t2 := time.Unix(1_700_000_100, 456) + return SessionView{ + Subject: Subject{Kind: SubjectUser, ID: "42"}, ProfileID: "家庭", SessionID: "session-1", MediaFileID: 7, PlayMethod: "direct", + MediaFileIDs: []int{7, 8}, MediaFileIDsOverflowed: true, PlayMethods: []string{"direct", "remux"}, PlayMethodsOverflowed: true, + StartedAt: t1, StartedAtSource: StartedAtSourceClaim, StartedAtDegraded: true, BytesAccepted: 99, + LastByteAccepted: t2, LastObservationEnd: time.Time{}, OpenObservations: 2, RealtimeConnectionAlive: true, RequestCount: 3, + Routes: []RouteActivityView{{Method: "GET", Pattern: "/媒体/{id}", Role: RoleViewerEgress, Class: ClassPlayback, CapRelevant: true, Open: 1, Requests: 2, BytesAccepted: 90, LastByteAccepted: t2}}, RoutesOverflowed: true, + ViewerIPs: []string{"192.0.2.1"}, ViewerIPsOverflowed: true, DeviceIDs: []string{"device"}, DeviceIDsOverflowed: true, + ClientVariants: []ClientVariant{{Name: "客户端", Version: "1", Build: "2", Channel: "beta"}}, ClientVariantsOverflowed: true, + UserAgents: []string{"播放器/日本語 🚀"}, UserAgentsOverflowed: true, TokenIssuedAts: []time.Time{t1, {}}, TokenIssuedAtsOverflowed: true, + TokenIssuedAtSources: map[TokenIssuedAtSource]int64{TokenIssuedAtSourceVerified: 2, TokenIssuedAtSource("future"): 1}, + Outcomes: map[httpstream.StreamOutcome]int64{httpstream.OutcomeCompleted: 1, httpstream.StreamOutcome("future_outcome"): 2}, + HasIdentityConflict: true, IdentityConflicts: []IdentityConflict{{Field: "profile_id", Existing: "a", Offered: "b", ObservedAt: t2}}, IdentityConflictsOverflowed: true, + } +} + +func TestSessionCodecRoundTrip(t *testing.T) { + want := populatedSessionView() + encoded, err := encodeSession(want) + if err != nil { + t.Fatal(err) + } + got, err := decodeSession(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip mismatch\n got: %#v\nwant: %#v", got, want) + } +} + +func TestSessionCodecEmptyRoundTrip(t *testing.T) { + encoded, err := encodeSession(SessionView{}) + if err != nil { + t.Fatal(err) + } + got, err := decodeSession(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, SessionView{}) { + t.Fatalf("empty round trip = %#v", got) + } +} + +func TestTransferCodecRoundTrip(t *testing.T) { + want := TransferView{ID: "transfer", Subject: Subject{Kind: SubjectIP, ID: "203.0.113.4"}, ProfileID: "p", MediaFileID: 5, + Method: "GET", Pattern: "/download", Role: RoleViewerEgress, BytesAccepted: 12, LastByteAccepted: time.Unix(10, 11), + OpenObservations: 1, RequestCount: 2, ViewerIP: "203.0.113.4", DeviceID: "d", Client: ClientVariant{Name: "c"}, UserAgent: "ua", + Outcomes: map[httpstream.StreamOutcome]int64{httpstream.OutcomeCompleted: 1}} + encoded, err := encodeTransfer(want) + if err != nil { + t.Fatal(err) + } + got, err := decodeTransfer(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip mismatch\n got: %#v\nwant: %#v", got, want) + } +} + +func TestPublisherMetaCodecRoundTrip(t *testing.T) { + want := publisherMeta{PublisherID: "pub", NodeID: "node", Epoch: 4, Sequence: 5, CapturedAtUnixNano: 6, Truncated: true, + DroppedObservations: 7, DroppedBytes: 8, UnattributedObservations: 9, UnattributedBytes: 10, SessionCount: 11, TransferCount: 12} + encoded, err := encodeMeta(want) + if err != nil { + t.Fatal(err) + } + got, err := decodeMeta(encoded) + if err != nil { + t.Fatal(err) + } + want.V = codecVersion + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip = %#v, want %#v", got, want) + } +} + +func TestCodecRejectsMalformedAndUnsupportedWithoutPanic(t *testing.T) { + for name, data := range map[string][]byte{"truncated": []byte(`{"v":1`), "unsupported": []byte(`{"v":999}`)} { + t.Run(name, func(t *testing.T) { + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("decode panicked: %v", recovered) + } + }() + _, err := decodeSession(data) + if err == nil { + t.Fatal("decode succeeded") + } + if name == "unsupported" { + var unsupported errUnsupportedCodecVersion + if !errors.As(err, &unsupported) { + t.Fatalf("error = %T %v", err, err) + } + } + }) + } +} + +func TestCodecKeepsUnknownOutcomeKey(t *testing.T) { + got, err := decodeSession([]byte(`{"v":1,"out":{"from_the_future":3}}`)) + if err != nil { + t.Fatal(err) + } + if got.Outcomes[httpstream.StreamOutcome("from_the_future")] != 3 { + t.Fatalf("outcomes = %#v", got.Outcomes) + } +} + +func TestCodecRejectsNegativeCounter(t *testing.T) { + if _, err := decodeSession([]byte(`{"v":1,"ba":-1}`)); err == nil { + t.Fatal("negative counter accepted") + } + if _, err := decodeMeta([]byte(`{"v":1,"do":-1}`)); err == nil { + t.Fatal("negative metadata counter accepted") + } +} diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index 9356720fa..b26bfdead 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -6,24 +6,42 @@ import ( "strconv" "strings" "time" + "unicode" ) const ( - enabledEnv = "SILO_STREAM_TELEMETRY_ENABLED" - sweepIntervalEnv = "SILO_STREAM_TELEMETRY_SWEEP_INTERVAL" - retentionEnv = "SILO_STREAM_TELEMETRY_RETENTION" - maxSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_SESSIONS" - maxTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_TRANSFERS" - maxObservationsEnv = "SILO_STREAM_TELEMETRY_MAX_OBSERVATIONS" + enabledEnv = "SILO_STREAM_TELEMETRY_ENABLED" + sweepIntervalEnv = "SILO_STREAM_TELEMETRY_SWEEP_INTERVAL" + retentionEnv = "SILO_STREAM_TELEMETRY_RETENTION" + maxSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_SESSIONS" + maxTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_TRANSFERS" + maxObservationsEnv = "SILO_STREAM_TELEMETRY_MAX_OBSERVATIONS" + distributedEnv = "SILO_STREAM_TELEMETRY_DISTRIBUTED" + freshnessEnv = "SILO_STREAM_TELEMETRY_FRESHNESS" + membershipTTLEnv = "SILO_STREAM_TELEMETRY_MEMBERSHIP_TTL" + keyPrefixEnv = "SILO_STREAM_TELEMETRY_KEY_PREFIX" + fullResyncEveryEnv = "SILO_STREAM_TELEMETRY_FULL_RESYNC_EVERY" + maxPublishersEnv = "SILO_STREAM_TELEMETRY_MAX_PUBLISHERS" + maxMergedSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_MERGED_SESSIONS" + maxMergedTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_MERGED_TRANSFERS" ) type Config struct { - Enabled bool - NodeID string - PublisherID string + Enabled bool + NodeID string + PublisherID string + PublisherEpoch int64 + Distributed bool - SweepInterval time.Duration - Retention time.Duration + SweepInterval time.Duration + Retention time.Duration + Freshness time.Duration + MembershipTTL time.Duration + KeyPrefix string + FullResyncEvery int + MaxPublishers int + MaxMergedSessions int + MaxMergedTransfers int MaxSessions int64 MaxTransfers int64 @@ -42,6 +60,8 @@ type Config struct { func DefaultConfig(nodeID string) Config { return Config{ NodeID: nodeID, SweepInterval: time.Second, Retention: 5 * time.Minute, + Freshness: 5 * time.Second, MembershipTTL: time.Minute, KeyPrefix: "silo:stelem", + FullResyncEvery: 60, MaxPublishers: 256, MaxMergedSessions: 50_000, MaxMergedTransfers: 50_000, MaxSessions: 10_000, MaxTransfers: 10_000, MaxObservations: 50_000, MaxObservationsPerSession: 64, MaxViewerIPsPerSession: 32, MaxIdentityConflictsPerSession: 16, MaxDeviceIDsPerSession: 32, @@ -51,12 +71,14 @@ func DefaultConfig(nodeID string) Config { } } -// ConfigFromEnv returns a safe configuration. Invalid telemetry settings are -// ignored while disabled; while enabled they disable telemetry and are logged. +// ConfigFromEnv returns a safe configuration. Invalid core settings disable +// telemetry; invalid distributed-only settings retain local telemetry. func ConfigFromEnv(nodeID string) Config { cfg := DefaultConfig(nodeID) cfg.Enabled = envEnabled(os.Getenv(enabledEnv)) - invalid := make([]string, 0) + coreInvalid := make([]string, 0) + distributedInvalid := make([]string, 0) + cfg.Distributed = envEnabled(os.Getenv(distributedEnv)) parseDuration := func(name string, dst *time.Duration) { value := strings.TrimSpace(os.Getenv(name)) if value == "" { @@ -64,7 +86,19 @@ func ConfigFromEnv(nodeID string) Config { } parsed, err := time.ParseDuration(value) if err != nil || parsed <= 0 { - invalid = append(invalid, name) + coreInvalid = append(coreInvalid, name) + return + } + *dst = parsed + } + parseDistributedDuration := func(name string, dst *time.Duration) { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed <= 0 { + distributedInvalid = append(distributedInvalid, name) return } *dst = parsed @@ -76,7 +110,19 @@ func ConfigFromEnv(nodeID string) Config { } parsed, err := strconv.ParseInt(value, 10, 64) if err != nil || parsed <= 0 { - invalid = append(invalid, name) + coreInvalid = append(coreInvalid, name) + return + } + *dst = parsed + } + parseDistributedPositive := func(name string, dst *int) { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed <= 0 { + distributedInvalid = append(distributedInvalid, name) return } *dst = parsed @@ -86,12 +132,42 @@ func ConfigFromEnv(nodeID string) Config { parsePositive(maxSessionsEnv, &cfg.MaxSessions) parsePositive(maxTransfersEnv, &cfg.MaxTransfers) parsePositive(maxObservationsEnv, &cfg.MaxObservations) - if len(invalid) > 0 { + parseDistributedDuration(freshnessEnv, &cfg.Freshness) + parseDistributedDuration(membershipTTLEnv, &cfg.MembershipTTL) + parseDistributedPositive(fullResyncEveryEnv, &cfg.FullResyncEvery) + parseDistributedPositive(maxPublishersEnv, &cfg.MaxPublishers) + parseDistributedPositive(maxMergedSessionsEnv, &cfg.MaxMergedSessions) + parseDistributedPositive(maxMergedTransfersEnv, &cfg.MaxMergedTransfers) + if value := os.Getenv(keyPrefixEnv); value != "" { + if strings.TrimSpace(value) == "" || strings.IndexFunc(value, unicode.IsSpace) >= 0 { + distributedInvalid = append(distributedInvalid, keyPrefixEnv) + } else { + cfg.KeyPrefix = value + } + } + if cfg.SweepInterval > time.Duration(1<<63-1)/3 || cfg.Freshness < 3*cfg.SweepInterval { + distributedInvalid = append(distributedInvalid, freshnessEnv) + } + if cfg.MembershipTTL <= cfg.Freshness { + distributedInvalid = append(distributedInvalid, membershipTTLEnv) + } + if cfg.MembershipTTL > time.Duration(1<<63-1)/10 { + distributedInvalid = append(distributedInvalid, membershipTTLEnv) + } + if len(coreInvalid) > 0 { if cfg.Enabled { cfg.Enabled = false - slog.Error("stream telemetry disabled because configuration is invalid", "variables", strings.Join(invalid, ",")) + slog.Error("stream telemetry disabled because configuration is invalid", "variables", strings.Join(coreInvalid, ",")) + } else { + slog.Warn("ignoring invalid disabled stream telemetry configuration", "variables", strings.Join(coreInvalid, ",")) + } + } + if len(distributedInvalid) > 0 { + if cfg.Distributed { + cfg.Distributed = false + slog.Error("stream telemetry distributed mode disabled because configuration is invalid", "variables", strings.Join(distributedInvalid, ",")) } else { - slog.Warn("ignoring invalid disabled stream telemetry configuration", "variables", strings.Join(invalid, ",")) + slog.Warn("ignoring invalid distributed stream telemetry configuration", "variables", strings.Join(distributedInvalid, ",")) } } return cfg diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index 8916676e5..d03ca6c49 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -9,10 +9,28 @@ func TestConfigFromEnvValidation(t *testing.T) { t.Run("defaults", func(t *testing.T) { clearConfigEnv(t) cfg := ConfigFromEnv("node") - if cfg.Enabled || cfg.SweepInterval != time.Second || cfg.Retention != 5*time.Minute || cfg.MaxObservations != 50_000 { + if cfg.Enabled || cfg.Distributed || cfg.SweepInterval != time.Second || cfg.Retention != 5*time.Minute || cfg.MaxObservations != 50_000 || + cfg.Freshness != 5*time.Second || cfg.MembershipTTL != time.Minute || cfg.KeyPrefix != "silo:stelem" || cfg.FullResyncEvery != 60 || cfg.MaxPublishers != 256 || cfg.MaxMergedSessions != 50_000 || cfg.MaxMergedTransfers != 50_000 { t.Fatalf("defaults = %+v", cfg) } }) + t.Run("valid distributed overrides", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(sweepIntervalEnv, "2s") + t.Setenv(freshnessEnv, "7s") + t.Setenv(membershipTTLEnv, "20s") + t.Setenv(keyPrefixEnv, "custom:telemetry") + t.Setenv(fullResyncEveryEnv, "7") + t.Setenv(maxPublishersEnv, "8") + t.Setenv(maxMergedSessionsEnv, "9") + t.Setenv(maxMergedTransfersEnv, "10") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.Distributed || cfg.Freshness != 7*time.Second || cfg.MembershipTTL != 20*time.Second || cfg.KeyPrefix != "custom:telemetry" || cfg.FullResyncEvery != 7 || cfg.MaxPublishers != 8 || cfg.MaxMergedSessions != 9 || cfg.MaxMergedTransfers != 10 { + t.Fatalf("distributed overrides = %+v", cfg) + } + }) t.Run("valid enabled overrides", func(t *testing.T) { clearConfigEnv(t) t.Setenv(enabledEnv, "true") @@ -40,11 +58,77 @@ func TestConfigFromEnvValidation(t *testing.T) { t.Fatalf("disabled invalid config = %+v", cfg) } }) + for name, variable := range map[string]string{ + "freshness": freshnessEnv, "membership ttl": membershipTTLEnv, "full resync": fullResyncEveryEnv, + "max publishers": maxPublishersEnv, "max sessions": maxMergedSessionsEnv, "max transfers": maxMergedTransfersEnv, + } { + t.Run("invalid distributed "+name+" falls back local", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(variable, "invalid") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed { + t.Fatalf("invalid distributed config = %+v", cfg) + } + }) + } + t.Run("invalid distributed while disabled warns and stays disabled", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(maxPublishersEnv, "0") + cfg := ConfigFromEnv("node") + if cfg.Enabled || cfg.Distributed { + t.Fatalf("disabled config = %+v", cfg) + } + }) + t.Run("freshness below three sweeps", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(sweepIntervalEnv, "2s") + t.Setenv(freshnessEnv, "5s") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed { + t.Fatalf("config = %+v", cfg) + } + }) + t.Run("membership not above freshness", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(freshnessEnv, "10s") + t.Setenv(membershipTTLEnv, "10s") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed { + t.Fatalf("config = %+v", cfg) + } + }) + t.Run("whitespace prefix", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(keyPrefixEnv, "bad prefix") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed { + t.Fatalf("config = %+v", cfg) + } + }) + t.Run("membership expiry overflow", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(membershipTTLEnv, "2562047h47m16s") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed { + t.Fatalf("config = %+v", cfg) + } + }) } func clearConfigEnv(t *testing.T) { t.Helper() - for _, name := range []string{enabledEnv, sweepIntervalEnv, retentionEnv, maxSessionsEnv, maxTransfersEnv, maxObservationsEnv} { + for _, name := range []string{enabledEnv, sweepIntervalEnv, retentionEnv, maxSessionsEnv, maxTransfersEnv, maxObservationsEnv, + distributedEnv, freshnessEnv, membershipTTLEnv, keyPrefixEnv, fullResyncEveryEnv, maxPublishersEnv, maxMergedSessionsEnv, maxMergedTransfersEnv} { t.Setenv(name, "") } } diff --git a/internal/streamtelemetry/doc.go b/internal/streamtelemetry/doc.go index 6e5c63ac2..930aca7b2 100644 --- a/internal/streamtelemetry/doc.go +++ b/internal/streamtelemetry/doc.go @@ -12,6 +12,10 @@ // where Observe is enrolled. It is wire bytes on bulk routes that bypass outer // compression, and pre-compression bytes on compressible subtitle/font routes. // -// P0b is deliberately local and observational. This package performs no -// admission, throttling, cutting, persistence, or distributed publication. +// The package is deliberately observational. It performs no admission, +// throttling, cutting, or PostgreSQL persistence. It does publish: when +// distributed mode is on, each process publishes its own snapshot to Redis and +// BuildGlobalView merges every fresh publisher into one read-only view. That +// path is still write-only telemetry — no enforcement reads it, and no /api/v1 +// response is served from it. package streamtelemetry diff --git a/internal/streamtelemetry/global.go b/internal/streamtelemetry/global.go new file mode 100644 index 000000000..25436ba0f --- /dev/null +++ b/internal/streamtelemetry/global.go @@ -0,0 +1,634 @@ +package streamtelemetry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" +) + +type PublisherRef struct { + PublisherID string + NodeID string +} + +type PublisherState string + +const ( + PublisherFresh PublisherState = "fresh" + PublisherStale PublisherState = "stale" + PublisherDegraded PublisherState = "degraded" + PublisherDeparted PublisherState = "departed" +) + +type PublisherStatus struct { + PublisherRef + Epoch int64 + Sequence uint64 + LastHeartbeat time.Time + CapturedAt time.Time + State PublisherState + DecodeErrors int + Truncated bool + Reason string +} + +type AttributedValue struct { + Value string + Publishers []PublisherRef +} + +type GlobalIdentityConflict struct { + Field string + Values []AttributedValue +} + +type AttributedIdentityConflict struct { + Publisher PublisherRef + Conflict IdentityConflict +} + +type PublisherValue struct { + Publisher PublisherRef + Value string +} + +type GlobalSessionView struct { + Subject Subject + ProfileID string + SessionID string + MediaFileID int + StartedAt time.Time + StartedAtSource StartedAtSource + StartedAtDegraded bool + ViewerBytesAccepted int64 + RelayBytesAccepted int64 + BytesDegraded bool + LastByteAccepted time.Time + LastObservationEnd time.Time + OpenObservations int64 + RealtimeConnectionAlive bool + RequestCount int64 + Routes []RouteActivityView + RoutesOverflowed bool + ViewerIPs []string + ViewerIPsOverflowed bool + DeviceIDs []string + DeviceIDsOverflowed bool + ClientVariants []ClientVariant + ClientVariantsOverflowed bool + UserAgents []string + UserAgentsOverflowed bool + MediaFileIDs []int + MediaFileIDsOverflowed bool + PlayMethods []string + PlayMethodsOverflowed bool + TokenIssuedAts []time.Time + TokenIssuedAtsOverflowed bool + TokenIssuedAtSources map[TokenIssuedAtSource]int64 + Outcomes map[httpstream.StreamOutcome]int64 + HasIdentityConflict bool + IdentityConflicts []GlobalIdentityConflict + LocalIdentityConflicts []AttributedIdentityConflict + IdentityConflictsOverflowed bool + Publishers []PublisherRef + ViewerEdgePublishers []PublisherRef + PerPublisherPlayMethods []PublisherValue +} + +type GlobalTransferView struct { + TransferView + Publisher PublisherRef +} + +type GlobalMonitoringView struct { + BuiltAt time.Time + Epoch string + Complete bool + IncompleteReasons []string + Publishers []PublisherStatus + MissingPublishers []PublisherRef + Sessions []GlobalSessionView + Transfers []GlobalTransferView + Truncated bool + DroppedObservations int64 + DroppedBytes int64 + UnattributedObservations int64 + UnattributedBytes int64 + DecodeErrors int + ClockSkewSuspected bool +} + +type ViewParams struct { + Freshness time.Duration + MembershipTTL time.Duration + MaxMergedSessions int + MaxMergedTransfers int + MaxViewerIPsPerSession int + MaxDeviceIDsPerSession int + MaxClientVariantsPerSession int + MaxUserAgentsPerSession int + MaxMediaFileIDsPerSession int + MaxPlayMethodsPerSession int + MaxTokenIssuedAtPerSession int + MaxRoutesPerSession int + MaxIdentityConflictsPerSession int +} + +type errGlobalSnapshotStoreUnsupported struct{} + +func (errGlobalSnapshotStoreUnsupported) Error() string { + return "stream telemetry store does not support global snapshots" +} + +func (r *Registry) viewParams() ViewParams { + return ViewParams{Freshness: r.cfg.Freshness, MembershipTTL: r.cfg.MembershipTTL, + MaxMergedSessions: r.cfg.MaxMergedSessions, MaxMergedTransfers: r.cfg.MaxMergedTransfers, + MaxViewerIPsPerSession: r.cfg.MaxViewerIPsPerSession, MaxDeviceIDsPerSession: r.cfg.MaxDeviceIDsPerSession, + MaxClientVariantsPerSession: r.cfg.MaxClientVariantsPerSession, MaxUserAgentsPerSession: r.cfg.MaxClientVariantsPerSession, + MaxMediaFileIDsPerSession: r.cfg.MaxMediaFileIDsPerSession, MaxPlayMethodsPerSession: r.cfg.MaxPlayMethodsPerSession, + MaxTokenIssuedAtPerSession: r.cfg.MaxTokenIssuedAtPerSession, MaxRoutesPerSession: r.cfg.MaxRoutesPerSession, + MaxIdentityConflictsPerSession: r.cfg.MaxIdentityConflictsPerSession} +} + +func (r *Registry) GlobalView(ctx context.Context) (GlobalMonitoringView, error) { + if r == nil || !r.cfg.Enabled { + return GlobalMonitoringView{}, nil + } + store, ok := r.store.(GlobalSnapshotStore) + if !ok { + return GlobalMonitoringView{}, errGlobalSnapshotStoreUnsupported{} + } + set, err := store.LoadAll(ctx) + if err != nil { + r.warnRateLimited("failed to load global stream telemetry", &r.lastPublishWarnUnixNano, "error", err) + return GlobalMonitoringView{}, err + } + return BuildGlobalView(set, now(), r.viewParams()), nil +} + +type publisherContribution struct { + ref PublisherRef + snapshot Snapshot +} + +type sessionContribution struct { + ref PublisherRef + view SessionView +} + +func BuildGlobalView(set PublisherSet, at time.Time, params ViewParams) GlobalMonitoringView { + view := GlobalMonitoringView{BuiltAt: at, Complete: true, Truncated: set.Truncated} + snapshots := make(map[string]Snapshot, len(set.Snapshots)) + for _, snapshot := range set.Snapshots { + snapshots[snapshot.PublisherID] = snapshot + } + errorsByPublisher := make(map[string][]PublisherError) + for _, problem := range set.Errors { + errorsByPublisher[problem.PublisherID] = append(errorsByPublisher[problem.PublisherID], problem) + view.DecodeErrors = saturatingInt(view.DecodeErrors, problem.DecodeErrors) + } + for publisherID := range errorsByPublisher { + problems := errorsByPublisher[publisherID] + sort.Slice(problems, func(i, j int) bool { + if problems[i].Reason == problems[j].Reason { + return problems[i].DecodeErrors < problems[j].DecodeErrors + } + return problems[i].Reason < problems[j].Reason + }) + errorsByPublisher[publisherID] = problems + } + members := append([]Member(nil), set.Members...) + sort.Slice(members, func(i, j int) bool { return members[i].PublisherID < members[j].PublisherID }) + contributions := make([]publisherContribution, 0, len(members)) + for _, member := range members { + snapshot, hasSnapshot := snapshots[member.PublisherID] + ref := PublisherRef{PublisherID: member.PublisherID, NodeID: snapshot.NodeID} + status := PublisherStatus{PublisherRef: ref, LastHeartbeat: member.LastHeartbeat, CapturedAt: snapshot.CapturedAt, + Epoch: snapshot.PublisherEpoch, Sequence: snapshot.Sequence, Truncated: snapshot.Truncated} + heartbeatAge := at.Sub(member.LastHeartbeat) + if heartbeatAge < -params.Freshness { + view.ClockSkewSuspected = true + } + if heartbeatAge > params.MembershipTTL { + status.State = PublisherDeparted + view.Publishers = append(view.Publishers, status) + continue + } + problems := errorsByPublisher[member.PublisherID] + unusable := false + for _, problem := range problems { + status.DecodeErrors = saturatingInt(status.DecodeErrors, problem.DecodeErrors) + if status.Reason == "" { + status.Reason = problem.Reason + } + if problem.Reason == publisherReasonOversized || problem.Reason == publisherReasonMetaMissing || problem.Reason == publisherReasonIdentityMismatch { + unusable = true + } + } + capturedAge := at.Sub(snapshot.CapturedAt) + if capturedAge < -params.Freshness { + view.ClockSkewSuspected = true + } + if !hasSnapshot || unusable || capturedAge > params.Freshness { + status.State = PublisherStale + view.MissingPublishers = append(view.MissingPublishers, ref) + view.Publishers = append(view.Publishers, status) + continue + } + status.State = PublisherFresh + if snapshot.Truncated || status.DecodeErrors > 0 || status.Reason == publisherReasonCountMismatch || status.Reason == publisherReasonDecode { + status.State = PublisherDegraded + } + view.Publishers = append(view.Publishers, status) + contributions = append(contributions, publisherContribution{ref: ref, snapshot: snapshot}) + } + sort.Slice(view.MissingPublishers, func(i, j int) bool { return refLess(view.MissingPublishers[i], view.MissingPublishers[j]) }) + if len(view.MissingPublishers) > 0 { + addReason(&view, "missing_publisher") + } + if set.Truncated { + addReason(&view, "truncated") + } + for _, status := range view.Publishers { + if (status.State == PublisherFresh || status.State == PublisherDegraded) && status.Truncated { + addReason(&view, "publisher_truncated") + } + if status.DecodeErrors > 0 || status.Reason == publisherReasonCountMismatch || status.Reason == publisherReasonDecode { + addReason(&view, "decode_errors") + } + } + mergeContributions(&view, contributions, params) + view.Epoch = globalEpoch(contributions) + view.Complete = len(view.IncompleteReasons) == 0 + return view +} + +func addReason(view *GlobalMonitoringView, reason string) { + for _, existing := range view.IncompleteReasons { + if existing == reason { + return + } + } + view.IncompleteReasons = append(view.IncompleteReasons, reason) + sort.Strings(view.IncompleteReasons) +} + +func mergeContributions(view *GlobalMonitoringView, contributions []publisherContribution, params ViewParams) { + sort.Slice(contributions, func(i, j int) bool { return refLess(contributions[i].ref, contributions[j].ref) }) + bySession := make(map[string][]sessionContribution) + for _, contribution := range contributions { + view.DroppedObservations = saturatingAdd(view.DroppedObservations, contribution.snapshot.DroppedObservations) + view.DroppedBytes = saturatingAdd(view.DroppedBytes, contribution.snapshot.DroppedBytes) + view.UnattributedObservations = saturatingAdd(view.UnattributedObservations, contribution.snapshot.UnattributedObservations) + view.UnattributedBytes = saturatingAdd(view.UnattributedBytes, contribution.snapshot.UnattributedBytes) + view.Truncated = view.Truncated || contribution.snapshot.Truncated + for _, session := range contribution.snapshot.Sessions { + bySession[session.SessionID] = append(bySession[session.SessionID], sessionContribution{ref: contribution.ref, view: session}) + } + for _, transfer := range contribution.snapshot.Transfers { + view.Transfers = append(view.Transfers, GlobalTransferView{TransferView: cloneTransfer(transfer), Publisher: contribution.ref}) + } + } + ids := make([]string, 0, len(bySession)) + for id := range bySession { + ids = append(ids, id) + } + sort.Strings(ids) + if params.MaxMergedSessions > 0 && len(ids) > params.MaxMergedSessions { + ids = ids[:params.MaxMergedSessions] + view.Truncated = true + addReason(view, "truncated") + } + for _, id := range ids { + view.Sessions = append(view.Sessions, mergeSession(id, bySession[id], params)) + } + sort.Slice(view.Transfers, func(i, j int) bool { + if view.Transfers[i].Publisher.PublisherID == view.Transfers[j].Publisher.PublisherID { + return view.Transfers[i].ID < view.Transfers[j].ID + } + return refLess(view.Transfers[i].Publisher, view.Transfers[j].Publisher) + }) + if params.MaxMergedTransfers > 0 && len(view.Transfers) > params.MaxMergedTransfers { + view.Transfers = view.Transfers[:params.MaxMergedTransfers] + view.Truncated = true + addReason(view, "truncated") + } +} + +func mergeSession(id string, contributions []sessionContribution, params ViewParams) GlobalSessionView { + result := GlobalSessionView{SessionID: id, TokenIssuedAtSources: make(map[TokenIssuedAtSource]int64), Outcomes: make(map[httpstream.StreamOutcome]int64)} + sort.Slice(contributions, func(i, j int) bool { return refLess(contributions[i].ref, contributions[j].ref) }) + viewerIPs, deviceIDs, userAgents, playMethods := map[string]struct{}{}, map[string]struct{}{}, map[string]struct{}{}, map[string]struct{}{} + mediaFileIDs, tokenTimes := map[int]struct{}{}, map[int64]struct{}{} + clients := map[ClientVariant]struct{}{} + routes := map[string]RouteActivityView{} + subjectValues, profileValues, mediaValues := map[string][]PublisherRef{}, map[string][]PublisherRef{}, map[string][]PublisherRef{} + winningRank := 0 + winningTimes := map[int64]struct{}{} + for _, contribution := range contributions { + session, ref := contribution.view, contribution.ref + result.Publishers = append(result.Publishers, ref) + viewerEdge := false + for _, route := range session.Routes { + if route.Role == RoleViewerEgress { + viewerEdge = true + break + } + } + if viewerEdge { + result.ViewerEdgePublishers = append(result.ViewerEdgePublishers, ref) + if session.Subject.Kind != "" && session.Subject.ID != "" { + key := string(session.Subject.Kind) + "\x00" + session.Subject.ID + subjectValues[key] = append(subjectValues[key], ref) + } + if session.ProfileID != "" { + profileValues[session.ProfileID] = append(profileValues[session.ProfileID], ref) + } + if session.MediaFileID != 0 { + mediaValues[strconv.Itoa(session.MediaFileID)] = append(mediaValues[strconv.Itoa(session.MediaFileID)], ref) + } + } + rank := startedAtRank(session.StartedAtSource) + if !session.StartedAt.IsZero() && rank > 0 { + if rank > winningRank { + winningRank = rank + result.StartedAt = session.StartedAt + result.StartedAtSource = session.StartedAtSource + winningTimes = map[int64]struct{}{session.StartedAt.UnixNano(): {}} + } else if rank == winningRank { + winningTimes[session.StartedAt.UnixNano()] = struct{}{} + if session.StartedAt.Before(result.StartedAt) { + result.StartedAt = session.StartedAt + } + } + } + result.StartedAtDegraded = result.StartedAtDegraded || session.StartedAtDegraded + result.OpenObservations = saturatingAdd(result.OpenObservations, int64(session.OpenObservations)) + result.RequestCount = saturatingAdd(result.RequestCount, session.RequestCount) + result.RealtimeConnectionAlive = result.RealtimeConnectionAlive || session.RealtimeConnectionAlive + if session.LastByteAccepted.After(result.LastByteAccepted) { + result.LastByteAccepted = session.LastByteAccepted + } + if session.LastObservationEnd.After(result.LastObservationEnd) { + result.LastObservationEnd = session.LastObservationEnd + } + result.RoutesOverflowed = result.RoutesOverflowed || session.RoutesOverflowed + result.BytesDegraded = result.BytesDegraded || session.RoutesOverflowed + for _, route := range session.Routes { + key := route.Method + "\x00" + route.Pattern + "\x00" + string(route.Role) + merged, seen := routes[key] + if !seen { + merged = route + } else { + merged.Open = saturatingInt64ToInt(saturatingAdd(int64(merged.Open), int64(route.Open))) + merged.Requests = saturatingAdd(merged.Requests, route.Requests) + merged.BytesAccepted = saturatingAdd(merged.BytesAccepted, route.BytesAccepted) + if route.LastByteAccepted.After(merged.LastByteAccepted) { + merged.LastByteAccepted = route.LastByteAccepted + } + if route.LastObservationEnd.After(merged.LastObservationEnd) { + merged.LastObservationEnd = route.LastObservationEnd + } + } + routes[key] = merged + if route.Role == RoleViewerEgress { + result.ViewerBytesAccepted = saturatingAdd(result.ViewerBytesAccepted, route.BytesAccepted) + } + if route.Role == RoleInternalRelay { + result.RelayBytesAccepted = saturatingAdd(result.RelayBytesAccepted, route.BytesAccepted) + } + } + for _, value := range session.ViewerIPs { + viewerIPs[value] = struct{}{} + } + for _, value := range session.DeviceIDs { + deviceIDs[value] = struct{}{} + } + for _, value := range session.UserAgents { + userAgents[value] = struct{}{} + } + for _, value := range session.ClientVariants { + clients[value] = struct{}{} + } + for _, value := range session.MediaFileIDs { + mediaFileIDs[value] = struct{}{} + } + if session.MediaFileID != 0 { + mediaFileIDs[session.MediaFileID] = struct{}{} + } + for _, value := range session.PlayMethods { + playMethods[value] = struct{}{} + } + if session.PlayMethod != "" { + playMethods[session.PlayMethod] = struct{}{} + result.PerPublisherPlayMethods = append(result.PerPublisherPlayMethods, PublisherValue{Publisher: ref, Value: session.PlayMethod}) + } + for _, value := range session.TokenIssuedAts { + tokenTimes[value.UnixNano()] = struct{}{} + } + for key, value := range session.TokenIssuedAtSources { + result.TokenIssuedAtSources[key] = saturatingAdd(result.TokenIssuedAtSources[key], value) + } + for key, value := range session.Outcomes { + result.Outcomes[key] = saturatingAdd(result.Outcomes[key], value) + } + result.ViewerIPsOverflowed = result.ViewerIPsOverflowed || session.ViewerIPsOverflowed + result.DeviceIDsOverflowed = result.DeviceIDsOverflowed || session.DeviceIDsOverflowed + result.ClientVariantsOverflowed = result.ClientVariantsOverflowed || session.ClientVariantsOverflowed + result.UserAgentsOverflowed = result.UserAgentsOverflowed || session.UserAgentsOverflowed + result.MediaFileIDsOverflowed = result.MediaFileIDsOverflowed || session.MediaFileIDsOverflowed + result.PlayMethodsOverflowed = result.PlayMethodsOverflowed || session.PlayMethodsOverflowed + result.TokenIssuedAtsOverflowed = result.TokenIssuedAtsOverflowed || session.TokenIssuedAtsOverflowed + result.IdentityConflictsOverflowed = result.IdentityConflictsOverflowed || session.IdentityConflictsOverflowed + for _, conflict := range session.IdentityConflicts { + result.LocalIdentityConflicts = append(result.LocalIdentityConflicts, AttributedIdentityConflict{Publisher: ref, Conflict: conflict}) + } + } + if winningRank == 1 || len(winningTimes) > 1 { + result.StartedAtDegraded = true + } + applyIdentity(&result, "subject", subjectValues) + applyIdentity(&result, identityFieldProfileID, profileValues) + applyIdentity(&result, "media_file_id", mediaValues) + result.ViewerIPs, result.ViewerIPsOverflowed = cappedStrings(viewerIPs, params.MaxViewerIPsPerSession, result.ViewerIPsOverflowed) + result.DeviceIDs, result.DeviceIDsOverflowed = cappedStrings(deviceIDs, params.MaxDeviceIDsPerSession, result.DeviceIDsOverflowed) + result.UserAgents, result.UserAgentsOverflowed = cappedStrings(userAgents, params.MaxUserAgentsPerSession, result.UserAgentsOverflowed) + result.PlayMethods, result.PlayMethodsOverflowed = cappedStrings(playMethods, params.MaxPlayMethodsPerSession, result.PlayMethodsOverflowed) + result.MediaFileIDs, result.MediaFileIDsOverflowed = cappedInts(mediaFileIDs, params.MaxMediaFileIDsPerSession, result.MediaFileIDsOverflowed) + result.ClientVariants, result.ClientVariantsOverflowed = cappedClients(clients, params.MaxClientVariantsPerSession, result.ClientVariantsOverflowed) + result.TokenIssuedAts, result.TokenIssuedAtsOverflowed = cappedTimes(tokenTimes, params.MaxTokenIssuedAtPerSession, result.TokenIssuedAtsOverflowed) + for _, route := range routes { + result.Routes = append(result.Routes, route) + } + sort.Slice(result.Routes, func(i, j int) bool { return routeViewKey(result.Routes[i]) < routeViewKey(result.Routes[j]) }) + if params.MaxRoutesPerSession > 0 && len(result.Routes) > params.MaxRoutesPerSession { + result.Routes = result.Routes[:params.MaxRoutesPerSession] + result.RoutesOverflowed = true + result.BytesDegraded = true + } + sort.Slice(result.IdentityConflicts, func(i, j int) bool { return result.IdentityConflicts[i].Field < result.IdentityConflicts[j].Field }) + if params.MaxIdentityConflictsPerSession > 0 && len(result.IdentityConflicts) > params.MaxIdentityConflictsPerSession { + result.IdentityConflicts = result.IdentityConflicts[:params.MaxIdentityConflictsPerSession] + result.IdentityConflictsOverflowed = true + } + sort.Slice(result.LocalIdentityConflicts, func(i, j int) bool { + a, b := result.LocalIdentityConflicts[i], result.LocalIdentityConflicts[j] + if a.Publisher.PublisherID != b.Publisher.PublisherID { + return refLess(a.Publisher, b.Publisher) + } + if a.Conflict.Field != b.Conflict.Field { + return a.Conflict.Field < b.Conflict.Field + } + if a.Conflict.Existing != b.Conflict.Existing { + return a.Conflict.Existing < b.Conflict.Existing + } + if a.Conflict.Offered != b.Conflict.Offered { + return a.Conflict.Offered < b.Conflict.Offered + } + return a.Conflict.ObservedAt.Before(b.Conflict.ObservedAt) + }) + if params.MaxIdentityConflictsPerSession > 0 && len(result.LocalIdentityConflicts) > params.MaxIdentityConflictsPerSession { + result.LocalIdentityConflicts = result.LocalIdentityConflicts[:params.MaxIdentityConflictsPerSession] + result.IdentityConflictsOverflowed = true + } + sort.Slice(result.PerPublisherPlayMethods, func(i, j int) bool { + if result.PerPublisherPlayMethods[i].Publisher.PublisherID == result.PerPublisherPlayMethods[j].Publisher.PublisherID { + return result.PerPublisherPlayMethods[i].Value < result.PerPublisherPlayMethods[j].Value + } + return refLess(result.PerPublisherPlayMethods[i].Publisher, result.PerPublisherPlayMethods[j].Publisher) + }) + return result +} + +func applyIdentity(result *GlobalSessionView, field string, values map[string][]PublisherRef) { + if len(values) == 0 { + return + } + if len(values) == 1 { + for value := range values { + switch field { + case "subject": + parts := strings.SplitN(value, "\x00", 2) + result.Subject = Subject{Kind: SubjectKind(parts[0]), ID: parts[1]} + case identityFieldProfileID: + result.ProfileID = value + case "media_file_id": + result.MediaFileID, _ = strconv.Atoi(value) + } + } + return + } + conflict := GlobalIdentityConflict{Field: field} + for value, publishers := range values { + sort.Slice(publishers, func(i, j int) bool { return refLess(publishers[i], publishers[j]) }) + conflict.Values = append(conflict.Values, AttributedValue{Value: strings.Replace(value, "\x00", ":", 1), Publishers: publishers}) + } + sort.Slice(conflict.Values, func(i, j int) bool { return conflict.Values[i].Value < conflict.Values[j].Value }) + result.IdentityConflicts = append(result.IdentityConflicts, conflict) + result.HasIdentityConflict = true +} + +func cappedStrings(values map[string]struct{}, maximum int, overflow bool) ([]string, bool) { + out := make([]string, 0, len(values)) + for value := range values { + out = append(out, value) + } + sort.Strings(out) + if maximum > 0 && len(out) > maximum { + out = out[:maximum] + overflow = true + } + return out, overflow +} +func cappedInts(values map[int]struct{}, maximum int, overflow bool) ([]int, bool) { + out := make([]int, 0, len(values)) + for value := range values { + out = append(out, value) + } + sort.Ints(out) + if maximum > 0 && len(out) > maximum { + out = out[:maximum] + overflow = true + } + return out, overflow +} +func cappedClients(values map[ClientVariant]struct{}, maximum int, overflow bool) ([]ClientVariant, bool) { + out := make([]ClientVariant, 0, len(values)) + for value := range values { + out = append(out, value) + } + sort.Slice(out, func(i, j int) bool { return clientVariantKey(out[i]) < clientVariantKey(out[j]) }) + if maximum > 0 && len(out) > maximum { + out = out[:maximum] + overflow = true + } + return out, overflow +} +func cappedTimes(values map[int64]struct{}, maximum int, overflow bool) ([]time.Time, bool) { + nanos := make([]int64, 0, len(values)) + for value := range values { + nanos = append(nanos, value) + } + sort.Slice(nanos, func(i, j int) bool { return nanos[i] < nanos[j] }) + if maximum > 0 && len(nanos) > maximum { + nanos = nanos[:maximum] + overflow = true + } + out := make([]time.Time, len(nanos)) + for i, value := range nanos { + out[i] = time.Unix(0, value) + } + return out, overflow +} + +func globalEpoch(contributions []publisherContribution) string { + hash := sha256.New() + for _, contribution := range contributions { + _, _ = fmt.Fprintf(hash, "%s\x00%d\x00%d\n", contribution.ref.PublisherID, contribution.snapshot.PublisherEpoch, contribution.snapshot.Sequence) + } + return hex.EncodeToString(hash.Sum(nil)[:16]) +} + +func saturatingAdd(a, b int64) int64 { + if b > 0 && a > math.MaxInt64-b { + return math.MaxInt64 + } + if b < 0 && a < math.MinInt64-b { + return math.MinInt64 + } + return a + b +} + +func saturatingInt(a, b int) int { + if b > 0 && a > math.MaxInt-b { + return math.MaxInt + } + return a + b +} +func saturatingInt64ToInt(value int64) int { + if value > int64(math.MaxInt) { + return math.MaxInt + } + return int(value) +} +func refLess(a, b PublisherRef) bool { + if a.PublisherID == b.PublisherID { + return a.NodeID < b.NodeID + } + return a.PublisherID < b.PublisherID +} +func routeViewKey(route RouteActivityView) string { + return route.Method + "\x00" + route.Pattern + "\x00" + string(route.Role) +} +func cloneTransfer(value TransferView) TransferView { + value.Outcomes = cloneOutcomes(value.Outcomes) + return value +} diff --git a/internal/streamtelemetry/global_test.go b/internal/streamtelemetry/global_test.go new file mode 100644 index 000000000..915d6486f --- /dev/null +++ b/internal/streamtelemetry/global_test.go @@ -0,0 +1,217 @@ +package streamtelemetry + +import ( + "encoding/json" + "math" + "reflect" + "slices" + "testing" + "time" +) + +func globalTestParams() ViewParams { + cfg := DefaultConfig("node") + return ViewParams{Freshness: cfg.Freshness, MembershipTTL: cfg.MembershipTTL, MaxMergedSessions: cfg.MaxMergedSessions, MaxMergedTransfers: cfg.MaxMergedTransfers, + MaxViewerIPsPerSession: cfg.MaxViewerIPsPerSession, MaxDeviceIDsPerSession: cfg.MaxDeviceIDsPerSession, + MaxClientVariantsPerSession: cfg.MaxClientVariantsPerSession, MaxUserAgentsPerSession: cfg.MaxClientVariantsPerSession, + MaxMediaFileIDsPerSession: cfg.MaxMediaFileIDsPerSession, MaxPlayMethodsPerSession: cfg.MaxPlayMethodsPerSession, + MaxTokenIssuedAtPerSession: cfg.MaxTokenIssuedAtPerSession, MaxRoutesPerSession: cfg.MaxRoutesPerSession, + MaxIdentityConflictsPerSession: cfg.MaxIdentityConflictsPerSession} +} + +func globalSet(at time.Time, snapshots ...Snapshot) PublisherSet { + set := PublisherSet{Snapshots: snapshots} + for _, snapshot := range snapshots { + set.Members = append(set.Members, Member{PublisherID: snapshot.PublisherID, LastHeartbeat: at}) + } + return set +} + +func viewerRoute(bytes int64) RouteActivityView { + return RouteActivityView{Method: "GET", Pattern: "/stream", Role: RoleViewerEgress, BytesAccepted: bytes} +} + +func TestBuildGlobalViewMergeRules(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + claim := at.Add(-time.Minute) + firstSeen := at.Add(-2 * time.Minute) + one := Snapshot{PublisherID: "p1", NodeID: "n1", PublisherEpoch: 1, Sequence: 1, CapturedAt: at, + Sessions: []SessionView{{SessionID: "session", Subject: UserSubject(1), ProfileID: "profile", MediaFileID: 10, PlayMethod: "direct", + StartedAt: firstSeen, StartedAtSource: StartedAtSourceFirstSeen, StartedAtDegraded: true, ViewerIPs: []string{"192.0.2.1"}, OpenObservations: 2, RequestCount: 3, + Routes: []RouteActivityView{viewerRoute(100), {Method: "GET", Pattern: "/relay", Role: RoleInternalRelay, BytesAccepted: 50}}, + MediaFileIDs: []int{10}, PlayMethods: []string{"direct"}}}} + two := Snapshot{PublisherID: "p2", NodeID: "n2", PublisherEpoch: 2, Sequence: 2, CapturedAt: at, + Sessions: []SessionView{{SessionID: "session", Subject: UserSubject(1), ProfileID: "profile", MediaFileID: 10, PlayMethod: "remux", + StartedAt: claim, StartedAtSource: StartedAtSourceClaim, ViewerIPs: []string{"192.0.2.2"}, OpenObservations: 4, RequestCount: 5, + Routes: []RouteActivityView{viewerRoute(200)}, MediaFileIDs: []int{10, 11}, PlayMethods: []string{"remux"}}}} + view := BuildGlobalView(globalSet(at, one, two), at, globalTestParams()) + if !view.Complete || len(view.Sessions) != 1 { + t.Fatalf("view = %+v", view) + } + session := view.Sessions[0] + if !reflect.DeepEqual(session.ViewerIPs, []string{"192.0.2.1", "192.0.2.2"}) { + t.Fatalf("viewer IPs = %v", session.ViewerIPs) + } + if session.OpenObservations != 6 || session.RequestCount != 8 { + t.Fatalf("counts = open %d requests %d", session.OpenObservations, session.RequestCount) + } + if session.ViewerBytesAccepted != 300 || session.RelayBytesAccepted != 50 { + t.Fatalf("bytes = viewer %d relay %d", session.ViewerBytesAccepted, session.RelayBytesAccepted) + } + if session.StartedAt != claim || session.StartedAtSource != StartedAtSourceClaim { + t.Fatalf("started = %v %s", session.StartedAt, session.StartedAtSource) + } + if !session.StartedAtDegraded { + t.Fatal("degraded first_seen contributor was not carried") + } + if !reflect.DeepEqual(session.PlayMethods, []string{"direct", "remux"}) { + t.Fatalf("play methods = %v", session.PlayMethods) + } +} + +func TestBuildGlobalViewRelayDoesNotSupplyIdentity(t *testing.T) { + at := time.Now() + snapshot := Snapshot{PublisherID: "relay", CapturedAt: at, Sessions: []SessionView{{SessionID: "s", Subject: UserSubject(9), ProfileID: "p", MediaFileID: 8, + Routes: []RouteActivityView{{Role: RoleInternalRelay, BytesAccepted: 20}}}}} + session := BuildGlobalView(globalSet(at, snapshot), at, globalTestParams()).Sessions[0] + if session.Subject != (Subject{}) || session.ProfileID != "" || session.MediaFileID != 0 || session.ViewerBytesAccepted != 0 || session.RelayBytesAccepted != 20 { + t.Fatalf("relay merge = %+v", session) + } +} + +func TestBuildGlobalViewMergesRoutesWithEmptyMethod(t *testing.T) { + at := time.Now() + one := Snapshot{PublisherID: "p1", CapturedAt: at, Sessions: []SessionView{{SessionID: "s", Routes: []RouteActivityView{{Pattern: "/stream", Role: RoleViewerEgress, Open: 1, Requests: 2, BytesAccepted: 3}}}}} + two := Snapshot{PublisherID: "p2", CapturedAt: at, Sessions: []SessionView{{SessionID: "s", Routes: []RouteActivityView{{Pattern: "/stream", Role: RoleViewerEgress, Open: 4, Requests: 5, BytesAccepted: 6}}}}} + session := BuildGlobalView(globalSet(at, one, two), at, globalTestParams()).Sessions[0] + if len(session.Routes) != 1 { + t.Fatalf("routes = %+v", session.Routes) + } + route := session.Routes[0] + if route.Open != 5 || route.Requests != 7 || route.BytesAccepted != 9 || session.ViewerBytesAccepted != 9 { + t.Fatalf("merged route = %+v, viewer bytes = %d", route, session.ViewerBytesAccepted) + } +} + +func TestBuildGlobalViewIdentityConflicts(t *testing.T) { + at := time.Now() + makeSnapshot := func(publisher string, user, media int, profile string) Snapshot { + return Snapshot{PublisherID: publisher, CapturedAt: at, Sessions: []SessionView{{SessionID: "s", Subject: UserSubject(user), ProfileID: profile, MediaFileID: media, + MediaFileIDs: []int{media}, Routes: []RouteActivityView{viewerRoute(1)}}}} + } + view := BuildGlobalView(globalSet(at, makeSnapshot("p1", 1, 10, "profile"), makeSnapshot("p2", 2, 11, "")), at, globalTestParams()) + session := view.Sessions[0] + if !session.HasIdentityConflict || session.Subject != (Subject{}) || session.MediaFileID != 0 || session.ProfileID != "profile" { + t.Fatalf("identity merge = %+v", session) + } + if len(session.IdentityConflicts) != 2 || session.IdentityConflicts[0].Field != "media_file_id" || session.IdentityConflicts[1].Field != "subject" { + t.Fatalf("conflicts = %+v", session.IdentityConflicts) + } + if !reflect.DeepEqual(session.MediaFileIDs, []int{10, 11}) { + t.Fatalf("media file union = %v", session.MediaFileIDs) + } + if len(session.IdentityConflicts[1].Values) != 2 || len(session.IdentityConflicts[1].Values[0].Publishers) != 1 { + t.Fatalf("attribution = %+v", session.IdentityConflicts) + } +} + +func TestBuildGlobalViewStartedAtDegradedRules(t *testing.T) { + at := time.Now() + snapshot := Snapshot{PublisherID: "p", CapturedAt: at, Sessions: []SessionView{{SessionID: "s", StartedAt: at.Add(-time.Minute), StartedAtSource: StartedAtSourceFirstSeen, Routes: []RouteActivityView{viewerRoute(0)}}}} + if !BuildGlobalView(globalSet(at, snapshot), at, globalTestParams()).Sessions[0].StartedAtDegraded { + t.Fatal("first_seen winner not degraded") + } +} + +func TestBuildGlobalViewCompleteness(t *testing.T) { + at := time.Now() + params := globalTestParams() + fresh := Snapshot{PublisherID: "p", NodeID: "n", CapturedAt: at} + tests := []struct { + name string + set PublisherSet + reason string + complete bool + }{ + {name: "fresh", set: globalSet(at, fresh), complete: true}, + {name: "stale", set: PublisherSet{Members: []Member{{PublisherID: "p", LastHeartbeat: at}}, Snapshots: []Snapshot{{PublisherID: "p", NodeID: "n", CapturedAt: at.Add(-params.Freshness - time.Second)}}}, reason: "missing_publisher"}, + {name: "never published", set: PublisherSet{Members: []Member{{PublisherID: "p", LastHeartbeat: at}}}, reason: "missing_publisher"}, + {name: "departed", set: PublisherSet{Members: []Member{{PublisherID: "p", LastHeartbeat: at.Add(-params.MembershipTTL - time.Second)}}}, complete: true}, + {name: "publisher truncated", set: globalSet(at, func() Snapshot { value := fresh; value.Truncated = true; return value }()), reason: "publisher_truncated"}, + {name: "reader truncated", set: PublisherSet{Members: globalSet(at, fresh).Members, Snapshots: []Snapshot{fresh}, Truncated: true}, reason: "truncated"}, + {name: "decode errors", set: PublisherSet{Members: globalSet(at, fresh).Members, Snapshots: []Snapshot{fresh}, Errors: []PublisherError{{PublisherID: "p", DecodeErrors: 1, Reason: "decode"}}}, reason: "decode_errors"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + view := BuildGlobalView(test.set, at, params) + if view.Complete != test.complete { + t.Fatalf("complete = %v, reasons %v", view.Complete, view.IncompleteReasons) + } + if test.reason != "" && !slices.Contains(view.IncompleteReasons, test.reason) { + t.Fatalf("reasons = %v", view.IncompleteReasons) + } + if (test.name == "stale" || test.name == "never published") && len(view.MissingPublishers) != 1 { + t.Fatalf("missing = %+v", view.MissingPublishers) + } + }) + } +} + +func TestBuildGlobalViewEpochAndClockSkew(t *testing.T) { + at := time.Now() + one := Snapshot{PublisherID: "a", PublisherEpoch: 1, Sequence: 1, CapturedAt: at.Add(time.Second)} + two := Snapshot{PublisherID: "b", PublisherEpoch: 2, Sequence: 2, CapturedAt: at} + view1 := BuildGlobalView(globalSet(at, one, two), at, globalTestParams()) + view2 := BuildGlobalView(globalSet(at, two, one), at, globalTestParams()) + if view1.Epoch != view2.Epoch || view1.ClockSkewSuspected { + t.Fatalf("epochs/skew = %q %q %v", view1.Epoch, view2.Epoch, view1.ClockSkewSuspected) + } + two.Sequence++ + if changed := BuildGlobalView(globalSet(at, one, two), at, globalTestParams()); changed.Epoch == view1.Epoch { + t.Fatal("epoch did not change with sequence") + } + one.CapturedAt = at.Add(-2 * globalTestParams().Freshness) + set := globalSet(at, one) + set.Members[0].LastHeartbeat = at.Add(2 * globalTestParams().Freshness) + if !BuildGlobalView(set, at, globalTestParams()).ClockSkewSuspected { + t.Fatal("far-future heartbeat did not flag skew") + } +} + +func TestBuildGlobalViewBoundsTransfersAndSaturation(t *testing.T) { + at := time.Now() + params := globalTestParams() + params.MaxViewerIPsPerSession = 1 + one := Snapshot{PublisherID: "a", CapturedAt: at, DroppedBytes: math.MaxInt64, Sessions: []SessionView{{SessionID: "s", ViewerIPs: []string{"b"}, RequestCount: math.MaxInt64, Routes: []RouteActivityView{{Role: RoleViewerEgress, BytesAccepted: math.MaxInt64}}}}, Transfers: []TransferView{{ID: "same"}}} + two := Snapshot{PublisherID: "b", CapturedAt: at, DroppedBytes: 1, Sessions: []SessionView{{SessionID: "s", ViewerIPs: []string{"a"}, RequestCount: 1, Routes: []RouteActivityView{{Role: RoleViewerEgress, BytesAccepted: 1}}}}, Transfers: []TransferView{{ID: "same"}}} + view := BuildGlobalView(globalSet(at, one, two), at, params) + if view.DroppedBytes != math.MaxInt64 || view.Sessions[0].RequestCount != math.MaxInt64 || view.Sessions[0].ViewerBytesAccepted != math.MaxInt64 { + t.Fatalf("sums wrapped: %+v", view) + } + if !view.Sessions[0].ViewerIPsOverflowed || !reflect.DeepEqual(view.Sessions[0].ViewerIPs, []string{"a"}) { + t.Fatalf("bounded viewer IPs = %v overflow=%v", view.Sessions[0].ViewerIPs, view.Sessions[0].ViewerIPsOverflowed) + } + if len(view.Transfers) != 2 || view.Transfers[0].Publisher.PublisherID == view.Transfers[1].Publisher.PublisherID { + t.Fatalf("transfers = %+v", view.Transfers) + } +} + +func TestBuildGlobalViewWholeViewPermutationInvariant(t *testing.T) { + at := time.Now() + one := Snapshot{PublisherID: "b", PublisherEpoch: 2, Sequence: 3, CapturedAt: at, Sessions: []SessionView{ + {SessionID: "z", ViewerIPs: []string{"2", "1"}, Routes: []RouteActivityView{{Method: "POST", Pattern: "/b", Role: RoleInternalRelay}, viewerRoute(2)}}, + {SessionID: "a", DeviceIDs: []string{"d2", "d1"}, Routes: []RouteActivityView{viewerRoute(1)}}, + }} + two := Snapshot{PublisherID: "a", PublisherEpoch: 1, Sequence: 4, CapturedAt: at, Sessions: []SessionView{{SessionID: "z", UserAgents: []string{"z", "a"}, Routes: []RouteActivityView{viewerRoute(3)}}}} + left := BuildGlobalView(globalSet(at, one, two), at, globalTestParams()) + slices.Reverse(one.Sessions) + slices.Reverse(one.Sessions[1].Routes) + slices.Reverse(one.Sessions[1].ViewerIPs) + slices.Reverse(two.Sessions[0].UserAgents) + right := BuildGlobalView(globalSet(at, two, one), at, globalTestParams()) + leftJSON, _ := json.Marshal(left) + rightJSON, _ := json.Marshal(right) + if string(leftJSON) != string(rightJSON) { + t.Fatalf("permutation changed view\nleft: %s\nright:%s", leftJSON, rightJSON) + } +} diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go index 8d3c756b3..d135713a4 100644 --- a/internal/streamtelemetry/registry.go +++ b/internal/streamtelemetry/registry.go @@ -43,19 +43,30 @@ type Registry struct { truncated atomic.Bool lastWarnUnixNano atomic.Int64 lastPublishWarnUnixNano atomic.Int64 + sequence atomic.Uint64 + startOnce sync.Once + stopOnce sync.Once + stop chan struct{} + done chan struct{} + started atomic.Bool + leaveMu sync.Mutex + left bool } func NewRegistry(cfg Config, store SnapshotStore, logger *slog.Logger) *Registry { if cfg.PublisherID == "" { cfg.PublisherID = uuid.NewString() } + if cfg.PublisherEpoch == 0 { + cfg.PublisherEpoch = now().UnixNano() + } if store == nil { store = NewLocalStore() } if logger == nil { logger = slog.Default() } - r := &Registry{cfg: cfg, store: store, logger: logger, seed: maphash.MakeSeed(), transfers: make(map[string]*transfer)} + r := &Registry{cfg: cfg, store: store, logger: logger, seed: maphash.MakeSeed(), transfers: make(map[string]*transfer), stop: make(chan struct{}), done: make(chan struct{})} for i := range r.shards { r.shards[i].sessions = make(map[string]*logicalSession) } @@ -298,21 +309,54 @@ func (r *Registry) Start(ctx context.Context) { if r == nil || !r.cfg.Enabled { return } - go func() { - ticker := time.NewTicker(r.cfg.SweepInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case sweepStart := <-ticker.C: - snapshot := r.sweep(sweepStart) - if err := r.store.Publish(ctx, snapshot); err != nil { - r.warnRateLimited("failed to publish stream telemetry snapshot", &r.lastPublishWarnUnixNano, "error", err) + r.startOnce.Do(func() { + r.started.Store(true) + go func() { + defer close(r.done) + ticker := time.NewTicker(r.cfg.SweepInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-r.stop: + return + case sweepStart := <-ticker.C: + snapshot := r.sweep(sweepStart) + snapshot.Sequence = r.sequence.Add(1) + if err := r.store.Publish(ctx, snapshot); err != nil { + r.warnRateLimited("failed to publish stream telemetry snapshot", &r.lastPublishWarnUnixNano, "error", err) + } } } - } - }() + }() + }) +} + +func (r *Registry) Stop(ctx context.Context) error { + if r == nil || !r.cfg.Enabled || !r.started.Load() { + return nil + } + r.stopOnce.Do(func() { close(r.stop) }) + select { + case <-r.done: + case <-ctx.Done(): + return ctx.Err() + } + global, ok := r.store.(GlobalSnapshotStore) + if !ok { + return nil + } + r.leaveMu.Lock() + defer r.leaveMu.Unlock() + if r.left { + return nil + } + if err := global.Leave(ctx); err != nil { + return err + } + r.left = true + return nil } func (r *Registry) Sweep() Snapshot { return r.sweep(now()) } @@ -381,7 +425,7 @@ func (r *Registry) sweep(sweepStart time.Time) Snapshot { func (r *Registry) Snapshot() Snapshot { return r.SnapshotAt(now()) } func (r *Registry) SnapshotAt(capturedAt time.Time) Snapshot { - view := Snapshot{PublisherID: r.cfg.PublisherID, NodeID: r.cfg.NodeID, CapturedAt: capturedAt, + view := Snapshot{PublisherID: r.cfg.PublisherID, NodeID: r.cfg.NodeID, PublisherEpoch: r.cfg.PublisherEpoch, Sequence: r.sequence.Load(), CapturedAt: capturedAt, Truncated: r.truncated.Load(), DroppedObservations: r.droppedObservations.Load(), DroppedBytes: r.droppedBytes.Load(), UnattributedObservations: r.unattributedObservations.Load(), UnattributedBytes: r.unattributedBytes.Load()} diff --git a/internal/streamtelemetry/registry_test.go b/internal/streamtelemetry/registry_test.go index 6403c8f96..fddbcc6a0 100644 --- a/internal/streamtelemetry/registry_test.go +++ b/internal/streamtelemetry/registry_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/http" "net/http/httptest" + "reflect" "sync" "sync/atomic" "testing" @@ -18,6 +19,7 @@ func testConfig() Config { cfg := DefaultConfig("test-node") cfg.Enabled = true cfg.PublisherID = "test-publisher" + cfg.PublisherEpoch = 1 cfg.Retention = time.Millisecond return cfg } @@ -253,11 +255,161 @@ func TestStartContinuesAfterPublishError(t *testing.T) { registry.Start(ctx) time.Sleep(8 * time.Millisecond) cancel() + // Wait for the collector to actually exit. Returning while it still runs + // leaks a goroutine that keeps reading the package-level now() seam, which + // races with any later test that replaces it. + stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second) + defer stopCancel() + if err := registry.Stop(stopCtx); err != nil { + t.Fatal(err) + } if store.published.Load() < 2 { t.Fatalf("collector stopped after publish error: %d publishes", store.published.Load()) } } +type lifecycleStore struct { + mu sync.Mutex + published []Snapshot + leaveCalls int + failFirstLeave bool +} + +func (s *lifecycleStore) Publish(_ context.Context, snapshot Snapshot) error { + s.mu.Lock() + s.published = append(s.published, snapshot) + s.mu.Unlock() + return nil +} +func (s *lifecycleStore) Load(context.Context) (Snapshot, error) { return Snapshot{}, nil } +func (s *lifecycleStore) LoadAll(context.Context) (PublisherSet, error) { return PublisherSet{}, nil } +func (s *lifecycleStore) Leave(ctx context.Context) error { + s.mu.Lock() + s.leaveCalls++ + call := s.leaveCalls + s.mu.Unlock() + if s.failFirstLeave && call == 1 { + <-ctx.Done() + return ctx.Err() + } + return nil +} + +func TestRegistryStartOnceAndPublishedSequence(t *testing.T) { + cfg := testConfig() + cfg.SweepInterval = time.Millisecond + store := &lifecycleStore{} + registry := NewRegistry(cfg, store, slog.New(slog.DiscardHandler)) + ctx, cancel := context.WithCancel(context.Background()) + registry.Start(ctx) + registry.Start(ctx) + time.Sleep(6 * time.Millisecond) + cancel() + stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second) + defer stopCancel() + if err := registry.Stop(stopCtx); err != nil { + t.Fatal(err) + } + store.mu.Lock() + defer store.mu.Unlock() + if len(store.published) < 2 { + t.Fatalf("publishes = %d", len(store.published)) + } + for index, snapshot := range store.published { + if snapshot.Sequence != uint64(index+1) { + t.Fatalf("sequence[%d] = %d", index, snapshot.Sequence) + } + if snapshot.PublisherEpoch != cfg.PublisherEpoch { + t.Fatalf("epoch = %d", snapshot.PublisherEpoch) + } + } +} + +func TestRegistryConcurrentStopLeavesOnce(t *testing.T) { + store := &lifecycleStore{} + registry := NewRegistry(testConfig(), store, nil) + registry.Start(context.Background()) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + if err := registry.Stop(context.Background()); err != nil { + t.Errorf("Stop: %v", err) + } + }() + } + wg.Wait() + store.mu.Lock() + defer store.mu.Unlock() + if store.leaveCalls != 1 { + t.Fatalf("leave calls = %d", store.leaveCalls) + } +} + +func TestRegistryStopTimeoutCanRetryLeave(t *testing.T) { + store := &lifecycleStore{failFirstLeave: true} + registry := NewRegistry(testConfig(), store, nil) + registry.Start(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if err := registry.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("first Stop = %v", err) + } + if err := registry.Stop(context.Background()); err != nil { + t.Fatalf("retry Stop = %v", err) + } + store.mu.Lock() + defer store.mu.Unlock() + if store.leaveCalls != 2 { + t.Fatalf("leave calls = %d", store.leaveCalls) + } +} + +func TestRegistryStopNilDisabledAndNeverStarted(t *testing.T) { + var nilRegistry *Registry + if err := nilRegistry.Stop(context.Background()); err != nil { + t.Fatal(err) + } + disabled := testConfig() + disabled.Enabled = false + if err := NewRegistry(disabled, NewLocalStore(), nil).Stop(context.Background()); err != nil { + t.Fatal(err) + } + if err := NewRegistry(testConfig(), NewLocalStore(), nil).Stop(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRegistryGlobalView(t *testing.T) { + at := time.Now() + store := NewLocalStore() + registry := NewRegistry(testConfig(), store, nil) + if err := store.Publish(context.Background(), Snapshot{PublisherID: "test-publisher", PublisherEpoch: 1, Sequence: 1, CapturedAt: at}); err != nil { + t.Fatal(err) + } + originalNow := now + now = func() time.Time { return at } + defer func() { now = originalNow }() + view, err := registry.GlobalView(context.Background()) + if err != nil || !view.Complete || len(view.Publishers) != 1 { + t.Fatalf("view = %+v, err=%v", view, err) + } + unsupported := NewRegistry(testConfig(), &failingStore{}, nil) + if _, err := unsupported.GlobalView(context.Background()); err == nil { + t.Fatal("non-global store accepted") + } else { + var typed errGlobalSnapshotStoreUnsupported + if !errors.As(err, &typed) { + t.Fatalf("error = %T %v", err, err) + } + } + var nilRegistry *Registry + if zero, err := nilRegistry.GlobalView(context.Background()); err != nil || !reflect.DeepEqual(zero, GlobalMonitoringView{}) { + t.Fatalf("nil view = %+v, %v", zero, err) + } +} + func TestLocalStoreDeepCopies(t *testing.T) { store := NewLocalStore() source := Snapshot{Sessions: []SessionView{{ViewerIPs: []string{"one"}, Routes: []RouteActivityView{{Pattern: "/one"}}, Outcomes: map[httpstream.StreamOutcome]int64{"completed": 1}}}} diff --git a/internal/streamtelemetry/session.go b/internal/streamtelemetry/session.go index 88cf8a295..6327e8ae2 100644 --- a/internal/streamtelemetry/session.go +++ b/internal/streamtelemetry/session.go @@ -151,7 +151,7 @@ func (s *logicalSession) recordConflicts(a Attachment, observedAt time.Time, max checks := []struct{ field, existing, offered string }{ {"subject.kind", string(s.subject.Kind), string(a.Subject.Kind)}, {"subject.id", s.subject.ID, a.Subject.ID}, - {"profile_id", s.profileID, a.ProfileID}, + {identityFieldProfileID, s.profileID, a.ProfileID}, } for _, check := range checks { if check.existing == "" || check.offered == "" || check.existing == check.offered { diff --git a/internal/streamtelemetry/store.go b/internal/streamtelemetry/store.go index 4d7d3a187..dbed960e1 100644 --- a/internal/streamtelemetry/store.go +++ b/internal/streamtelemetry/store.go @@ -3,6 +3,17 @@ package streamtelemetry import ( "context" "sync" + "time" +) + +const ( + publisherMetaField = "meta" + publisherReasonDecode = "decode" + publisherReasonOversized = "oversized" + publisherReasonMetaMissing = "meta_missing" + publisherReasonIdentityMismatch = "identity_mismatch" + publisherReasonCountMismatch = "count_mismatch" + identityFieldProfileID = "profile_id" ) type SnapshotStore interface { @@ -10,9 +21,34 @@ type SnapshotStore interface { Load(context.Context) (Snapshot, error) } +type Member struct { + PublisherID string + LastHeartbeat time.Time +} + +type PublisherError struct { + PublisherID string + DecodeErrors int + Reason string +} + +type PublisherSet struct { + Members []Member + Snapshots []Snapshot + Errors []PublisherError + Truncated bool +} + +type GlobalSnapshotStore interface { + SnapshotStore + LoadAll(context.Context) (PublisherSet, error) + Leave(context.Context) error +} + type LocalStore struct { mu sync.RWMutex snapshot Snapshot + departed bool } func NewLocalStore() *LocalStore { return &LocalStore{} } @@ -20,6 +56,25 @@ func NewLocalStore() *LocalStore { return &LocalStore{} } func (s *LocalStore) Publish(_ context.Context, snapshot Snapshot) error { s.mu.Lock() s.snapshot = cloneSnapshot(snapshot) + s.departed = false + s.mu.Unlock() + return nil +} + +func (s *LocalStore) LoadAll(_ context.Context) (PublisherSet, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if s.departed || s.snapshot.PublisherID == "" { + return PublisherSet{}, nil + } + snapshot := cloneSnapshot(s.snapshot) + return PublisherSet{Members: []Member{{PublisherID: snapshot.PublisherID, LastHeartbeat: snapshot.CapturedAt}}, Snapshots: []Snapshot{snapshot}}, nil +} + +func (s *LocalStore) Leave(_ context.Context) error { + s.mu.Lock() + s.snapshot = Snapshot{} + s.departed = true s.mu.Unlock() return nil } diff --git a/internal/streamtelemetry/store_redis.go b/internal/streamtelemetry/store_redis.go new file mode 100644 index 000000000..8d73892b0 --- /dev/null +++ b/internal/streamtelemetry/store_redis.go @@ -0,0 +1,392 @@ +package streamtelemetry + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "log/slog" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" +) + +const redisFieldChunk = 512 + +type RedisStore struct { + client *redis.Client + cfg Config + logger *slog.Logger + + mu sync.Mutex + publisherID string + published map[string][16]byte + needFullResync bool + publishCount uint64 +} + +func NewRedisStore(client *redis.Client, cfg Config, logger *slog.Logger) *RedisStore { + if logger == nil { + logger = slog.Default() + } + return &RedisStore{client: client, cfg: cfg, logger: logger, publisherID: cfg.PublisherID, published: make(map[string][16]byte), needFullResync: true} +} + +func (s *RedisStore) snapshotKey(publisherID string) string { + return s.cfg.KeyPrefix + ":snap:" + publisherID +} + +func (s *RedisStore) rosterKey() string { return s.cfg.KeyPrefix + ":roster" } + +func digest128(value []byte) [16]byte { + full := sha256.Sum256(value) + var digest [16]byte + copy(digest[:], full[:16]) + return digest +} + +func snapshotHashFields(snapshot Snapshot) (map[string][]byte, error) { + fields := make(map[string][]byte, len(snapshot.Sessions)+len(snapshot.Transfers)+1) + meta, err := encodeMeta(publisherMeta{ + PublisherID: snapshot.PublisherID, NodeID: snapshot.NodeID, Epoch: snapshot.PublisherEpoch, + Sequence: snapshot.Sequence, CapturedAtUnixNano: timeToUnixNano(snapshot.CapturedAt), Truncated: snapshot.Truncated, + DroppedObservations: snapshot.DroppedObservations, DroppedBytes: snapshot.DroppedBytes, + UnattributedObservations: snapshot.UnattributedObservations, UnattributedBytes: snapshot.UnattributedBytes, + SessionCount: len(snapshot.Sessions), TransferCount: len(snapshot.Transfers), + }) + if err != nil { + return nil, err + } + fields[publisherMetaField] = meta + for _, session := range snapshot.Sessions { + encoded, encodeErr := encodeSession(session) + if encodeErr != nil { + return nil, fmt.Errorf("encode session %q: %w", session.SessionID, encodeErr) + } + fields["s:"+session.SessionID] = encoded + } + for _, transfer := range snapshot.Transfers { + encoded, encodeErr := encodeTransfer(transfer) + if encodeErr != nil { + return nil, fmt.Errorf("encode transfer %q: %w", transfer.ID, encodeErr) + } + fields["t:"+transfer.ID] = encoded + } + return fields, nil +} + +func (s *RedisStore) plan(snapshot Snapshot) (sets map[string][]byte, dels []string, fields map[string][]byte, full bool, err error) { + fields, err = snapshotHashFields(snapshot) + if err != nil { + return nil, nil, nil, false, err + } + full = s.needFullResync || s.publishCount == 0 || (s.cfg.FullResyncEvery > 0 && s.publishCount%uint64(s.cfg.FullResyncEvery) == 0) + sets = make(map[string][]byte, len(fields)) + if full { + for field, value := range fields { + sets[field] = value + } + return sets, nil, fields, true, nil + } + for field, value := range fields { + if old, ok := s.published[field]; !ok || old != digest128(value) { + sets[field] = value + } + } + sets[publisherMetaField] = fields[publisherMetaField] + for field := range s.published { + if _, ok := fields[field]; !ok { + dels = append(dels, field) + } + } + sort.Strings(dels) + return sets, dels, fields, false, nil +} + +func (s *RedisStore) Publish(ctx context.Context, snapshot Snapshot) error { + if s == nil || s.client == nil { + return fmt.Errorf("stream telemetry redis client is nil") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.publisherID == "" { + s.publisherID = snapshot.PublisherID + } else if s.publisherID != snapshot.PublisherID { + return fmt.Errorf("stream telemetry publisher id changed from %q to %q", s.publisherID, snapshot.PublisherID) + } + sets, dels, fields, full, err := s.plan(snapshot) + if err != nil { + return err + } + key := s.snapshotKey(snapshot.PublisherID) + _, err = s.client.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + if full { + pipe.Del(ctx, key) + } + for start := 0; start < len(dels); start += redisFieldChunk { + end := min(start+redisFieldChunk, len(dels)) + pipe.HDel(ctx, key, dels[start:end]...) + } + setNames := make([]string, 0, len(sets)) + for field := range sets { + if field != publisherMetaField { + setNames = append(setNames, field) + } + } + sort.Strings(setNames) + for start := 0; start < len(setNames); start += redisFieldChunk { + end := min(start+redisFieldChunk, len(setNames)) + values := make([]any, 0, (end-start)*2) + for _, field := range setNames[start:end] { + values = append(values, field, sets[field]) + } + pipe.HSet(ctx, key, values...) + } + pipe.HSet(ctx, key, publisherMetaField, sets[publisherMetaField]) + pipe.PExpire(ctx, key, s.cfg.MembershipTTL) + pipe.ZAdd(ctx, s.rosterKey(), redis.Z{Score: float64(snapshot.CapturedAt.UnixNano()), Member: snapshot.PublisherID}) + cutoff := snapshot.CapturedAt.Add(-2 * s.cfg.MembershipTTL).UnixNano() + pipe.ZRemRangeByScore(ctx, s.rosterKey(), "-inf", "("+strconv.FormatInt(cutoff, 10)) + pipe.PExpire(ctx, s.rosterKey(), 10*s.cfg.MembershipTTL) + return nil + }) + if err != nil { + s.needFullResync = true + clear(s.published) + return err + } + clear(s.published) + for field, value := range fields { + s.published[field] = digest128(value) + } + s.needFullResync = false + s.publishCount++ + return nil +} + +func (s *RedisStore) Load(ctx context.Context) (Snapshot, error) { + if s == nil || s.client == nil { + return Snapshot{}, fmt.Errorf("stream telemetry redis client is nil") + } + s.mu.Lock() + publisherID := s.publisherID + s.mu.Unlock() + if publisherID == "" { + return Snapshot{}, nil + } + fields, err := s.client.HGetAll(ctx, s.snapshotKey(publisherID)).Result() + if err != nil { + return Snapshot{}, err + } + snapshot, _, err := decodeSnapshotHash(publisherID, fields, s.cfg.MaxMergedSessions, s.cfg.MaxMergedTransfers) + return snapshot, err +} + +func (s *RedisStore) LoadAll(ctx context.Context) (PublisherSet, error) { + if s == nil || s.client == nil { + return PublisherSet{}, fmt.Errorf("stream telemetry redis client is nil") + } + set := PublisherSet{} + minimum := "(" + strconv.FormatInt(now().Add(-s.cfg.MembershipTTL).UnixNano(), 10) + readLimit := int64(s.cfg.MaxPublishers) + if readLimit < int64(^uint64(0)>>1) { + readLimit++ + } + members, err := s.client.ZRangeByScoreWithScores(ctx, s.rosterKey(), &redis.ZRangeBy{Min: minimum, Max: "+inf", Offset: 0, Count: readLimit}).Result() + if err != nil { + return set, err + } + if len(members) > s.cfg.MaxPublishers { + set.Truncated = true + set.Errors = append(set.Errors, PublisherError{Reason: "publisher_cap"}) + members = members[:s.cfg.MaxPublishers] + } + for _, member := range members { + publisherID, ok := member.Member.(string) + if !ok { + continue + } + set.Members = append(set.Members, Member{PublisherID: publisherID, LastHeartbeat: time.Unix(0, int64(member.Score))}) + } + sort.Slice(set.Members, func(i, j int) bool { return set.Members[i].PublisherID < set.Members[j].PublisherID }) + + pipe := s.client.Pipeline() + hlens := make(map[string]*redis.IntCmd, len(set.Members)) + for _, member := range set.Members { + hlens[member.PublisherID] = pipe.HLen(ctx, s.snapshotKey(member.PublisherID)) + } + if _, err = pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + return set, err + } + maxFields := s.maxFieldsPerPublisher() + readPipe := s.client.Pipeline() + reads := make(map[string]*redis.MapStringStringCmd, len(set.Members)) + for _, member := range set.Members { + length, lengthErr := hlens[member.PublisherID].Result() + if lengthErr != nil && !errors.Is(lengthErr, redis.Nil) { + return set, lengthErr + } + if length > maxFields { + set.Errors = append(set.Errors, PublisherError{PublisherID: member.PublisherID, Reason: publisherReasonOversized}) + continue + } + reads[member.PublisherID] = readPipe.HGetAll(ctx, s.snapshotKey(member.PublisherID)) + } + if len(reads) > 0 { + if _, err = readPipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + return set, err + } + } + remainingSessions, remainingTransfers := s.cfg.MaxMergedSessions, s.cfg.MaxMergedTransfers + for _, member := range set.Members { + read := reads[member.PublisherID] + if read == nil { + continue + } + fields, readErr := read.Result() + if readErr != nil && !errors.Is(readErr, redis.Nil) { + return set, readErr + } + snapshot, publisherErr, decodeErr := decodeSnapshotHash(member.PublisherID, fields, remainingSessions, remainingTransfers) + if decodeErr != nil { + set.Errors = append(set.Errors, PublisherError{PublisherID: member.PublisherID, DecodeErrors: 1, Reason: publisherReasonDecode}) + continue + } + if publisherErr.Reason != "" || publisherErr.DecodeErrors > 0 { + set.Errors = append(set.Errors, publisherErr) + } + if snapshot.PublisherID == "" { + continue + } + if len(snapshot.Sessions) == remainingSessions && countFields(fields, "s:") > remainingSessions { + set.Truncated = true + } + if len(snapshot.Transfers) == remainingTransfers && countFields(fields, "t:") > remainingTransfers { + set.Truncated = true + } + remainingSessions -= len(snapshot.Sessions) + remainingTransfers -= len(snapshot.Transfers) + set.Snapshots = append(set.Snapshots, snapshot) + } + sort.Slice(set.Snapshots, func(i, j int) bool { return set.Snapshots[i].PublisherID < set.Snapshots[j].PublisherID }) + sort.Slice(set.Errors, func(i, j int) bool { + if set.Errors[i].PublisherID == set.Errors[j].PublisherID { + return set.Errors[i].Reason < set.Errors[j].Reason + } + return set.Errors[i].PublisherID < set.Errors[j].PublisherID + }) + return set, nil +} + +func (s *RedisStore) maxFieldsPerPublisher() int64 { + maximum := s.cfg.MaxSessions + s.cfg.MaxTransfers + 16 + if maximum < 16 { + return int64(^uint64(0) >> 1) + } + return maximum +} + +func countFields(fields map[string]string, prefix string) int { + count := 0 + for field := range fields { + if strings.HasPrefix(field, prefix) { + count++ + } + } + return count +} + +func decodeSnapshotHash(publisherID string, fields map[string]string, maxSessions, maxTransfers int) (Snapshot, PublisherError, error) { + problem := PublisherError{PublisherID: publisherID} + metaBytes, ok := fields[publisherMetaField] + if !ok { + problem.Reason = publisherReasonMetaMissing + return Snapshot{}, problem, nil + } + meta, err := decodeMeta([]byte(metaBytes)) + if err != nil { + problem.DecodeErrors = 1 + problem.Reason = publisherReasonDecode + return Snapshot{}, problem, err + } + if meta.PublisherID != publisherID { + problem.Reason = publisherReasonIdentityMismatch + return Snapshot{}, problem, nil + } + snapshot := Snapshot{PublisherID: meta.PublisherID, NodeID: meta.NodeID, PublisherEpoch: meta.Epoch, Sequence: meta.Sequence, + CapturedAt: timeFromUnixNano(meta.CapturedAtUnixNano), Truncated: meta.Truncated, DroppedObservations: meta.DroppedObservations, + DroppedBytes: meta.DroppedBytes, UnattributedObservations: meta.UnattributedObservations, UnattributedBytes: meta.UnattributedBytes} + names := make([]string, 0, len(fields)) + for field := range fields { + if field != publisherMetaField { + names = append(names, field) + } + } + sort.Strings(names) + decodedSessions, decodedTransfers := 0, 0 + for _, field := range names { + switch { + case strings.HasPrefix(field, "s:"): + decodedSessions++ + if len(snapshot.Sessions) >= maxSessions { + continue + } + value, decodeErr := decodeSession([]byte(fields[field])) + if decodeErr != nil { + problem.DecodeErrors++ + continue + } + snapshot.Sessions = append(snapshot.Sessions, value) + case strings.HasPrefix(field, "t:"): + decodedTransfers++ + if len(snapshot.Transfers) >= maxTransfers { + continue + } + value, decodeErr := decodeTransfer([]byte(fields[field])) + if decodeErr != nil { + problem.DecodeErrors++ + continue + } + snapshot.Transfers = append(snapshot.Transfers, value) + default: + problem.DecodeErrors++ + } + } + if decodedSessions != meta.SessionCount || decodedTransfers != meta.TransferCount { + problem.Reason = publisherReasonCountMismatch + } else if problem.DecodeErrors > 0 { + problem.Reason = publisherReasonDecode + } + sort.Slice(snapshot.Sessions, func(i, j int) bool { return snapshot.Sessions[i].SessionID < snapshot.Sessions[j].SessionID }) + sort.Slice(snapshot.Transfers, func(i, j int) bool { return snapshot.Transfers[i].ID < snapshot.Transfers[j].ID }) + return snapshot, problem, nil +} + +func (s *RedisStore) Leave(ctx context.Context) error { + if s == nil || s.client == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if s.publisherID == "" { + return nil + } + _, err := s.client.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + pipe.ZRem(ctx, s.rosterKey(), s.publisherID) + pipe.Del(ctx, s.snapshotKey(s.publisherID)) + return nil + }) + if errors.Is(err, redis.Nil) { + err = nil + } + if err != nil { + return err + } + clear(s.published) + s.needFullResync = true + return nil +} diff --git a/internal/streamtelemetry/store_redis_test.go b/internal/streamtelemetry/store_redis_test.go new file mode 100644 index 000000000..53aa42ac3 --- /dev/null +++ b/internal/streamtelemetry/store_redis_test.go @@ -0,0 +1,339 @@ +package streamtelemetry + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "reflect" + "sort" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" + "github.com/google/uuid" + "github.com/redis/go-redis/v9" +) + +func testRedisStoreConfig(publisherID string) Config { + cfg := DefaultConfig("node-" + publisherID) + cfg.Enabled = true + cfg.Distributed = true + cfg.PublisherID = publisherID + cfg.PublisherEpoch = 10 + cfg.KeyPrefix = "test:stelem:" + uuid.NewString() + cfg.MembershipTTL = time.Minute + return cfg +} + +func markPlanPublished(t *testing.T, store *RedisStore, snapshot Snapshot) { + t.Helper() + fields, err := snapshotHashFields(snapshot) + if err != nil { + t.Fatal(err) + } + clear(store.published) + for field, value := range fields { + store.published[field] = digest128(value) + } + store.needFullResync = false + store.publishCount++ +} + +func TestRedisStorePlan(t *testing.T) { + cfg := testRedisStoreConfig("publisher") + cfg.FullResyncEvery = 3 + store := NewRedisStore(nil, cfg, slog.New(slog.DiscardHandler)) + base := Snapshot{PublisherID: cfg.PublisherID, PublisherEpoch: 1, Sequence: 1, CapturedAt: time.Unix(10, 0), Sessions: []SessionView{ + {SessionID: "a", RequestCount: 1, TokenIssuedAtSources: map[TokenIssuedAtSource]int64{}, Outcomes: map[httpstream.StreamOutcome]int64{}}, + {SessionID: "b", TokenIssuedAtSources: map[TokenIssuedAtSource]int64{}, Outcomes: map[httpstream.StreamOutcome]int64{}}, + }} + sets, dels, _, full, err := store.plan(base) + if err != nil { + t.Fatal(err) + } + if !full || len(dels) != 0 || len(sets) != 3 { + t.Fatalf("first plan: full=%v sets=%v dels=%v", full, keysOf(sets), dels) + } + markPlanPublished(t, store, base) + + changed := cloneSnapshot(base) + changed.Sequence++ + changed.Sessions[0].RequestCount++ + sets, dels, _, full, err = store.plan(changed) + if err != nil { + t.Fatal(err) + } + if full || len(dels) != 0 || !reflect.DeepEqual(keysOf(sets), []string{"meta", "s:a"}) { + t.Fatalf("changed plan: full=%v sets=%v dels=%v", full, keysOf(sets), dels) + } + markPlanPublished(t, store, changed) + + removed := cloneSnapshot(changed) + removed.Sequence++ + removed.Sessions = removed.Sessions[:1] + sets, dels, _, full, err = store.plan(removed) + if err != nil { + t.Fatal(err) + } + if full || !reflect.DeepEqual(dels, []string{"s:b"}) || !reflect.DeepEqual(keysOf(sets), []string{"meta"}) { + t.Fatalf("removed plan: full=%v sets=%v dels=%v", full, keysOf(sets), dels) + } + + store.needFullResync = true + clear(store.published) + if _, _, _, full, err = store.plan(removed); err != nil || !full { + t.Fatalf("failed publish recovery: full=%v err=%v", full, err) + } + markPlanPublished(t, store, removed) + store.publishCount = uint64(cfg.FullResyncEvery) + if _, _, _, full, err = store.plan(removed); err != nil || !full { + t.Fatalf("periodic resync: full=%v err=%v", full, err) + } +} + +func TestRedisStoreFailedPublishForcesFullResync(t *testing.T) { + cfg := testRedisStoreConfig("publisher") + client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"}) + t.Cleanup(func() { _ = client.Close() }) + store := NewRedisStore(client, cfg, slog.New(slog.DiscardHandler)) + snapshot := Snapshot{PublisherID: cfg.PublisherID, Sequence: 1, CapturedAt: time.Now()} + markPlanPublished(t, store, snapshot) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := store.Publish(ctx, snapshot); err == nil { + t.Fatal("publish unexpectedly succeeded") + } + if !store.needFullResync || len(store.published) != 0 { + t.Fatalf("failed state: full=%v published=%d", store.needFullResync, len(store.published)) + } + if _, _, _, full, err := store.plan(snapshot); err != nil || !full { + t.Fatalf("next plan: full=%v err=%v", full, err) + } +} + +func TestRedisStoreKeyBuilders(t *testing.T) { + cfg := testRedisStoreConfig("publisher") + store := NewRedisStore(nil, cfg, nil) + if got := store.snapshotKey("other"); got != cfg.KeyPrefix+":snap:other" { + t.Fatalf("snapshot key = %q", got) + } + if got := store.rosterKey(); got != cfg.KeyPrefix+":roster" { + t.Fatalf("roster key = %q", got) + } +} + +func TestSnapshotHashFieldsRoundTripAndDeterministicCap(t *testing.T) { + snapshot := Snapshot{PublisherID: "publisher", NodeID: "node", PublisherEpoch: 1, Sequence: 2, CapturedAt: time.Unix(3, 4), + Sessions: []SessionView{ + {SessionID: "z", TokenIssuedAtSources: map[TokenIssuedAtSource]int64{}, Outcomes: map[httpstream.StreamOutcome]int64{}}, + {SessionID: "a", TokenIssuedAtSources: map[TokenIssuedAtSource]int64{}, Outcomes: map[httpstream.StreamOutcome]int64{}}, + }, Transfers: []TransferView{{ID: "z", Outcomes: map[httpstream.StreamOutcome]int64{}}, {ID: "a", Outcomes: map[httpstream.StreamOutcome]int64{}}}} + encoded, err := snapshotHashFields(snapshot) + if err != nil { + t.Fatal(err) + } + fields := make(map[string]string, len(encoded)) + for field, value := range encoded { + fields[field] = string(value) + } + got, problem, err := decodeSnapshotHash("publisher", fields, 10, 10) + if err != nil { + t.Fatal(err) + } + if problem.Reason != "" { + t.Fatalf("problem = %+v", problem) + } + want := cloneSnapshot(snapshot) + sort.Slice(want.Sessions, func(i, j int) bool { return want.Sessions[i].SessionID < want.Sessions[j].SessionID }) + sort.Slice(want.Transfers, func(i, j int) bool { return want.Transfers[i].ID < want.Transfers[j].ID }) + if !reflect.DeepEqual(got, want) { + t.Fatalf("hash round trip mismatch\n got: %#v\nwant: %#v", got, want) + } + capped, _, err := decodeSnapshotHash("publisher", fields, 1, 1) + if err != nil { + t.Fatal(err) + } + if len(capped.Sessions) != 1 || capped.Sessions[0].SessionID != "a" || len(capped.Transfers) != 1 || capped.Transfers[0].ID != "a" { + t.Fatalf("deterministic cap = %+v", capped) + } +} + +func keysOf(values map[string][]byte) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func redisTestClient(t *testing.T) *redis.Client { + t.Helper() + url := os.Getenv("SILO_TEST_REDIS_URL") + if url == "" { + url = "redis://127.0.0.1:6380/15" + } + options, err := redis.ParseURL(url) + if err != nil { + t.Fatalf("parse SILO_TEST_REDIS_URL: %v", err) + } + client := redis.NewClient(options) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + t.Skipf("Redis integration test skipped: no server answers %s: %v", url, err) + } + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func TestRedisStoreIntegration(t *testing.T) { + client := redisTestClient(t) + ctx := context.Background() + newStore := func(prefix, publisher string) *RedisStore { + cfg := testRedisStoreConfig(publisher) + cfg.KeyPrefix = prefix + return NewRedisStore(client, cfg, slog.New(slog.DiscardHandler)) + } + t.Run("publish delta load and leave", func(t *testing.T) { + prefix := "test:stelem:" + uuid.NewString() + store := newStore(prefix, "p1") + t.Cleanup(func() { _ = client.Del(ctx, store.rosterKey(), store.snapshotKey("p1")).Err() }) + first := Snapshot{PublisherID: "p1", NodeID: "n1", PublisherEpoch: 1, Sequence: 1, CapturedAt: time.Now(), Sessions: []SessionView{{SessionID: "a"}, {SessionID: "removed"}}} + if err := store.Publish(ctx, first); err != nil { + t.Fatal(err) + } + set, err := store.LoadAll(ctx) + if err != nil || len(set.Snapshots) != 1 || len(set.Snapshots[0].Sessions) != 2 { + t.Fatalf("first load = %+v, %v", set, err) + } + second := cloneSnapshot(first) + second.Sequence++ + second.CapturedAt = time.Now() + second.Sessions = []SessionView{{SessionID: "a", RequestCount: 9}} + if err := store.Publish(ctx, second); err != nil { + t.Fatal(err) + } + set, err = store.LoadAll(ctx) + if err != nil || len(set.Snapshots) != 1 || len(set.Snapshots[0].Sessions) != 1 || set.Snapshots[0].Sessions[0].RequestCount != 9 { + t.Fatalf("delta load = %+v, %v", set, err) + } + if err := store.Leave(ctx); err != nil { + t.Fatal(err) + } + if _, _, _, full, planErr := store.plan(second); planErr != nil || !full { + t.Fatalf("plan after leave: full=%v err=%v", full, planErr) + } + if err := store.Leave(ctx); err != nil { + t.Fatal(err) + } + set, err = store.LoadAll(ctx) + if err != nil || len(set.Members) != 0 { + t.Fatalf("after leave = %+v, %v", set, err) + } + }) + + t.Run("publisher cap counts only live roster", func(t *testing.T) { + prefix := "test:stelem:" + uuid.NewString() + store := newStore(prefix, "reader") + store.cfg.MaxPublishers = 2 + t.Cleanup(func() { _ = client.Del(ctx, store.rosterKey()).Err() }) + observedAt := time.Now() + staleScore := observedAt.Add(-store.cfg.MembershipTTL - store.cfg.MembershipTTL/2).UnixNano() + if err := client.ZAdd(ctx, store.rosterKey(), + redis.Z{Score: float64(staleScore), Member: "stale-1"}, + redis.Z{Score: float64(staleScore), Member: "stale-2"}, + redis.Z{Score: float64(observedAt.UnixNano()), Member: "live-1"}, + ).Err(); err != nil { + t.Fatal(err) + } + set, err := store.LoadAll(ctx) + if err != nil { + t.Fatal(err) + } + if set.Truncated || hasPublisherReason(set.Errors, "", "publisher_cap") { + t.Fatalf("stale roster entries triggered cap: %+v", set) + } + if err := client.ZAdd(ctx, store.rosterKey(), + redis.Z{Score: float64(observedAt.UnixNano()), Member: "live-2"}, + redis.Z{Score: float64(observedAt.UnixNano()), Member: "live-3"}, + ).Err(); err != nil { + t.Fatal(err) + } + set, err = store.LoadAll(ctx) + if err != nil { + t.Fatal(err) + } + if !set.Truncated || !hasPublisherReason(set.Errors, "", "publisher_cap") { + t.Fatalf("live roster cap not reported: %+v", set) + } + }) + + t.Run("two publishers missing meta oversized and pruning", func(t *testing.T) { + prefix := "test:stelem:" + uuid.NewString() + one, two := newStore(prefix, "p1"), newStore(prefix, "p2") + one.cfg.MembershipTTL, two.cfg.MembershipTTL = 100*time.Millisecond, 100*time.Millisecond + t.Cleanup(func() { + keys := []string{one.rosterKey(), one.snapshotKey("p1"), one.snapshotKey("p2"), one.snapshotKey("missing"), one.snapshotKey("oversized"), one.snapshotKey("crashed")} + _ = client.Del(ctx, keys...).Err() + }) + now := time.Now() + if err := one.Publish(ctx, Snapshot{PublisherID: "p1", NodeID: "n1", CapturedAt: now, Sessions: []SessionView{{SessionID: "one"}}}); err != nil { + t.Fatal(err) + } + if err := two.Publish(ctx, Snapshot{PublisherID: "p2", NodeID: "n2", CapturedAt: now, Sessions: []SessionView{{SessionID: "two"}}}); err != nil { + t.Fatal(err) + } + set, err := one.LoadAll(ctx) + if err != nil || len(set.Snapshots) != 2 { + t.Fatalf("two publisher load = %+v, %v", set, err) + } + + if err := client.ZAdd(ctx, one.rosterKey(), redis.Z{Score: float64(time.Now().UnixNano()), Member: "missing"}).Err(); err != nil { + t.Fatal(err) + } + set, err = one.LoadAll(ctx) + if err != nil || !hasPublisherReason(set.Errors, "missing", "meta_missing") { + t.Fatalf("missing meta = %+v, %v", set, err) + } + + one.cfg.MaxSessions, one.cfg.MaxTransfers = 1, 1 + oversized := map[string]any{} + for i := 0; i < 19; i++ { + oversized[fmt.Sprintf("junk:%02d", i)] = "x" + } + if err := client.HSet(ctx, one.snapshotKey("oversized"), oversized).Err(); err != nil { + t.Fatal(err) + } + if err := client.ZAdd(ctx, one.rosterKey(), redis.Z{Score: float64(time.Now().UnixNano()), Member: "oversized"}).Err(); err != nil { + t.Fatal(err) + } + set, err = one.LoadAll(ctx) + if err != nil || !hasPublisherReason(set.Errors, "oversized", "oversized") { + t.Fatalf("oversized = %+v, %v", set, err) + } + + old := time.Now().Add(-300 * time.Millisecond) + if err := client.ZAdd(ctx, one.rosterKey(), redis.Z{Score: float64(old.UnixNano()), Member: "crashed"}).Err(); err != nil { + t.Fatal(err) + } + if err := one.Publish(ctx, Snapshot{PublisherID: "p1", NodeID: "n1", CapturedAt: time.Now()}); err != nil { + t.Fatal(err) + } + if score := client.ZScore(ctx, one.rosterKey(), "crashed"); !errors.Is(score.Err(), redis.Nil) { + t.Fatalf("crashed publisher remains: %v", score.Err()) + } + }) +} + +func hasPublisherReason(problems []PublisherError, publisher, reason string) bool { + for _, problem := range problems { + if problem.PublisherID == publisher && problem.Reason == reason { + return true + } + } + return false +} diff --git a/internal/streamtelemetry/view.go b/internal/streamtelemetry/view.go index 6941be265..4a814d1c4 100644 --- a/internal/streamtelemetry/view.go +++ b/internal/streamtelemetry/view.go @@ -81,6 +81,8 @@ type TransferView struct { type Snapshot struct { PublisherID string NodeID string + PublisherEpoch int64 + Sequence uint64 CapturedAt time.Time Sessions []SessionView Transfers []TransferView From d332db9a419de6d3a5d2faf049e5bb6429703423 Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Mon, 17 Aug 2026 01:48:30 +0000 Subject: [PATCH 07/44] feat(streamtelemetry): enrol the proxy and transcode-node families MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0b shipped with only the native family Enrolled. This enrols the proxy viewer edge and the transcode node, the pair that first exercises the relay-vs-viewer byte split §2.2's Role field exists for. Proxy routes attach after the handler's last authorization check — after verifyToken for the stream routes, after the PlayMethodDownload check on the local download branch, and after ValidArtifactID inside relayDownloadArtifact — so a rejected request creates no logical activity. Downloads attach as Transfers, never sessions: proxy download tokens mint a fresh session id per redirect by construction. The node publishes a correlation key and nothing else. Its URL {session_id} is the transcode transport id, not the canonical playback session id, so canonicalSessionID resolves the viewer edge's id from the forwarded X-Silo-Stream-Token and falls back to node-transport: rather than joining a session it cannot prove. Its capture hook records no viewer IP, device or client: the peer is an API or proxy process behind requireBearer, and recording its address would put a server address in ViewerIPs. §4.3 — a node cannot know who is watching. Merged start-time authority now comes from viewer-edge contributions only. A relay contribution carries a publisher-local first-seen stamp, which normalizeStartedAt marks degraded; mergeSession previously ORed that across every publisher, so correlating a node would have flipped an authoritative proxy session to degraded. CopyChunked no longer nests io.LimitReader. The kernel sendfile path unwraps exactly one limiter before it looks for the *os.File, so every accounting layer that re-wrapped its source silently forfeited sendfile — including on origin/main, where the egress meter forwarded no ReadFrom at all. Measured with strace over an 8 MiB body through the mounted proxy direct-play router: 0 sendfile syscalls before, 6 after, through three accounting wrappers. Standalone proxy and transcode processes now build a registry and join the Redis roster, with Stop deferred so a deploy does not leave a stale roster entry degrading the global view for MembershipTTL. No new feature flag: those are separate processes, so SILO_STREAM_TELEMETRY_ENABLED already gates each family independently. Measured cost, paired sub-benchmarks in one run at -count=5: direct play +10 allocs/op and ~1.1 KB/op; transcode segment +11 allocs/op and ~1.3 KB/op. Throughput ranges overlap on both. Built via a Claude<->Codex relay: Claude (Opus 5) planned and reviewed, Codex gpt-5.6-sol adversarially reviewed the plan and implemented it, Claude ran the gates and confirmed three defects, Codex fixed them. Part of #135 --- cmd/silo/main.go | 51 +++-- internal/api/handlers/playback.go | 16 +- internal/httpstream/readfrom.go | 26 +++ internal/httpstream/readfrom_test.go | 87 ++++++++ internal/playback/streamtelemetry.go | 60 ++++++ internal/playback/streamtelemetry_test.go | 37 ++++ internal/playback/transcode.go | 6 + internal/proxy/media_routes.go | 41 +++- internal/proxy/media_routes_test.go | 4 +- internal/proxy/router_socket_test.go | 128 ++++++++++++ internal/proxy/server.go | 40 ++-- internal/proxy/streamtelemetry.go | 38 ++++ internal/proxy/streamtelemetry_bench_test.go | 93 +++++++++ internal/proxy/testdata/media_routes.txt | 44 ++--- internal/streamtelemetry/global.go | 36 ++-- internal/streamtelemetry/global_test.go | 63 ++++++ internal/streamtelemetry/observation.go | 11 ++ internal/streamtelemetry/registry.go | 6 + internal/transcodenode/media_routes.go | 28 ++- internal/transcodenode/media_routes_test.go | 4 +- internal/transcodenode/server.go | 19 +- internal/transcodenode/streamtelemetry.go | 44 +++++ .../transcodenode/streamtelemetry_test.go | 185 ++++++++++++++++++ .../transcodenode/testdata/media_routes.txt | 16 +- 24 files changed, 991 insertions(+), 92 deletions(-) create mode 100644 internal/playback/streamtelemetry.go create mode 100644 internal/playback/streamtelemetry_test.go create mode 100644 internal/proxy/streamtelemetry.go create mode 100644 internal/proxy/streamtelemetry_bench_test.go create mode 100644 internal/transcodenode/streamtelemetry.go create mode 100644 internal/transcodenode/streamtelemetry_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 02861fd01..efafc2e86 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -30,6 +30,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/jackc/pgx/v5/pgxpool" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" sdkcapability "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability" @@ -169,6 +170,29 @@ func registerClientIPConfigReload(watcher *nodeconfig.Watcher, resolver *clienti }) } +// newStreamTelemetryRegistry builds the telemetry registry for this process, +// preferring the Redis-backed store in distributed mode. It never falls back to +// LocalStore on a failed ping: cache.NewRedisClient builds a lazy client that +// never dials, and a Redis restart mid-deploy must not strand a publisher +// local-only for the life of the process. +func newStreamTelemetryRegistry(ctx context.Context, nodeID string, redisClient *redis.Client) *streamtelemetry.Registry { + streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) + store := streamtelemetry.GlobalSnapshotStore(streamtelemetry.NewLocalStore()) + if streamTelemetryConfig.Enabled && streamTelemetryConfig.Distributed { + if redisClient != nil { + store = streamtelemetry.NewRedisStore(redisClient, streamTelemetryConfig, slog.Default()) + pingCtx, pingCancel := context.WithTimeout(ctx, 2*time.Second) + if pingErr := redisClient.Ping(pingCtx).Err(); pingErr != nil { + slog.ErrorContext(ctx, "stream telemetry distributed mode cannot reach redis; publisher will retry each sweep", "address", redisClient.Options().Addr, "error", pingErr) + } + pingCancel() + } else { + slog.ErrorContext(ctx, "stream telemetry distributed mode requested but redis is not configured; using local store") + } + } + return streamtelemetry.NewRegistry(streamTelemetryConfig, store, slog.Default()) +} + func resolvePluginCacheDir() string { if v := strings.TrimSpace(os.Getenv("SILO_PLUGIN_CACHE_DIR")); v != "" { return v @@ -720,6 +744,8 @@ func main() { slog.Error("redis is required for this mode", "mode", mode, "error", err) os.Exit(1) } + streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, redisClient) + streamTelemetryRegistry.Start(appCtx) bootstrap := nodeconfig.BootstrapOverrides{ Listen: cfg.Server.Listen, @@ -751,6 +777,13 @@ func main() { defer cleanupCancel() tracker.Cleanup(cleanupCtx) }() + defer func() { + telemetryCtx, telemetryCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer telemetryCancel() + if stopErr := streamTelemetryRegistry.Stop(telemetryCtx); stopErr != nil { + slog.Error("stream telemetry shutdown error", "error", stopErr) + } + }() var handler http.Handler if mode == "proxy" { @@ -761,6 +794,7 @@ func main() { } registerClientIPConfigReload(watcher, proxyIPResolver) srv.SetClientIPResolver(proxyIPResolver) + srv.SetStreamTelemetry(streamTelemetryRegistry) srv.SetRemoteArtifactMissReporter(downloads.NewArtifactManager( downloads.NewArtifactRepository(pool), downloads.NewRepository(pool), @@ -779,6 +813,7 @@ func main() { // start, so this node can rebuild a Jellyfin transcode after its own // restart (the node hop token is recipe-less). Shares the offload Redis. srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0)) + srv.SetStreamTelemetry(streamTelemetryRegistry) // Reclaim orphaned transcode dirs at boot and hourly thereafter, bound // to appCtx so it stops on shutdown. srv.StartOrphanSweeper(appCtx) @@ -846,21 +881,7 @@ func main() { } if mode == "" || mode == "integrated" || mode == "api" { - streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) - store := streamtelemetry.GlobalSnapshotStore(streamtelemetry.NewLocalStore()) - if streamTelemetryConfig.Enabled && streamTelemetryConfig.Distributed { - if apiRedisClient != nil { - store = streamtelemetry.NewRedisStore(apiRedisClient, streamTelemetryConfig, slog.Default()) - pingCtx, pingCancel := context.WithTimeout(appCtx, 2*time.Second) - if pingErr := apiRedisClient.Ping(pingCtx).Err(); pingErr != nil { - slog.Error("stream telemetry distributed mode cannot reach redis; publisher will retry each sweep", "address", apiRedisClient.Options().Addr, "error", pingErr) - } - pingCancel() - } else { - slog.Error("stream telemetry distributed mode requested but redis is not configured; using local store") - } - } - streamTelemetryRegistry = streamtelemetry.NewRegistry(streamTelemetryConfig, store, slog.Default()) + streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, apiRedisClient) streamTelemetryRegistry.Start(appCtx) } diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 17e40bab9..f2a200eff 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -1111,21 +1111,7 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req // PlaybackClientInfoFromRequest captures and normalizes playback client headers // at the HTTP request boundary. func PlaybackClientInfoFromRequest(r *http.Request) playback.ClientInfo { - if r == nil { - return playback.ClientInfo{} - } - // Clamped here, at the boundary, rather than only where the session stamps - // them: the decision logs and playback_route_events are written from this - // value directly, so a client sending a header-sized build would otherwise - // reach both despite the published bound. Values stay opaque — trimmed and - // length-clamped, never parsed or validated against an enum. - return playback.ClientInfo{ - Name: r.Header.Get("X-Silo-Client"), - Version: r.Header.Get("X-Silo-Client-Version"), - Build: r.Header.Get("X-Silo-Client-Build"), - Channel: r.Header.Get("X-Silo-Client-Channel"), - UserAgent: r.UserAgent(), - }.Normalized() + return playback.ClientInfoFromRequest(r) } func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo { diff --git a/internal/httpstream/readfrom.go b/internal/httpstream/readfrom.go index fea3dd075..8a3e49bab 100644 --- a/internal/httpstream/readfrom.go +++ b/internal/httpstream/readfrom.go @@ -28,6 +28,32 @@ func CopyChunked(rf io.ReaderFrom, src io.Reader, chunk int64, record func(n int return n, err } + // A nested *io.LimitedReader defeats the kernel sendfile path, which unwraps + // exactly one limiter before it looks for the *os.File. When src is already + // limited, slice it by handing down a limiter over the same underlying reader + // and decrementing the caller's budget, so the innermost ReaderFrom still sees + // one limiter over the file. + if lr, ok := src.(*io.LimitedReader); ok { + var total int64 + for lr.N > 0 { + sliceSize := min(chunk, lr.N) + slice := &io.LimitedReader{R: lr.R, N: sliceSize} + n, err := rf.ReadFrom(slice) + lr.N -= n + total += n + if record != nil { + record(n, err) + } + if err != nil { + return total, err + } + if n < sliceSize { + return total, nil + } + } + return total, nil + } + var total int64 for { n, err := rf.ReadFrom(io.LimitReader(src, chunk)) diff --git a/internal/httpstream/readfrom_test.go b/internal/httpstream/readfrom_test.go index ff130190d..c9f26656f 100644 --- a/internal/httpstream/readfrom_test.go +++ b/internal/httpstream/readfrom_test.go @@ -3,9 +3,11 @@ package httpstream import ( "bytes" "compress/gzip" + "errors" "io" "net/http" "net/http/httptest" + "os" "testing" ) @@ -35,6 +37,91 @@ func TestCopyChunkedUsesReaderFromPerSlice(t *testing.T) { } } +type recordingReaderFrom struct { + readers []io.Reader + errAt int + err error +} + +func (f *recordingReaderFrom) ReadFrom(r io.Reader) (int64, error) { + f.readers = append(f.readers, r) + n, err := io.Copy(io.Discard, r) + if f.errAt == len(f.readers) { + return n, f.err + } + return n, err +} + +func TestCopyChunkedLimitedReaderPreservesSendfileShape(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "copy-chunked-") + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + if _, err := file.Write(make([]byte, 10)); err != nil { + t.Fatal(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + fake := &recordingReaderFrom{} + n, err := CopyChunked(fake, io.LimitReader(file, 10), 4, nil) + if err != nil || n != 10 || len(fake.readers) != 3 { + t.Fatalf("CopyChunked = n=%d err=%v calls=%d", n, err, len(fake.readers)) + } + for i, reader := range fake.readers { + lr, ok := reader.(*io.LimitedReader) + if !ok || lr.R != file { + t.Fatalf("reader %d = %T %+v; want one limiter over file", i, reader, reader) + } + } +} + +func TestCopyChunkedAccounting(t *testing.T) { + tests := []struct { + name string + size int + limit int64 + chunk int64 + wantCalls int + wantRecords int + }{ + {name: "limited shorter than chunk", size: 3, limit: 3, chunk: 4, wantCalls: 1, wantRecords: 1}, + {name: "limited exact multiple", size: 8, limit: 8, chunk: 4, wantCalls: 2, wantRecords: 2}, + {name: "unlimited shorter than chunk", size: 3, limit: -1, chunk: 4, wantCalls: 1, wantRecords: 1}, + {name: "unlimited exact multiple", size: 8, limit: -1, chunk: 4, wantCalls: 3, wantRecords: 3}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := io.Reader(bytes.NewReader(make([]byte, test.size))) + if test.limit >= 0 { + src = io.LimitReader(src, test.limit) + } + fake := &recordingReaderFrom{} + var records []int64 + n, err := CopyChunked(fake, src, test.chunk, func(n int64, _ error) { records = append(records, n) }) + if err != nil || n != int64(test.size) || len(fake.readers) != test.wantCalls || len(records) != test.wantRecords { + t.Fatalf("CopyChunked = n=%d err=%v calls=%d records=%v", n, err, len(fake.readers), records) + } + }) + } +} + +func TestCopyChunkedLimitedReaderErrorUpdatesBudget(t *testing.T) { + wantErr := errors.New("read failed") + fake := &recordingReaderFrom{errAt: 1, err: wantErr} + lr := &io.LimitedReader{R: bytes.NewReader(make([]byte, 10)), N: 10} + var recordedN int64 + var recordedErr error + n, err := CopyChunked(fake, lr, 4, func(n int64, err error) { + recordedN, recordedErr = n, err + }) + if !errors.Is(err, wantErr) || n != 4 || lr.N != 6 || recordedN != 4 || !errors.Is(recordedErr, wantErr) { + t.Fatalf("CopyChunked = n=%d err=%v remaining=%d record=(%d,%v)", n, err, lr.N, recordedN, recordedErr) + } +} + func TestWriterOnlyHidesReaderFrom(t *testing.T) { w := &readerFromResponseWriter{header: make(http.Header)} if _, ok := WriterOnly(w).(io.ReaderFrom); ok { diff --git a/internal/playback/streamtelemetry.go b/internal/playback/streamtelemetry.go new file mode 100644 index 000000000..b1f8568f1 --- /dev/null +++ b/internal/playback/streamtelemetry.go @@ -0,0 +1,60 @@ +package playback + +import ( + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// TelemetryTokenTiming resolves the timing a viewer-edge handler attaches from +// a verified stream token. It never invents a start time: a token with no usable +// timestamp yields a zero time and StartedAtSourceFirstSeen, and the caller +// supplies its own fallback if it has a better one. +func TelemetryTokenTiming(claims *streamtoken.Claims) ( + startedAt time.Time, + startedSource streamtelemetry.StartedAtSource, + tokenIssuedAt time.Time, + tokenSource streamtelemetry.TokenIssuedAtSource, +) { + startedSource = streamtelemetry.StartedAtSourceFirstSeen + tokenSource = streamtelemetry.TokenIssuedAtSourceNone + if claims == nil { + return + } + if resolved, source := claims.StartedAt(); !resolved.IsZero() { + startedAt = resolved + switch source { + case streamtoken.StartedAtSourceClaim: + startedSource = streamtelemetry.StartedAtSourceClaim + case streamtoken.StartedAtSourceIssuedAt: + startedSource = streamtelemetry.StartedAtSourceIssuedAt + } + } + if claims.IssuedAt != nil { + tokenIssuedAt = claims.IssuedAt.Time + tokenSource = streamtelemetry.TokenIssuedAtSourceVerified + } + return +} + +// ClientInfoFromRequest captures and normalizes playback client headers at the +// HTTP request boundary. +func ClientInfoFromRequest(r *http.Request) ClientInfo { + if r == nil { + return ClientInfo{} + } + // Clamped here, at the boundary, rather than only where the session stamps + // them: the decision logs and playback_route_events are written from this + // value directly, so a client sending a header-sized build would otherwise + // reach both despite the published bound. Values stay opaque — trimmed and + // length-clamped, never parsed or validated against an enum. + return ClientInfo{ + Name: r.Header.Get("X-Silo-Client"), + Version: r.Header.Get("X-Silo-Client-Version"), + Build: r.Header.Get("X-Silo-Client-Build"), + Channel: r.Header.Get("X-Silo-Client-Channel"), + UserAgent: r.UserAgent(), + }.Normalized() +} diff --git a/internal/playback/streamtelemetry_test.go b/internal/playback/streamtelemetry_test.go new file mode 100644 index 000000000..08409bb19 --- /dev/null +++ b/internal/playback/streamtelemetry_test.go @@ -0,0 +1,37 @@ +package playback + +import ( + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func TestTelemetryTokenTiming(t *testing.T) { + iat := time.Unix(1_700_000_000, 0).UTC() + claimStart := iat.Add(-time.Minute) + tests := []struct { + name string + claims *streamtoken.Claims + wantStarted time.Time + wantSource streamtelemetry.StartedAtSource + wantIssued time.Time + wantIssuedSrc streamtelemetry.TokenIssuedAtSource + }{ + {name: "nil", wantSource: streamtelemetry.StartedAtSourceFirstSeen, wantIssuedSrc: streamtelemetry.TokenIssuedAtSourceNone}, + {name: "original start claim", claims: &streamtoken.Claims{OriginalStartedAtUnixNano: claimStart.UnixNano(), RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(iat)}}, wantStarted: claimStart, wantSource: streamtelemetry.StartedAtSourceClaim, wantIssued: iat, wantIssuedSrc: streamtelemetry.TokenIssuedAtSourceVerified}, + {name: "issued at only", claims: &streamtoken.Claims{RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(iat)}}, wantStarted: iat, wantSource: streamtelemetry.StartedAtSourceIssuedAt, wantIssued: iat, wantIssuedSrc: streamtelemetry.TokenIssuedAtSourceVerified}, + {name: "neither", claims: &streamtoken.Claims{}, wantSource: streamtelemetry.StartedAtSourceFirstSeen, wantIssuedSrc: streamtelemetry.TokenIssuedAtSourceNone}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + started, source, issued, issuedSource := TelemetryTokenTiming(test.claims) + if !started.Equal(test.wantStarted) || source != test.wantSource || !issued.Equal(test.wantIssued) || issuedSource != test.wantIssuedSrc { + t.Fatalf("timing = (%v, %q, %v, %q)", started, source, issued, issuedSource) + } + }) + } +} diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index aa0c9bf95..82f340510 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -134,6 +134,12 @@ type TranscodeSession struct { reserveHWDeviceOnRestart bool } +// NewTranscodeSessionForTest exposes only the output directory needed by tests +// in other packages that exercise the mounted transcode-node media routes. +func NewTranscodeSessionForTest(outputDir string) *TranscodeSession { + return &TranscodeSession{outputDir: outputDir} +} + // SetRestartHook registers a callback fired after every successful Restart. // The owning handler uses it to re-arm the transcode throttler and the exit // monitor; firing it from Restart itself keeps every restart caller (web diff --git a/internal/proxy/media_routes.go b/internal/proxy/media_routes.go index 64fa1f263..fc3ae6e92 100644 --- a/internal/proxy/media_routes.go +++ b/internal/proxy/media_routes.go @@ -1,8 +1,12 @@ package proxy import ( + "net" "net/http" + "time" + "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -23,7 +27,42 @@ var proxyMediaRoutes = []streamtelemetry.MediaRoute{ func proxyRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyProxy, Method: method, Pattern: pattern, Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "verified_stream_token", - CapRelevant: capRelevant, Enrolled: false} + CapRelevant: capRelevant, Enrolled: true, Capture: proxyCapture(pattern)} } func declareProxyMediaRoutes() { streamtelemetry.DeclareRoutes(proxyMediaRoutes...) } + +func proxyCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { + return func(r *http.Request) streamtelemetry.CaptureSet { + client := playback.ClientInfoFromRequest(r) + viewerIP := clientip.FromContext(r.Context()) + if viewerIP == "" { + viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) + if viewerIP == "" { + viewerIP = r.RemoteAddr + } + } + return streamtelemetry.CaptureSet{ + Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, + DeviceID: r.Header.Get("X-Silo-Device-ID"), + Client: streamtelemetry.ClientVariant{Name: client.Name, Version: client.Version, Build: client.Build, Channel: client.Channel}, + UserAgent: client.UserAgent, ReceivedAt: time.Now(), + } + } +} + +func proxyMediaRoute(method, pattern string) streamtelemetry.MediaRoute { + for _, route := range proxyMediaRoutes { + if route.Method == method && route.Pattern == pattern { + return route + } + } + panic("undeclared proxy media route: " + method + " " + pattern) +} + +func observeProxy(registry *streamtelemetry.Registry, method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if registry == nil { + return handler + } + return registry.Observe(proxyMediaRoute(method, pattern))(handler).ServeHTTP +} diff --git a/internal/proxy/media_routes_test.go b/internal/proxy/media_routes_test.go index ca238a35b..4762c4d55 100644 --- a/internal/proxy/media_routes_test.go +++ b/internal/proxy/media_routes_test.go @@ -44,8 +44,8 @@ func assertMediaManifest(t *testing.T, fixtures []chi.Routes, declared []streamt t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") } for _, route := range declared { - if route.Enrolled { - t.Fatalf("non-native route enrolled: %s %s", route.Method, route.Pattern) + if !route.Enrolled || route.Capture == nil { + t.Fatalf("proxy route not fully enrolled: %s %s", route.Method, route.Pattern) } } } diff --git a/internal/proxy/router_socket_test.go b/internal/proxy/router_socket_test.go index 05e198329..569ed9186 100644 --- a/internal/proxy/router_socket_test.go +++ b/internal/proxy/router_socket_test.go @@ -1,12 +1,16 @@ package proxy import ( + "bytes" + "context" + "fmt" "io" "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -17,6 +21,7 @@ import ( "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" ) @@ -35,6 +40,11 @@ func newSocketProxyServer(t *testing.T, secret string, resolver *clientip.Resolv w.SetConfigForTest(cfg) srv := NewServer(w, nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy")) srv.SetClientIPResolver(resolver) + telemetryConfig := streamtelemetry.DefaultConfig("socket-proxy") + telemetryConfig.Enabled = true + registry := streamtelemetry.NewRegistry(telemetryConfig, streamtelemetry.NewLocalStore(), nil) + srv.SetStreamTelemetry(registry) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) return srv } @@ -111,6 +121,17 @@ func TestMountedProxyRouterServesMediaOverSocket(t *testing.T) { if got.status != http.StatusOK || got.body != socketProxyMedia { t.Fatalf("GET = %d %q, want 200 %q", got.status, got.body, socketProxyMedia) } + snapshot := srv.telemetry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + session := snapshot.Sessions[0] + if session.SessionID != "socket-proxy-1" || session.Subject != streamtelemetry.UserSubject(7) || session.ProfileID != "profile-1" || session.MediaFileID != 42 { + t.Fatalf("session identity = %+v", session) + } + if len(session.Routes) != 1 || session.Routes[0].Role != streamtelemetry.RoleViewerEgress || session.Routes[0].BytesAccepted != int64(len(socketProxyMedia)) { + t.Fatalf("session routes = %+v", session.Routes) + } etag := got.header.Get("ETag") if got = socketProxyRequest(t, client, http.MethodHead, mediaURL, nil); got.status != http.StatusOK { @@ -140,6 +161,93 @@ func TestMountedProxyRouterServesMediaOverSocket(t *testing.T) { } } +func TestMountedProxyRouterAuthorizationAndTransfersOverSocket(t *testing.T) { + const secret = "socket-proxy-auth-secret" + path := writeSocketProxyMedia(t) + tests := []struct { + name string + claims streamtoken.Claims + signingSecret string + routePrefix string + wantStatus int + wantSessions int + wantTransfers int + }{ + {name: "wrong signature", claims: streamtoken.Claims{SessionID: "bad", MediaPath: path, PlayMethod: "direct"}, signingSecret: "wrong", routePrefix: "/stream/direct/", wantStatus: http.StatusUnauthorized}, + {name: "download transfer", claims: streamtoken.Claims{SessionID: "download", MediaPath: path, PlayMethod: streamtoken.PlayMethodDownload, UserID: 7, ProfileID: "profile-1", MediaFileID: 42}, signingSecret: secret, wantStatus: http.StatusOK, wantTransfers: 1}, + {name: "playback token rejected from download", claims: streamtoken.Claims{SessionID: "playback", MediaPath: path, PlayMethod: "direct"}, signingSecret: secret, wantStatus: http.StatusUnauthorized}, + {name: "invalid remote artifact rejected", claims: streamtoken.Claims{SessionID: "artifact", PlayMethod: streamtoken.PlayMethodDownload, DownloadArtifactID: "../bad", TranscodeNode: "http://127.0.0.1"}, signingSecret: secret, wantStatus: http.StatusUnauthorized}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + srv := newSocketProxyServer(t, secret, nil) + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + token, err := streamtoken.Sign(test.claims, test.signingSecret, time.Minute) + if err != nil { + t.Fatal(err) + } + routePrefix := test.routePrefix + if routePrefix == "" { + routePrefix = "/downloads/file/" + } + got := socketProxyRequest(t, server.Client(), http.MethodGet, server.URL+routePrefix+token, nil) + if got.status != test.wantStatus { + t.Fatalf("status = %d, want %d", got.status, test.wantStatus) + } + snapshot := srv.telemetry.Sweep() + if len(snapshot.Sessions) != test.wantSessions || len(snapshot.Transfers) != test.wantTransfers { + t.Fatalf("snapshot = %+v", snapshot) + } + if test.wantTransfers == 1 { + transfer := snapshot.Transfers[0] + if transfer.Subject != streamtelemetry.UserSubject(7) || transfer.BytesAccepted != int64(len(socketProxyMedia)) { + t.Fatalf("transfer = %+v", transfer) + } + } + }) + } +} + +func TestMountedProxyRouterLargeRangeIsByteExact(t *testing.T) { + const secret = "socket-proxy-range-secret" + body := bytes.Repeat([]byte("range-body-"), 400_000) + path := filepath.Join(t.TempDir(), "large.mp4") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + srv := newSocketProxyServer(t, secret, nil) + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + start, end := 12345, len(body)-23456 + got := socketProxyRequest(t, server.Client(), http.MethodGet, server.URL+"/stream/direct/"+socketProxyMediaToken(t, secret, path), map[string]string{ + "Range": "bytes=" + strconv.Itoa(start) + "-" + strconv.Itoa(end), + }) + if got.status != http.StatusPartialContent || got.header.Get("Content-Range") != fmt.Sprintf("bytes %d-%d/%d", start, end, len(body)) || got.body != string(body[start:end+1]) { + t.Fatalf("large range = status %d content-range %q body bytes %d", got.status, got.header.Get("Content-Range"), len(got.body)) + } +} + +func TestMountedProxyRouterTelemetryDisabledIsInert(t *testing.T) { + const secret = "socket-proxy-disabled-secret" + path := writeSocketProxyMedia(t) + srv := newSocketProxyServer(t, secret, nil) + cfg := streamtelemetry.DefaultConfig("disabled") + disabled := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + t.Cleanup(func() { _ = disabled.Stop(context.Background()) }) + srv.SetStreamTelemetry(disabled) + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + got := socketProxyRequest(t, server.Client(), http.MethodGet, server.URL+"/stream/direct/"+socketProxyMediaToken(t, secret, path), nil) + if got.status != http.StatusOK || got.body != socketProxyMedia { + t.Fatalf("request = %d %q", got.status, got.body) + } + snapshot := disabled.Snapshot() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("disabled snapshot = %+v", snapshot) + } +} + // TestMountedProxyRouterResolvesViewerIPOverSocket is the trust-boundary test for // the resolver P0a mounted on the proxy. It runs over a real socket because the // resolver reads RemoteAddr, which only a real connection populates: the peer is @@ -170,6 +278,16 @@ func TestMountedProxyRouterResolvesViewerIPOverSocket(t *testing.T) { }) server := httptest.NewServer(mounted) t.Cleanup(server.Close) + path := writeSocketProxyMedia(t) + mediaURL := server.URL + "/stream/direct/" + socketProxyMediaToken(t, secret, path) + mediaResult := socketProxyRequest(t, server.Client(), http.MethodGet, mediaURL, map[string]string{"X-Forwarded-For": "203.0.113.9"}) + if mediaResult.status != http.StatusOK { + t.Fatalf("media status = %d", mediaResult.status) + } + snapshot := srv.telemetry.Sweep() + if len(snapshot.Sessions) != 1 || len(snapshot.Sessions[0].ViewerIPs) != 1 || snapshot.Sessions[0].ViewerIPs[0] != "203.0.113.9" { + t.Fatalf("viewer IPs = %+v", snapshot.Sessions) + } probe := func(t *testing.T, headers map[string]string) string { t.Helper() @@ -223,11 +341,13 @@ func TestMountedProxyRouterRelaysToNode(t *testing.T) { const secret = "socket-proxy-relay-secret" const segment = "segment-bytes-from-node" + var forwardedToken string node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.URL.Path, "/segment/") { http.NotFound(w, r) return } + forwardedToken = r.Header.Get("X-Silo-Stream-Token") w.Header().Set("Content-Type", "video/mp2t") _, _ = io.WriteString(w, segment) })) @@ -261,4 +381,12 @@ func TestMountedProxyRouterRelaysToNode(t *testing.T) { if srv.egress.RateKbps() < before { t.Fatal("relayed bytes were not counted by the egress meter") } + snapshot := srv.telemetry.Sweep() + if len(snapshot.Sessions) != 1 || len(snapshot.Sessions[0].Routes) != 1 || snapshot.Sessions[0].Routes[0].Role != streamtelemetry.RoleViewerEgress || snapshot.Sessions[0].Routes[0].BytesAccepted != int64(len(segment)) { + t.Fatalf("proxy telemetry = %+v", snapshot.Sessions) + } + forwardedClaims, err := streamtoken.Verify(forwardedToken, secret) + if err != nil || forwardedClaims.SessionID != snapshot.Sessions[0].SessionID { + t.Fatalf("forwarded claims = %+v, err=%v", forwardedClaims, err) + } } diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 0f210ad50..869346c75 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -24,6 +24,7 @@ import ( "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" ) @@ -35,6 +36,7 @@ type Server struct { artifactMissReporter remoteArtifactMissReporter egress *egressMeter clientIP *clientip.Resolver + telemetry *streamtelemetry.Registry // subCache stores full-track PGS (.sup) extracts under the transcode dir // so repeat selections skip the whole-file ffmpeg demux. subCache *playback.SubtitleCache @@ -85,6 +87,12 @@ func (s *Server) SetClientIPResolver(resolver *clientip.Resolver) { s.clientIP = resolver } +// SetStreamTelemetry wires local stream observation. A nil registry is a +// complete no-op. +func (s *Server) SetStreamTelemetry(registry *streamtelemetry.Registry) { + s.telemetry = registry +} + // newStreamTransport tunes the proxy→transcode-node connection pool. Many // concurrent viewers fan their segment fetches through one proxy→node pair, // and Go's default of 2 idle connections per host causes constant connection @@ -132,17 +140,17 @@ func (s *Server) Handler() http.Handler { r.Group(func(r chi.Router) { // Streaming and download bytes count toward the node's measured egress. r.Use(s.meterEgress) - r.Head("/stream/direct/{token}", s.handleDirectPlay) - r.Get("/stream/direct/{token}", s.handleDirectPlay) - r.Head("/stream/remux/{token}", s.handleRemux) - r.Get("/stream/remux/{token}", s.handleRemux) - r.Head("/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest) - r.Get("/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest) - r.Get("/stream/transcode/{token}/segment/{name}", s.handleTranscodeSegment) - r.Get("/stream/subtitles/{token}/{track}/fonts", s.handleSubtitleFonts) - r.Get("/stream/subtitles/{token}/{track}", s.handleSubtitle) - r.Head("/downloads/file/{token}", s.handleDownloadFile) - r.Get("/downloads/file/{token}", s.handleDownloadFile) + r.Head("/stream/direct/{token}", observeProxy(s.telemetry, http.MethodHead, "/stream/direct/{token}", s.handleDirectPlay)) + r.Get("/stream/direct/{token}", observeProxy(s.telemetry, http.MethodGet, "/stream/direct/{token}", s.handleDirectPlay)) + r.Head("/stream/remux/{token}", observeProxy(s.telemetry, http.MethodHead, "/stream/remux/{token}", s.handleRemux)) + r.Get("/stream/remux/{token}", observeProxy(s.telemetry, http.MethodGet, "/stream/remux/{token}", s.handleRemux)) + r.Head("/stream/transcode/{token}/master.m3u8", observeProxy(s.telemetry, http.MethodHead, "/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest)) + r.Get("/stream/transcode/{token}/master.m3u8", observeProxy(s.telemetry, http.MethodGet, "/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest)) + r.Get("/stream/transcode/{token}/segment/{name}", observeProxy(s.telemetry, http.MethodGet, "/stream/transcode/{token}/segment/{name}", s.handleTranscodeSegment)) + r.Get("/stream/subtitles/{token}/{track}/fonts", observeProxy(s.telemetry, http.MethodGet, "/stream/subtitles/{token}/{track}/fonts", s.handleSubtitleFonts)) + r.Get("/stream/subtitles/{token}/{track}", observeProxy(s.telemetry, http.MethodGet, "/stream/subtitles/{token}/{track}", s.handleSubtitle)) + r.Head("/downloads/file/{token}", observeProxy(s.telemetry, http.MethodHead, "/downloads/file/{token}", s.handleDownloadFile)) + r.Get("/downloads/file/{token}", observeProxy(s.telemetry, http.MethodGet, "/downloads/file/{token}", s.handleDownloadFile)) }) // Admin routes — bearer-auth protected. @@ -225,6 +233,7 @@ func (s *Server) handleDirectPlay(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + attachStream(r.Context(), claims) info := sessionInfo(s.tracker, claims, "direct_play") s.tracker.Track(r.Context(), info) @@ -248,6 +257,9 @@ func (s *Server) handleDownloadFile(w http.ResponseWriter, r *http.Request) { http.Error(w, "unauthorized", http.StatusUnauthorized) return } + if !remoteArtifact { + attachTransfer(r.Context(), claims) + } // HEAD is a capability/path preflight, not an active transfer. Counting it // would briefly consume job capacity and could make a health report retire @@ -291,6 +303,7 @@ func (s *Server) relayDownloadArtifact(w http.ResponseWriter, r *http.Request, c http.Error(w, "unauthorized", http.StatusUnauthorized) return } + attachTransfer(r.Context(), claims) cfg := s.watcher.Config() if cfg == nil || strings.TrimSpace(cfg.Auth.JWTSecret) == "" { http.Error(w, "download unavailable", http.StatusServiceUnavailable) @@ -363,6 +376,7 @@ func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + attachStream(r.Context(), claims) info := sessionInfo(s.tracker, claims, "remux") s.tracker.Track(r.Context(), info) @@ -392,6 +406,7 @@ func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request) if claims == nil { return } + attachStream(r.Context(), claims) s.touchTranscodeSession(r, claims) s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/master.m3u8") } @@ -401,6 +416,7 @@ func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) if claims == nil { return } + attachStream(r.Context(), claims) s.touchTranscodeSession(r, claims) name := chi.URLParam(r, "name") s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/segment/"+name) @@ -450,6 +466,7 @@ func (s *Server) handleSubtitle(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + attachStream(r.Context(), claims) cfg := s.watcher.Config() trackParam := chi.URLParam(r, "track") trackIndex, requestedFormat, err := playback.ParseSubtitleTrackParam(trackParam) @@ -523,6 +540,7 @@ func (s *Server) handleSubtitleFonts(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + attachStream(r.Context(), claims) cfg := s.watcher.Config() trackParam := chi.URLParam(r, "track") trackIndex, _, err := playback.ParseSubtitleTrackParam(trackParam) diff --git a/internal/proxy/streamtelemetry.go b/internal/proxy/streamtelemetry.go new file mode 100644 index 000000000..eb305972a --- /dev/null +++ b/internal/proxy/streamtelemetry.go @@ -0,0 +1,38 @@ +package proxy + +import ( + "context" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func attachStream(ctx context.Context, claims *streamtoken.Claims) { + if claims == nil { + return + } + startedAt, startedSource, tokenIssuedAt, tokenSource := playback.TelemetryTokenTiming(claims) + if startedAt.IsZero() { + startedAt = time.Now().UTC() + } + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{ + Subject: streamtelemetry.UserSubject(claims.UserID), ProfileID: claims.ProfileID, + SessionID: claims.SessionID, MediaFileID: claims.MediaFileID, PlayMethod: claims.PlayMethod, + StartedAt: startedAt, StartedAtSource: startedSource, + TokenIssuedAt: tokenIssuedAt, TokenIssuedAtSource: tokenSource, + }) +} + +func attachTransfer(ctx context.Context, claims *streamtoken.Claims) { + if claims == nil { + return + } + startedAt, startedSource, tokenIssuedAt, tokenSource := playback.TelemetryTokenTiming(claims) + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{ + Subject: streamtelemetry.UserSubject(claims.UserID), ProfileID: claims.ProfileID, + MediaFileID: claims.MediaFileID, StartedAt: startedAt, StartedAtSource: startedSource, + TokenIssuedAt: tokenIssuedAt, TokenIssuedAtSource: tokenSource, + }) +} diff --git a/internal/proxy/streamtelemetry_bench_test.go b/internal/proxy/streamtelemetry_bench_test.go new file mode 100644 index 000000000..36388f854 --- /dev/null +++ b/internal/proxy/streamtelemetry_bench_test.go @@ -0,0 +1,93 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func BenchmarkProxyStreamTelemetry(b *testing.B) { + for _, endpoint := range []string{"direct_play", "transcode_segment"} { + b.Run(endpoint, func(b *testing.B) { + for _, enabled := range []bool{false, true} { + name := "disabled" + if enabled { + name = "enabled" + } + b.Run(name, func(b *testing.B) { + benchmarkProxyMediaRoute(b, endpoint, enabled) + }) + } + }) + } +} + +func benchmarkProxyMediaRoute(b *testing.B, endpoint string, enabled bool) { + b.Helper() + const secret = "proxy-benchmark-secret" + watcher := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = secret + cfg.Playback.TranscodeDir = b.TempDir() + watcher.SetConfigForTest(cfg) + srv := NewServer(watcher, nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy")) + telemetryConfig := streamtelemetry.DefaultConfig("proxy-benchmark") + telemetryConfig.Enabled = enabled + registry := streamtelemetry.NewRegistry(telemetryConfig, streamtelemetry.NewLocalStore(), nil) + srv.SetStreamTelemetry(registry) + b.Cleanup(func() { _ = registry.Stop(context.Background()) }) + + body := make([]byte, 64<<10) + claims := streamtoken.Claims{SessionID: "benchmark-session", PlayMethod: "direct", UserID: 7, ProfileID: "profile", MediaFileID: 42} + var node *httptest.Server + if endpoint == "direct_play" { + path := filepath.Join(b.TempDir(), "media.mp4") + if err := os.WriteFile(path, body, 0o600); err != nil { + b.Fatal(err) + } + claims.MediaPath = path + } else { + node = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) })) + b.Cleanup(node.Close) + claims.PlayMethod = "transcode" + claims.TranscodeNode = node.URL + claims.TranscodeTransportID = "benchmark-transport" + } + token, err := streamtoken.Sign(claims, secret, time.Hour) + if err != nil { + b.Fatal(err) + } + path := "/stream/direct/" + token + if endpoint == "transcode_segment" { + path = "/stream/transcode/" + token + "/segment/seg1.ts" + } + server := httptest.NewServer(srv.Handler()) + b.Cleanup(server.Close) + client := server.Client() + b.Cleanup(client.CloseIdleConnections) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := client.Get(server.URL + path) + if err != nil { + b.Fatal(err) + } + _, copyErr := io.Copy(io.Discard, resp.Body) + closeErr := resp.Body.Close() + if copyErr != nil || closeErr != nil || resp.StatusCode != http.StatusOK { + b.Fatalf("request = status %d, copy %v, close %v", resp.StatusCode, copyErr, closeErr) + } + } +} diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index c069fab55..b73c8ca4f 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -1,32 +1,32 @@ # fixture 1 POST /admin/force-reload non-media GET /api/v1/health non-media -GET /downloads/file/{token} media transfer viewer_egress false false -HEAD /downloads/file/{token} media transfer viewer_egress false false +GET /downloads/file/{token} media transfer viewer_egress false true +HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media GET /status non-media -GET /stream/direct/{token} media playback viewer_egress true false -HEAD /stream/direct/{token} media playback viewer_egress true false -GET /stream/remux/{token} media playback viewer_egress true false -HEAD /stream/remux/{token} media playback viewer_egress true false -GET /stream/subtitles/{token}/{track} media playback viewer_egress true false -GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true false -GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false -HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false -GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true false +GET /stream/direct/{token} media playback viewer_egress true true +HEAD /stream/direct/{token} media playback viewer_egress true true +GET /stream/remux/{token} media playback viewer_egress true true +HEAD /stream/remux/{token} media playback viewer_egress true true +GET /stream/subtitles/{token}/{track} media playback viewer_egress true true +GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true true +GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true +HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true +GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true true # fixture 2 POST /admin/force-reload non-media GET /api/v1/health non-media -GET /downloads/file/{token} media transfer viewer_egress false false -HEAD /downloads/file/{token} media transfer viewer_egress false false +GET /downloads/file/{token} media transfer viewer_egress false true +HEAD /downloads/file/{token} media transfer viewer_egress false true GET /hw-capabilities non-media GET /status non-media -GET /stream/direct/{token} media playback viewer_egress true false -HEAD /stream/direct/{token} media playback viewer_egress true false -GET /stream/remux/{token} media playback viewer_egress true false -HEAD /stream/remux/{token} media playback viewer_egress true false -GET /stream/subtitles/{token}/{track} media playback viewer_egress true false -GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true false -GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false -HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true false -GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true false +GET /stream/direct/{token} media playback viewer_egress true true +HEAD /stream/direct/{token} media playback viewer_egress true true +GET /stream/remux/{token} media playback viewer_egress true true +HEAD /stream/remux/{token} media playback viewer_egress true true +GET /stream/subtitles/{token}/{track} media playback viewer_egress true true +GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true true +GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true +HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true +GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true true diff --git a/internal/streamtelemetry/global.go b/internal/streamtelemetry/global.go index 25436ba0f..03b692c73 100644 --- a/internal/streamtelemetry/global.go +++ b/internal/streamtelemetry/global.go @@ -333,6 +333,18 @@ func mergeSession(id string, contributions []sessionContribution, params ViewPar subjectValues, profileValues, mediaValues := map[string][]PublisherRef{}, map[string][]PublisherRef{}, map[string][]PublisherRef{} winningRank := 0 winningTimes := map[int64]struct{}{} + hasViewerEdge := false + for _, contribution := range contributions { + for _, route := range contribution.view.Routes { + if route.Role == RoleViewerEgress { + hasViewerEdge = true + break + } + } + if hasViewerEdge { + break + } + } for _, contribution := range contributions { session, ref := contribution.view, contribution.ref result.Publishers = append(result.Publishers, ref) @@ -356,21 +368,23 @@ func mergeSession(id string, contributions []sessionContribution, params ViewPar mediaValues[strconv.Itoa(session.MediaFileID)] = append(mediaValues[strconv.Itoa(session.MediaFileID)], ref) } } - rank := startedAtRank(session.StartedAtSource) - if !session.StartedAt.IsZero() && rank > 0 { - if rank > winningRank { - winningRank = rank - result.StartedAt = session.StartedAt - result.StartedAtSource = session.StartedAtSource - winningTimes = map[int64]struct{}{session.StartedAt.UnixNano(): {}} - } else if rank == winningRank { - winningTimes[session.StartedAt.UnixNano()] = struct{}{} - if session.StartedAt.Before(result.StartedAt) { + if viewerEdge || !hasViewerEdge { + rank := startedAtRank(session.StartedAtSource) + if !session.StartedAt.IsZero() && rank > 0 { + if rank > winningRank { + winningRank = rank result.StartedAt = session.StartedAt + result.StartedAtSource = session.StartedAtSource + winningTimes = map[int64]struct{}{session.StartedAt.UnixNano(): {}} + } else if rank == winningRank { + winningTimes[session.StartedAt.UnixNano()] = struct{}{} + if session.StartedAt.Before(result.StartedAt) { + result.StartedAt = session.StartedAt + } } } + result.StartedAtDegraded = result.StartedAtDegraded || session.StartedAtDegraded } - result.StartedAtDegraded = result.StartedAtDegraded || session.StartedAtDegraded result.OpenObservations = saturatingAdd(result.OpenObservations, int64(session.OpenObservations)) result.RequestCount = saturatingAdd(result.RequestCount, session.RequestCount) result.RealtimeConnectionAlive = result.RealtimeConnectionAlive || session.RealtimeConnectionAlive diff --git a/internal/streamtelemetry/global_test.go b/internal/streamtelemetry/global_test.go index 915d6486f..d2755e70b 100644 --- a/internal/streamtelemetry/global_test.go +++ b/internal/streamtelemetry/global_test.go @@ -3,6 +3,7 @@ package streamtelemetry import ( "encoding/json" "math" + "net/http" "reflect" "slices" "testing" @@ -123,6 +124,68 @@ func TestBuildGlobalViewStartedAtDegradedRules(t *testing.T) { } } +func TestBuildGlobalViewStartAuthorityComesFromViewerEdges(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + proxyStart := at.Add(-time.Minute) + nodeStart := at.Add(-30 * time.Second) + proxy := Snapshot{PublisherID: "proxy", CapturedAt: at, Sessions: []SessionView{{ + SessionID: "session", StartedAt: proxyStart, StartedAtSource: StartedAtSourceClaim, + Routes: []RouteActivityView{viewerRoute(10)}, + }}} + node := Snapshot{PublisherID: "node", CapturedAt: at, Sessions: []SessionView{{ + SessionID: "session", StartedAt: nodeStart, StartedAtSource: StartedAtSourceFirstSeen, StartedAtDegraded: true, + Routes: []RouteActivityView{{Role: RoleInternalRelay, BytesAccepted: 5}}, + }}} + session := BuildGlobalView(globalSet(at, proxy, node), at, globalTestParams()).Sessions[0] + if !session.StartedAt.Equal(proxyStart) || session.StartedAtSource != StartedAtSourceClaim || session.StartedAtDegraded { + t.Fatalf("viewer-edge start authority = %+v", session) + } + + nodeOnly := BuildGlobalView(globalSet(at, node), at, globalTestParams()).Sessions[0] + if !nodeOnly.StartedAt.Equal(nodeStart) || !nodeOnly.StartedAtDegraded { + t.Fatalf("node-only fallback = %+v", nodeOnly) + } + + otherProxy := proxy + otherProxy.PublisherID = "proxy-2" + otherProxy.Sessions = []SessionView{{SessionID: "session", StartedAt: proxyStart.Add(time.Second), StartedAtSource: StartedAtSourceClaim, Routes: []RouteActivityView{viewerRoute(1)}}} + conflicted := BuildGlobalView(globalSet(at, proxy, otherProxy), at, globalTestParams()).Sessions[0] + if !conflicted.StartedAtDegraded { + t.Fatalf("equal-rank viewer-edge disagreement was not degraded: %+v", conflicted) + } +} + +func TestBuildGlobalViewMergesSeparateViewerAndRelayPublishers(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + started := at.Add(-time.Minute) + proxy := Snapshot{PublisherID: "proxy", NodeID: "proxy-node", CapturedAt: at, Sessions: []SessionView{{ + SessionID: "session", Subject: UserSubject(7), ProfileID: "profile", MediaFileID: 42, + StartedAt: started, StartedAtSource: StartedAtSourceClaim, + Routes: []RouteActivityView{{Method: http.MethodGet, Pattern: "/stream", Role: RoleViewerEgress, BytesAccepted: 100}}, + }}} + node := Snapshot{PublisherID: "node", NodeID: "transcode-node", CapturedAt: at, Sessions: []SessionView{{ + SessionID: "session", StartedAt: at.Add(-30 * time.Second), StartedAtSource: StartedAtSourceFirstSeen, StartedAtDegraded: true, + Routes: []RouteActivityView{{Method: http.MethodGet, Pattern: "/segment", Role: RoleInternalRelay, BytesAccepted: 40}}, + }}} + view := BuildGlobalView(globalSet(at, proxy, node), at, globalTestParams()) + if len(view.Sessions) != 1 { + t.Fatalf("sessions = %+v", view.Sessions) + } + session := view.Sessions[0] + if session.ViewerBytesAccepted != 100 || session.RelayBytesAccepted != 40 { + t.Fatalf("bytes = viewer %d relay %d", session.ViewerBytesAccepted, session.RelayBytesAccepted) + } + if session.Subject != UserSubject(7) || session.ProfileID != "profile" || session.MediaFileID != 42 || session.HasIdentityConflict { + t.Fatalf("identity = %+v", session) + } + if len(session.ViewerEdgePublishers) != 1 || session.ViewerEdgePublishers[0].PublisherID != "proxy" || len(session.Publishers) != 2 { + t.Fatalf("publishers = all %+v viewer %+v", session.Publishers, session.ViewerEdgePublishers) + } + if !session.StartedAt.Equal(started) || session.StartedAtSource != StartedAtSourceClaim || session.StartedAtDegraded { + t.Fatalf("started = %+v", session) + } +} + func TestBuildGlobalViewCompleteness(t *testing.T) { at := time.Now() params := globalTestParams() diff --git a/internal/streamtelemetry/observation.go b/internal/streamtelemetry/observation.go index 764cc85a6..4cdfb9a28 100644 --- a/internal/streamtelemetry/observation.go +++ b/internal/streamtelemetry/observation.go @@ -35,6 +35,17 @@ type observationTarget struct { transfer *transfer } +// Observing reports whether this request is being observed, so a caller can +// skip building an Attachment — and any verification work it needs — when +// telemetry is off or the route is not enrolled. +func Observing(ctx context.Context) bool { + if ctx == nil { + return false + } + obs, _ := ctx.Value(observationContextKey{}).(*Observation) + return obs != nil && obs.registry != nil +} + func (o *Observation) AddBytes(n int64) { if o != nil && n > 0 { o.bytesAccepted.Add(n) diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go index d135713a4..ca746b42d 100644 --- a/internal/streamtelemetry/registry.go +++ b/internal/streamtelemetry/registry.go @@ -422,8 +422,14 @@ func (r *Registry) sweep(sweepStart time.Time) Snapshot { return r.SnapshotAt(sweepStart) } +// Snapshot renders the registry state without sweeping live observations. Byte +// totals and LastByteAccepted reflect lastSweptBytes from the most recent sweep; +// callers that need current totals must call Sweep. func (r *Registry) Snapshot() Snapshot { return r.SnapshotAt(now()) } +// SnapshotAt renders the registry state at capturedAt without sweeping live +// observations. Byte totals and LastByteAccepted reflect lastSweptBytes from the +// most recent sweep; callers that need current totals must call Sweep. func (r *Registry) SnapshotAt(capturedAt time.Time) Snapshot { view := Snapshot{PublisherID: r.cfg.PublisherID, NodeID: r.cfg.NodeID, PublisherEpoch: r.cfg.PublisherEpoch, Sequence: r.sequence.Load(), CapturedAt: capturedAt, Truncated: r.truncated.Load(), DroppedObservations: r.droppedObservations.Load(), diff --git a/internal/transcodenode/media_routes.go b/internal/transcodenode/media_routes.go index 572d9edcb..dc4d84440 100644 --- a/internal/transcodenode/media_routes.go +++ b/internal/transcodenode/media_routes.go @@ -2,6 +2,7 @@ package transcodenode import ( "net/http" + "time" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -16,7 +17,32 @@ var transcodeNodeMediaRoutes = []streamtelemetry.MediaRoute{ func nodeRoute(method, pattern string, class streamtelemetry.Class) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyTranscodeNode, Method: method, Pattern: pattern, Class: class, Role: streamtelemetry.RoleInternalRelay, CanonicalSessionKey: "transport_session_id", - CapRelevant: false, Enrolled: false} + CapRelevant: false, Enrolled: true, Capture: nodeCapture(pattern)} } func declareTranscodeNodeMediaRoutes() { streamtelemetry.DeclareRoutes(transcodeNodeMediaRoutes...) } + +func nodeCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { + return func(r *http.Request) streamtelemetry.CaptureSet { + // The peer is an API or proxy process authenticated by requireBearer. + // Recording its address would falsely put a server address in ViewerIPs; + // a transcode node cannot know viewer identity. + return streamtelemetry.CaptureSet{Method: r.Method, Pattern: pattern, ReceivedAt: time.Now()} + } +} + +func transcodeNodeMediaRoute(method, pattern string) streamtelemetry.MediaRoute { + for _, route := range transcodeNodeMediaRoutes { + if route.Method == method && route.Pattern == pattern { + return route + } + } + panic("undeclared transcode-node media route: " + method + " " + pattern) +} + +func observeNode(registry *streamtelemetry.Registry, method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if registry == nil { + return handler + } + return registry.Observe(transcodeNodeMediaRoute(method, pattern))(handler).ServeHTTP +} diff --git a/internal/transcodenode/media_routes_test.go b/internal/transcodenode/media_routes_test.go index b832f073f..0aaf3e838 100644 --- a/internal/transcodenode/media_routes_test.go +++ b/internal/transcodenode/media_routes_test.go @@ -42,8 +42,8 @@ func TestMediaRouteManifest(t *testing.T) { t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") } for _, route := range transcodeNodeMediaRoutes { - if route.Enrolled { - t.Fatalf("transcode-node route enrolled: %s %s", route.Method, route.Pattern) + if !route.Enrolled || route.Capture == nil { + t.Fatalf("transcode-node route not fully enrolled: %s %s", route.Method, route.Pattern) } } } diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index f7c2031ed..4f38d126d 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -24,6 +24,7 @@ import ( "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" ) @@ -101,6 +102,7 @@ type Server struct { inputPaths InputPathAuthorizer transcodeDir string artifactRoot string + telemetry *streamtelemetry.Registry sessions map[string]*playback.TranscodeSession // lastAccess records, per registered session id, when a manifest or segment // request last touched the job (registration counts as the first access). @@ -445,6 +447,12 @@ func (s *Server) SetInputPathAuthorizer(authorizer InputPathAuthorizer) { s.inputPaths = authorizer } +// SetStreamTelemetry wires local stream observation. A nil registry is a +// complete no-op. +func (s *Server) SetStreamTelemetry(registry *streamtelemetry.Registry) { + s.telemetry = registry +} + // Handler returns the chi.Router with all transcode routes. func (s *Server) Handler() http.Handler { declareTranscodeNodeMediaRoutes() @@ -457,13 +465,13 @@ func (s *Server) Handler() http.Handler { r.Get("/hw-capabilities", s.handleHWCapabilities) r.Post("/chapter-thumbnails/extract", s.handleChapterThumbnailExtract) r.Post("/downloads/prepare", s.handleDownloadPrepare) - r.Head("/downloads/artifacts/{artifact_id}", s.handleDownloadArtifact) - r.Get("/downloads/artifacts/{artifact_id}", s.handleDownloadArtifact) + r.Head("/downloads/artifacts/{artifact_id}", observeNode(s.telemetry, http.MethodHead, "/downloads/artifacts/{artifact_id}", s.handleDownloadArtifact)) + r.Get("/downloads/artifacts/{artifact_id}", observeNode(s.telemetry, http.MethodGet, "/downloads/artifacts/{artifact_id}", s.handleDownloadArtifact)) r.Delete("/downloads/artifacts/{artifact_id}", s.handleDeleteDownloadArtifact) r.Post("/transcode/start", s.handleStart) r.Delete("/transcode/{session_id}", s.handleStop) - r.Get("/transcode/{session_id}/master.m3u8", s.handleManifest) - r.Get("/transcode/{session_id}/segment/{name}", s.handleSegment) + r.Get("/transcode/{session_id}/master.m3u8", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/master.m3u8", s.handleManifest)) + r.Get("/transcode/{session_id}/segment/{name}", observeNode(s.telemetry, http.MethodGet, "/transcode/{session_id}/segment/{name}", s.handleSegment)) r.Post("/admin/force-reload", s.handleForceReload) r.Get("/status", s.handleStatus) }) @@ -576,6 +584,7 @@ func (s *Server) handleDownloadArtifact(w http.ResponseWriter, r *http.Request) http.Error(w, "artifact unavailable", http.StatusInternalServerError) return } + streamtelemetry.Attach(r.Context(), streamtelemetry.Attachment{}) w.Header().Set("Content-Disposition", `attachment; filename="`+artifactID+`.mp4"`) w.Header().Set("Content-Type", playback.MimeFromExtension(path)) w.Header().Set("ETag", `"`+artifactID+`-`+strconv.FormatInt(stat.Size(), 10)+`"`) @@ -1145,6 +1154,7 @@ func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) { // not recorded this hit; count it so the reaper sees the liveness. s.touchSession(sessionID) } + s.attachTelemetrySession(r, sessionID) var manifest []byte var err error @@ -1190,6 +1200,7 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { // not recorded this hit; count it so the reaper sees the liveness. s.touchSession(sessionID) } + s.attachTelemetrySession(r, sessionID) segPath, err := session.GetSegment(name) if err != nil && err == playback.ErrSegmentNotFound { diff --git a/internal/transcodenode/streamtelemetry.go b/internal/transcodenode/streamtelemetry.go new file mode 100644 index 000000000..e367bffdc --- /dev/null +++ b/internal/transcodenode/streamtelemetry.go @@ -0,0 +1,44 @@ +package transcodenode + +import ( + "net/http" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// canonicalSessionID resolves the id the viewer-facing edge publishes this +// session under, so relay bytes merge into that session instead of a phantom +// one. The returned claims are the verified token, or nil, so callers need not +// verify twice. +func (s *Server) canonicalSessionID(r *http.Request, transportID string) (string, *streamtoken.Claims) { + fallback := "node-transport:" + transportID + if r == nil || s.watcher == nil { + return fallback, nil + } + tokenStr := r.Header.Get("X-Silo-Stream-Token") + cfg := s.watcher.Config() + if tokenStr == "" || cfg == nil { + return fallback, nil + } + claims, err := streamtoken.Verify(tokenStr, cfg.Auth.JWTSecret) + if err != nil { + return fallback, nil + } + expectedTransportID := claims.SessionID + if claims.TranscodeTransportID != "" { + expectedTransportID = claims.TranscodeTransportID + } + if expectedTransportID != transportID || claims.SessionID == "" { + return fallback, nil + } + return claims.SessionID, claims +} + +func (s *Server) attachTelemetrySession(r *http.Request, transportID string) { + if r == nil || !streamtelemetry.Observing(r.Context()) { + return + } + sessionID, _ := s.canonicalSessionID(r, transportID) + streamtelemetry.Attach(r.Context(), streamtelemetry.Attachment{SessionID: sessionID}) +} diff --git a/internal/transcodenode/streamtelemetry_test.go b/internal/transcodenode/streamtelemetry_test.go new file mode 100644 index 000000000..ef31395d4 --- /dev/null +++ b/internal/transcodenode/streamtelemetry_test.go @@ -0,0 +1,185 @@ +package transcodenode + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func telemetryNodeServer(t *testing.T) (*Server, *streamtelemetry.Registry, *httptest.Server) { + t.Helper() + srv := newTestServer(t) + cfg := streamtelemetry.DefaultConfig("transcode-node-test") + cfg.Enabled = true + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + srv.SetStreamTelemetry(registry) + server := httptest.NewServer(srv.Handler()) + t.Cleanup(server.Close) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + return srv, registry, server +} + +func nodeMediaRequest(t *testing.T, server *httptest.Server, path, token string) (int, []byte) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, server.URL+path, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+testSecret) + if token != "" { + req.Header.Set("X-Silo-Stream-Token", token) + } + resp, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, body +} + +func signedNodeTelemetryToken(t *testing.T, sessionID, transportID, secret string) string { + t.Helper() + token, err := streamtoken.Sign(streamtoken.Claims{ + SessionID: sessionID, TranscodeTransportID: transportID, PlayMethod: string(playback.PlayTranscode), + UserID: 7, ProfileID: "profile-1", MediaFileID: 42, + }, secret, time.Hour) + if err != nil { + t.Fatal(err) + } + return token +} + +func TestMountedTranscodeNodeSegmentTelemetry(t *testing.T) { + tests := []struct { + name string + withToken bool + wantSession string + }{ + {name: "canonical viewer session", withToken: true, wantSession: "viewer-session"}, + {name: "transport fallback", wantSession: "node-transport:transport-session"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + srv, registry, server := telemetryNodeServer(t) + const transportID = "transport-session" + const segment = "node-segment-bytes" + outputDir := t.TempDir() + if err := os.WriteFile(filepath.Join(outputDir, "seg1.ts"), []byte(segment), 0o600); err != nil { + t.Fatal(err) + } + srv.mu.Lock() + srv.sessions[transportID] = playback.NewTranscodeSessionForTest(outputDir) + srv.mu.Unlock() + token := "" + if test.withToken { + token = signedNodeTelemetryToken(t, "viewer-session", transportID, testSecret) + } + status, body := nodeMediaRequest(t, server, "/transcode/"+transportID+"/segment/seg1.ts", token) + if status != http.StatusOK || string(body) != segment { + t.Fatalf("segment = %d %q", status, body) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + session := snapshot.Sessions[0] + if session.SessionID != test.wantSession || session.Subject != (streamtelemetry.Subject{}) || session.ProfileID != "" || len(session.ViewerIPs) != 0 { + t.Fatalf("session = %+v", session) + } + if len(session.Routes) != 1 || session.Routes[0].Role != streamtelemetry.RoleInternalRelay || session.Routes[0].BytesAccepted != int64(len(segment)) { + t.Fatalf("routes = %+v", session.Routes) + } + }) + } +} + +func TestMountedTranscodeNodeArtifactTelemetry(t *testing.T) { + t.Run("successful relay", func(t *testing.T) { + srv, registry, server := telemetryNodeServer(t) + const artifactID = "telemetry-artifact" + const body = "artifact-bytes" + if err := os.MkdirAll(srv.artifactRoot, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(srv.artifactRoot, artifactID+".mp4"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + status, got := nodeMediaRequest(t, server, "/downloads/artifacts/"+artifactID, "") + if status != http.StatusOK || string(got) != body { + t.Fatalf("artifact = %d %q", status, got) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 1 { + t.Fatalf("snapshot = %+v", snapshot) + } + transfer := snapshot.Transfers[0] + if transfer.Role != streamtelemetry.RoleInternalRelay || transfer.BytesAccepted != int64(len(body)) || transfer.Subject != (streamtelemetry.Subject{}) || transfer.ViewerIP != "" { + t.Fatalf("transfer = %+v", transfer) + } + }) + + t.Run("missing artifact", func(t *testing.T) { + _, registry, server := telemetryNodeServer(t) + status, _ := nodeMediaRequest(t, server, "/downloads/artifacts/missing-artifact", "") + if status != http.StatusNotFound { + t.Fatalf("status = %d", status) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("snapshot = %+v", snapshot) + } + }) +} + +func TestMountedTranscodeNodeUnknownSessionCreatesNothing(t *testing.T) { + _, registry, server := telemetryNodeServer(t) + status, _ := nodeMediaRequest(t, server, "/transcode/unknown/segment/seg1.ts", "") + if status != http.StatusNotFound { + t.Fatalf("status = %d", status) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("snapshot = %+v", snapshot) + } +} + +func TestCanonicalSessionID(t *testing.T) { + srv := newTestServer(t) + const transportID = "transport-id" + tests := []struct { + name string + token string + want string + wantClaim bool + }{ + {name: "matching", token: signedNodeTelemetryToken(t, "viewer-id", transportID, testSecret), want: "viewer-id", wantClaim: true}, + {name: "different transport", token: signedNodeTelemetryToken(t, "viewer-id", "other", testSecret), want: "node-transport:" + transportID}, + {name: "wrong secret", token: signedNodeTelemetryToken(t, "viewer-id", transportID, "wrong"), want: "node-transport:" + transportID}, + {name: "no header", want: "node-transport:" + transportID}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/transcode/"+transportID+"/master.m3u8", nil) + if test.token != "" { + req.Header.Set("X-Silo-Stream-Token", test.token) + } + got, claims := srv.canonicalSessionID(req, transportID) + if got != test.want || (claims != nil) != test.wantClaim { + t.Fatalf("canonical = %q, claims=%+v", got, claims) + } + }) + } +} diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt index 5f7a8c77a..125f4d408 100644 --- a/internal/transcodenode/testdata/media_routes.txt +++ b/internal/transcodenode/testdata/media_routes.txt @@ -3,26 +3,26 @@ POST /admin/force-reload non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media -GET /downloads/artifacts/{artifact_id} media transfer internal_relay false false -HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false false +GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true +HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media -GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false false -GET /transcode/{session_id}/segment/{name} media playback internal_relay false false +GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false true +GET /transcode/{session_id}/segment/{name} media playback internal_relay false true # fixture 2 POST /admin/force-reload non-media GET /api/v1/health non-media POST /chapter-thumbnails/extract non-media DELETE /downloads/artifacts/{artifact_id} non-media -GET /downloads/artifacts/{artifact_id} media transfer internal_relay false false -HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false false +GET /downloads/artifacts/{artifact_id} media transfer internal_relay false true +HEAD /downloads/artifacts/{artifact_id} media transfer internal_relay false true POST /downloads/prepare non-media GET /hw-capabilities non-media GET /status non-media POST /transcode/start non-media DELETE /transcode/{session_id} non-media -GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false false -GET /transcode/{session_id}/segment/{name} media playback internal_relay false false +GET /transcode/{session_id}/master.m3u8 media manifest internal_relay false true +GET /transcode/{session_id}/segment/{name} media playback internal_relay false true From 29083144d95499dfb87fc26043697c5314df09fb Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:26:44 +0000 Subject: [PATCH 08/44] feat(streamtelemetry): enrol the jellycompat and ABS families MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes P0b's enrolment. Every declared media route in the repository is now observed; no family is left blind. Adds SILO_STREAM_TELEMETRY_FAMILIES, which the proxy and transcode-node change deliberately did without. Those are separate processes, so their own SILO_STREAM_TELEMETRY_ENABLED already gated them per family. Jellycompat and ABS share the API process with native, so without a gate this change would widen instrumentation across two more live byte paths on upgrade alone. The default set is therefore what shipped before this commit — native, proxy, transcode_node — and a shared-process family is named explicitly to enable it. The same variable is the kill switch: one misbehaving family can be dropped without losing observation of the rest. The resolved set is logged at startup. An unrecognized name disables telemetry and names the variable; a typo that silently observed nothing would be worse than no telemetry. The gate is read once per route at mount time, so it costs nothing per request. The attachment boundary is stated precisely and applied consistently: a logical session exists from AUTHORIZATION SUCCESS, not from a 2xx. Requests rejected before that point create nothing; a failure after it records an outcome on a real session, because it is real traffic by an authorized principal. This decides HandleMasterManifest, which finishes authorization at the CompatToken and media-source checks and then starts a transcode before writing a byte — the attach lands before that side effect, which is the whole reason §4.2 enrols manifest routes. Compat identity comes from the authenticated compat session, whose StreamAppUserID is the numeric silo account id, and its capture hook reads DeviceId/Client/Version through firstMediaBrowserAuthorizationValue — the parser the negotiation path already uses — rather than X-Silo-Client*, which Jellyfin clients never send. ABS reuses absPlaybackClientInfoFromRequest for the same reason, and absSubject maps a positive ABS user id onto UserSubject so ABS bytes sum with native and compat per user (§4.2b identity normalization); "0" and "-1" parse but name no account, so they stay abs_user. ABS routes are wrapped per route, never as another r.Use on the group Mount shares with socket.io. TestMountedStandaloneRouterPreservesSocketIOHijack now runs with telemetry both off and on — the §4.4 websocket regression the design owed, which only means anything with the wrapper mounted. BytesAccepted is pre-compression on any compat media route still compressed (subtitles), and wire bytes on the ones skipCompatMediaCompression exempts. Documented at compatCapture rather than "fixed". Measured cost, paired sub-benchmarks in one run at -count=5: jellycompat direct stream +10 allocs/op and ~1.2 KB/op; ABS public track +10 allocs/op and ~1.2 KB/op. Both match the native and proxy families. Planned via a Claude<->Codex relay (Claude Opus 5 planned, Codex gpt-5.6-sol adversarially reviewed the plan: nine findings, seven accepted, including the conservative family-gate default and the attachment-boundary correction). Codex hit its usage limit before the implementation step, so the implementation and review are Claude's alone — the plan review is the only cross-model step in this commit. Part of #135 --- cmd/silo/main.go | 7 + internal/audiobooks/abs/extras_handlers.go | 4 + internal/audiobooks/abs/file_handler.go | 6 + internal/audiobooks/abs/handler.go | 23 +- internal/audiobooks/abs/media_routes.go | 47 ++- internal/audiobooks/abs/media_routes_test.go | 10 +- internal/audiobooks/abs/router_socket_test.go | 20 +- internal/audiobooks/abs/rss_feeds_handler.go | 4 + internal/audiobooks/abs/streamtelemetry.go | 65 ++++ .../abs/streamtelemetry_bench_test.go | 45 +++ .../audiobooks/abs/streamtelemetry_test.go | 347 ++++++++++++++++++ .../audiobooks/abs/testdata/media_routes.txt | 44 +-- internal/jellycompat/handlers_playback.go | 5 + internal/jellycompat/media_routes.go | 60 ++- internal/jellycompat/media_routes_test.go | 11 +- internal/jellycompat/router.go | 24 +- internal/jellycompat/server.go | 18 +- internal/jellycompat/streams.go | 22 +- internal/jellycompat/streamtelemetry.go | 71 ++++ .../jellycompat/streamtelemetry_bench_test.go | 44 +++ internal/jellycompat/streamtelemetry_test.go | 310 ++++++++++++++++ .../jellycompat/testdata/media_routes.txt | 48 +-- internal/streamtelemetry/config.go | 75 ++++ internal/streamtelemetry/config_test.go | 68 +++- internal/streamtelemetry/writer.go | 5 +- internal/streamtelemetry/writer_test.go | 44 +++ 26 files changed, 1346 insertions(+), 81 deletions(-) create mode 100644 internal/audiobooks/abs/streamtelemetry.go create mode 100644 internal/audiobooks/abs/streamtelemetry_bench_test.go create mode 100644 internal/audiobooks/abs/streamtelemetry_test.go create mode 100644 internal/jellycompat/streamtelemetry.go create mode 100644 internal/jellycompat/streamtelemetry_bench_test.go create mode 100644 internal/jellycompat/streamtelemetry_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index efafc2e86..aee3b0a04 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -190,6 +190,10 @@ func newStreamTelemetryRegistry(ctx context.Context, nodeID string, redisClient slog.ErrorContext(ctx, "stream telemetry distributed mode requested but redis is not configured; using local store") } } + if streamTelemetryConfig.Enabled { + slog.InfoContext(ctx, "stream telemetry observing families", + "families", strings.Join(streamTelemetryConfig.ObservedFamilies(), ",")) + } return streamtelemetry.NewRegistry(streamTelemetryConfig, store, slog.Default()) } @@ -2385,6 +2389,8 @@ func main() { SessionSyncer: deps.SessionSyncer, } absH := audiobooksService.BuildABSHandler(absHDeps) + // Must precede Mount: Mount is what registers the observed handlers. + absH.SetStreamTelemetry(streamTelemetryRegistry) deps.ABSHandler = absH } _ = audiobooksService @@ -2626,6 +2632,7 @@ func main() { DB: deps.DB, SecretCipher: dataCipher, ClientIPResolver: ipResolver, + StreamTelemetry: streamTelemetryRegistry, NodePlanner: deps.NodePlanner, JWTSecret: cfg.Auth.JWTSecret, RecWorker: recWorker, diff --git a/internal/audiobooks/abs/extras_handlers.go b/internal/audiobooks/abs/extras_handlers.go index 126edd365..5afb97860 100644 --- a/internal/audiobooks/abs/extras_handlers.go +++ b/internal/audiobooks/abs/extras_handlers.go @@ -176,6 +176,10 @@ func (h *Handler) handleSetEpisodeProgress(w http.ResponseWriter, r *http.Reques // 404. The shape was intentionally chosen over 501 because the ABS web // reader treats 404 as "no ebook available for this item" and degrades // cleanly; 501 surfaces an alarming error banner. +// The route is enrolled in stream telemetry but attaches nothing: this handler +// unconditionally 404s, and a rejected request creates no logical activity. When +// it grows a real body, attach an abs transfer here — the wiring is already in +// place, only the Attach call and the *http.Request parameter are missing. func (h *Handler) handleEbookFile(w http.ResponseWriter, _ *http.Request) { http.Error(w, "ebook not available", http.StatusNotFound) } diff --git a/internal/audiobooks/abs/file_handler.go b/internal/audiobooks/abs/file_handler.go index 4c1ec2802..7f01fc932 100644 --- a/internal/audiobooks/abs/file_handler.go +++ b/internal/audiobooks/abs/file_handler.go @@ -100,6 +100,9 @@ func (h *Handler) handleFileStream(w http.ResponseWriter, r *http.Request) { } mediaFile := files[fileIdx] + // §4.2b: a bare file stream has a user but no stable playback session, so it + // is a Transfer, never cap-relevant and never subject to per-session rules. + attachABSTransfer(r.Context(), a.UserID, a.ProfileID, mediaFile.ID) // /download variant: hint the client to save rather than stream. if strings.HasSuffix(r.URL.Path, "/download") { @@ -191,6 +194,9 @@ func (h *Handler) handlePublicTrack(w http.ResponseWriter, r *http.Request) { return } mediaFile := files[idx-1] + // The session id IS the capability on this route, and it is also the + // canonical session key. Everything above this point is authorization. + attachABSSession(r.Context(), sid, sess.UserID, sess.ProfileID, mediaFile.ID, sess.StartedAt) ext := strings.ToLower(filepath.Ext(mediaFile.FilePath)) if ct := audioContentType(ext); ct != "" { diff --git a/internal/audiobooks/abs/handler.go b/internal/audiobooks/abs/handler.go index 0df981d6c..1e7284060 100644 --- a/internal/audiobooks/abs/handler.go +++ b/internal/audiobooks/abs/handler.go @@ -21,6 +21,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) // --------------------------------------------------------------------------- @@ -290,6 +291,16 @@ type Dependencies struct { // Handler wires the /abs/api/* and canonical ABS-client paths. type Handler struct { deps Dependencies + // telemetry is the local observation-only stream registry, shared with the + // native API process. Must be set before Mount: Mount is what registers the + // wrapped handlers, so a later call would have no effect. + telemetry *streamtelemetry.Registry +} + +// SetStreamTelemetry wires local stream observation. A nil registry is a +// complete no-op. Call before Mount. +func (h *Handler) SetStreamTelemetry(registry *streamtelemetry.Registry) { + h.telemetry = registry } // SkipMediaCompression reports whether an ABS media route must retain the @@ -413,14 +424,14 @@ func (h *Handler) mountRoutes(r chi.Router) { // the capability. Mounted at both /public/session and /abs/public/session // for compatibility with clients that pin either prefix. for _, prefix := range []string{"", "/abs"} { - r.Get(prefix+"/public/session/{sid}/track/{idx}", h.handlePublicTrack) - r.Head(prefix+"/public/session/{sid}/track/{idx}", h.handlePublicTrack) + r.Get(prefix+"/public/session/{sid}/track/{idx}", observeABS(h.telemetry, http.MethodGet, prefix+"/public/session/{sid}/track/{idx}", h.handlePublicTrack)) + r.Head(prefix+"/public/session/{sid}/track/{idx}", observeABS(h.telemetry, http.MethodHead, prefix+"/public/session/{sid}/track/{idx}", h.handlePublicTrack)) } // Public RSS feed routes — slug is the capability token, no auth. r.Get("/feed/{slug}.xml", h.handlePublicFeed) r.Get("/feed/{slug}", h.handlePublicFeed) - r.Get("/feed/{slug}/file/{ino}", h.handlePublicFeedFile) + r.Get("/feed/{slug}/file/{ino}", observeABS(h.telemetry, http.MethodGet, "/feed/{slug}/file/{ino}", h.handlePublicFeedFile)) // Server discovery — unauthenticated. Mounted at both /api and the // canonical root so curl-style network probes, the official ABS app's @@ -446,8 +457,8 @@ func (h *Handler) mountRoutes(r chi.Router) { // GET /api/items/{libraryItemId}/file/{ino} — stream a specific audio file. // /download variant is the same handler; Content-Disposition is set when // the path ends in /download. - r.Get(prefix+"/items/{libraryItemId}/file/{ino}", h.handleFileStream) - r.Get(prefix+"/items/{libraryItemId}/file/{ino}/download", h.handleFileStream) + r.Get(prefix+"/items/{libraryItemId}/file/{ino}", observeABS(h.telemetry, http.MethodGet, prefix+"/items/{libraryItemId}/file/{ino}", h.handleFileStream)) + r.Get(prefix+"/items/{libraryItemId}/file/{ino}/download", observeABS(h.telemetry, http.MethodGet, prefix+"/items/{libraryItemId}/file/{ino}/download", h.handleFileStream)) } }) @@ -537,7 +548,7 @@ func (h *Handler) mountRoutes(r chi.Router) { r.Get(prefix+"/me/stats/year/{year}", h.handleYearStats) // Ebook surface — stubs until the ebook scanner lands. // Mobile clients call these but degrade cleanly on empty/404. - r.Get(prefix+"/items/{id}/ebook/{fileid}", h.handleEbookFile) + r.Get(prefix+"/items/{id}/ebook/{fileid}", observeABS(h.telemetry, http.MethodGet, prefix+"/items/{id}/ebook/{fileid}", h.handleEbookFile)) r.Patch(prefix+"/items/{id}/ebook/{fileid}/status", h.handleEbookStatus) // E-reader devices + ebook email delivery — empty list / 503 // until SMTP integration is wired. diff --git a/internal/audiobooks/abs/media_routes.go b/internal/audiobooks/abs/media_routes.go index 79b9df489..bc9e358f5 100644 --- a/internal/audiobooks/abs/media_routes.go +++ b/internal/audiobooks/abs/media_routes.go @@ -2,6 +2,7 @@ package abs import ( "net/http" + "time" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -29,7 +30,51 @@ var absMediaRoutes = func() []streamtelemetry.MediaRoute { func absRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool, key string) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyABS, Method: method, Pattern: pattern, Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: key, - CapRelevant: capRelevant, Enrolled: false} + CapRelevant: capRelevant, Enrolled: true, Capture: absCapture(pattern)} +} + +// absCapture records the §2.2 request-time set for an ABS client. +// +// Client identity reuses absPlaybackClientInfoFromRequest (native_sessions.go), +// which the package already trusts for native-session mirroring — telemetry must +// not establish a second, poorer client-identity policy for the same headers. +// DeviceID stays empty: the bearer context (ctxAuth) does not carry the JWT's +// optional device id, and there is no other honest source. +func absCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { + return func(r *http.Request) streamtelemetry.CaptureSet { + client := absPlaybackClientInfoFromRequest(r) + return streamtelemetry.CaptureSet{ + Method: r.Method, Pattern: pattern, ViewerIP: requestClientIP(r), + Client: streamtelemetry.ClientVariant{ + Name: client.Name, Version: client.Version, Build: client.Build, Channel: client.Channel, + }, + UserAgent: client.UserAgent, ReceivedAt: time.Now(), + } + } } func declareABSMediaRoutes() { streamtelemetry.DeclareRoutes(absMediaRoutes...) } + +func absMediaRoute(method, pattern string) streamtelemetry.MediaRoute { + for _, route := range absMediaRoutes { + if route.Method == method && route.Pattern == pattern { + return route + } + } + panic("undeclared abs media route: " + method + " " + pattern) +} + +// observeABS wraps one ABS media handler. +// +// Wrapping PER ROUTE is deliberate and load-bearing. Mount puts h.accessLog on a +// group covering every route — media and socket.io alike (handler.go:362) — so +// adding telemetry as another r.Use there would put an extra ResponseWriter +// between engine.io and the raw connection. §4.4: "ABS mounts one access-log +// wrapper across both media and socket.io, so middleware placement decides +// whether websockets survive." +func observeABS(registry *streamtelemetry.Registry, method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if registry == nil { + return handler + } + return registry.Observe(absMediaRoute(method, pattern))(handler).ServeHTTP +} diff --git a/internal/audiobooks/abs/media_routes_test.go b/internal/audiobooks/abs/media_routes_test.go index 8e30de6d8..2d5437fdc 100644 --- a/internal/audiobooks/abs/media_routes_test.go +++ b/internal/audiobooks/abs/media_routes_test.go @@ -39,9 +39,15 @@ func TestMediaRouteManifest(t *testing.T) { if string(want) != actual { t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") } + // Every declared ABS route is enrolled and carries a capture hook. A nil + // Capture would fall back to genericCapture and lose the client identity + // absCapture reads through absPlaybackClientInfoFromRequest. for _, route := range absMediaRoutes { - if route.Enrolled { - t.Fatalf("ABS route enrolled: %s %s", route.Method, route.Pattern) + if !route.Enrolled { + t.Fatalf("abs route not enrolled: %s %s", route.Method, route.Pattern) + } + if route.Capture == nil { + t.Fatalf("abs route has no capture hook: %s %s", route.Method, route.Pattern) } } } diff --git a/internal/audiobooks/abs/router_socket_test.go b/internal/audiobooks/abs/router_socket_test.go index 4e5a331ba..c3f1d768c 100644 --- a/internal/audiobooks/abs/router_socket_test.go +++ b/internal/audiobooks/abs/router_socket_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/Silo-Server/silo-server/internal/httpstream" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) type hijackingSocketIOServer struct{} @@ -31,10 +32,27 @@ func (hijackingSocketIOServer) Handler() http.Handler { } func TestMountedStandaloneRouterPreservesSocketIOHijack(t *testing.T) { + t.Run("telemetry disabled", func(t *testing.T) { + assertSocketIOHijackSurvives(t, nil) + }) + // §4.4: "ABS mounts one access-log wrapper across both media and socket.io, + // so middleware placement decides whether websockets survive." The regression + // only means anything with the telemetry wrapper actually mounted — observeABS + // wraps per route precisely so nothing lands between engine.io and the raw + // connection. + t.Run("telemetry enabled", func(t *testing.T) { + assertSocketIOHijackSurvives(t, telemetryRegistry(t)) + }) +} + +func assertSocketIOHijackSurvives(t *testing.T, registry *streamtelemetry.Registry) { + t.Helper() router := chi.NewRouter() router.Use(middleware.Recoverer) router.Use(httpstream.CompressExcept(5, SkipMediaCompression)) - New(Dependencies{MediaStore: noopMediaStore{}, SocketIO: hijackingSocketIOServer{}}).Mount(router) + handler := New(Dependencies{MediaStore: noopMediaStore{}, SocketIO: hijackingSocketIOServer{}}) + handler.SetStreamTelemetry(registry) + handler.Mount(router) server := httptest.NewUnstartedServer(router) server.Start() diff --git a/internal/audiobooks/abs/rss_feeds_handler.go b/internal/audiobooks/abs/rss_feeds_handler.go index 47678dcc8..171e79b65 100644 --- a/internal/audiobooks/abs/rss_feeds_handler.go +++ b/internal/audiobooks/abs/rss_feeds_handler.go @@ -235,5 +235,9 @@ func (h *Handler) handlePublicFeedFile(w http.ResponseWriter, r *http.Request) { http.Error(w, "file not found", http.StatusNotFound) return } + // §4.2b: "the RSS feed route must resolve the feed owner". The slug is the + // capability and there is no authenticated caller, so the subject is the + // feed's owner rather than whoever fetched it. + attachABSTransfer(r.Context(), f.UserID, f.ProfileID, mf.ID) http.ServeFile(w, r, mf.FilePath) } diff --git a/internal/audiobooks/abs/streamtelemetry.go b/internal/audiobooks/abs/streamtelemetry.go new file mode 100644 index 000000000..b0e508970 --- /dev/null +++ b/internal/audiobooks/abs/streamtelemetry.go @@ -0,0 +1,65 @@ +package abs + +import ( + "context" + "strconv" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +// absSubject normalizes an ABS user id onto the shared telemetry subject space. +// +// ABS carries the silo account id as a string — handler.go and native_sessions.go +// both recover it with strconv.Atoi — so a valid account id maps onto the same +// UserSubject native, compat and proxy publish, which is what lets a per-user +// total sum across families (§4.2b's identity normalization). Only a positive +// integer is a valid account id: "0" and "-1" parse but name no account, so they +// stay abs_user rather than being merged into the shared user space. +func absSubject(userID string) streamtelemetry.Subject { + if userID == "" { + return streamtelemetry.Subject{} + } + if id, err := strconv.Atoi(userID); err == nil && id > 0 { + return streamtelemetry.UserSubject(id) + } + return streamtelemetry.Subject{Kind: streamtelemetry.SubjectABSUser, ID: userID} +} + +// attachABSSession attributes a public-track observation to its ABS playback +// session. The session id is the capability on this route — there is no bearer +// token — so it is also the canonical session key. +// +// The attachment boundary here, as everywhere in this module, is AUTHORIZATION +// SUCCESS: the handler has resolved the session, passed accessFilterForAuth and +// resolved the track. Requests rejected before that point create no logical +// activity; a failure after it records an outcome on a real session. +func attachABSSession(ctx context.Context, sessionID, userID, profileID string, mediaFileID int, startedAt time.Time) { + if sessionID == "" { + return + } + attachment := streamtelemetry.Attachment{ + Subject: absSubject(userID), ProfileID: profileID, SessionID: sessionID, + MediaFileID: mediaFileID, PlayMethod: "direct", + StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, + // ABS public tracks carry no signed stream token, so there is no issued-at + // to verify. Recording anything else would be a fabrication. + TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone, + } + if !startedAt.IsZero() { + attachment.StartedAt = startedAt + attachment.StartedAtSource = streamtelemetry.StartedAtSourceSession + } + streamtelemetry.Attach(ctx, attachment) +} + +// attachABSTransfer attributes a download-class pour: the bare file routes, the +// RSS feed file, and anything else with a user but no stable playback session +// (§4.2b). Never a SessionID, never a play method, never cap-relevant. +func attachABSTransfer(ctx context.Context, userID, profileID string, mediaFileID int) { + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{ + Subject: absSubject(userID), ProfileID: profileID, MediaFileID: mediaFileID, + StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, + TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone, + }) +} diff --git a/internal/audiobooks/abs/streamtelemetry_bench_test.go b/internal/audiobooks/abs/streamtelemetry_bench_test.go new file mode 100644 index 000000000..e48e45e1f --- /dev/null +++ b/internal/audiobooks/abs/streamtelemetry_bench_test.go @@ -0,0 +1,45 @@ +package abs + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +// BenchmarkABSStreamTelemetry pairs the enabled and disabled sub-benchmarks in +// one run so the comparison is not across process invocations. Run with +// -count=5: a single run of either side is inside run-to-run variance. +func BenchmarkABSStreamTelemetry(b *testing.B) { + b.Run("public_track/disabled", func(b *testing.B) { benchmarkABSPublicTrack(b, false) }) + b.Run("public_track/enabled", func(b *testing.B) { benchmarkABSPublicTrack(b, true) }) +} + +func benchmarkABSPublicTrack(b *testing.B, enabled bool) { + body := append([]byte("\xff\xfb\x00\x00"), bytes.Repeat([]byte("audio"), 400)...) + var registry *streamtelemetry.Registry + if enabled { + registry = telemetryRegistry(b) + } + server := absTelemetryServer(b, registry, absPublicTrackDeps(b, "sid-bench", "book-1", "42", body)) + url := server.URL + "/public/session/sid-bench/track/1" + client := server.Client() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := client.Get(url) + if err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + b.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b.Fatalf("status = %d", resp.StatusCode) + } + } +} diff --git a/internal/audiobooks/abs/streamtelemetry_test.go b/internal/audiobooks/abs/streamtelemetry_test.go new file mode 100644 index 000000000..6e62e2dca --- /dev/null +++ b/internal/audiobooks/abs/streamtelemetry_test.go @@ -0,0 +1,347 @@ +package abs + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + + "github.com/Silo-Server/silo-server/internal/httpstream" + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +func TestABSSubjectNormalization(t *testing.T) { + for _, test := range []struct { + name string + userID string + want streamtelemetry.Subject + }{ + // A positive integer is a silo account id, so it lands in the same + // subject space native, compat and proxy publish — that is what lets a + // per-user total sum across families (§4.2b). + {"positive account id", "42", streamtelemetry.UserSubject(42)}, + // "0" and "-1" parse but name no account. Merging them into the shared + // user space would attribute ABS bytes to a user that does not exist. + {"zero", "0", streamtelemetry.Subject{Kind: streamtelemetry.SubjectABSUser, ID: "0"}}, + {"negative", "-1", streamtelemetry.Subject{Kind: streamtelemetry.SubjectABSUser, ID: "-1"}}, + {"non numeric", "abc", streamtelemetry.Subject{Kind: streamtelemetry.SubjectABSUser, ID: "abc"}}, + {"overflow", "99999999999999999999", streamtelemetry.Subject{Kind: streamtelemetry.SubjectABSUser, ID: "99999999999999999999"}}, + {"empty", "", streamtelemetry.Subject{}}, + } { + t.Run(test.name, func(t *testing.T) { + if got := absSubject(test.userID); got != test.want { + t.Fatalf("absSubject(%q) = %+v, want %+v", test.userID, got, test.want) + } + }) + } +} + +// telemetryRegistry builds an enabled registry observing the ABS family. Every +// test that starts one must Stop it: the package-level now seam in +// streamtelemetry races leaked collector goroutines otherwise. +func telemetryRegistry(t testing.TB, families ...streamtelemetry.Family) *streamtelemetry.Registry { + t.Helper() + cfg := streamtelemetry.DefaultConfig("abs-test") + cfg.Enabled = true + cfg.Retention = time.Minute + if len(families) == 0 { + families = []streamtelemetry.Family{streamtelemetry.FamilyABS} + } + cfg.Families = make(map[streamtelemetry.Family]bool, len(families)) + for _, family := range families { + cfg.Families[family] = true + } + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + return registry +} + +// absTelemetryServer mounts the real ABS router — access log, compression and +// all — behind a real socket. Handler-level tests bypass the middleware under +// test, which is how this project once shipped a feature that was a no-op for +// weeks. +func absTelemetryServer(t testing.TB, registry *streamtelemetry.Registry, deps Dependencies) *httptest.Server { + t.Helper() + handler := New(deps) + handler.SetStreamTelemetry(registry) + router := chi.NewRouter() + router.Use(middleware.Recoverer) + router.Use(httpstream.CompressExcept(5, SkipMediaCompression)) + handler.Mount(router) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + return server +} + +func absPublicTrackDeps(t testing.TB, sid, contentID, userID string, body []byte) Dependencies { + t.Helper() + path := filepath.Join(t.TempDir(), "track.mp3") + if err := os.WriteFile(path, body, 0o644); err != nil { + t.Fatal(err) + } + store := &fakePlaybackSessionStore{} + _ = store.InsertPlaybackSession(context.Background(), ABSPlaybackSession{ + ID: sid, UserID: userID, ProfileID: "profile-1", ContentID: contentID, + StartedAt: time.Now().Add(-time.Hour).UTC().Truncate(time.Second), + }) + return Dependencies{ + MediaStore: &filesMediaStore{contentID: contentID, files: []*models.MediaFile{{ID: 77, FilePath: path}}}, + PlaybackSessionStore: store, + } +} + +// absResponse carries just what the assertions need. Returning the live +// *http.Response instead would leak an unclosed body past this helper. +type absResponse struct { + status int + header http.Header + body []byte +} + +func getWithHeaders(t *testing.T, client *http.Client, method, url string, headers map[string]string) absResponse { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatal(err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(resp.Body); err != nil { + t.Fatal(err) + } + return absResponse{status: resp.StatusCode, header: resp.Header.Clone(), body: buf.Bytes()} +} + +func TestMountedABSRouterAttributesPublicTrack(t *testing.T) { + body := []byte("\xff\xfb\x00\x00" + strings.Repeat("audio", 400)) + registry := telemetryRegistry(t) + deps := absPublicTrackDeps(t, "sid-telemetry", "book-1", "42", body) + server := absTelemetryServer(t, registry, deps) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + + got := getWithHeaders(t, client, http.MethodGet, server.URL+"/public/session/sid-telemetry/track/1", + map[string]string{"X-Silo-Client": "Silo Audiobooks", "X-Silo-Client-Version": "3.1.0"}) + if got.status != http.StatusOK || !bytes.Equal(got.body, body) { + t.Fatalf("GET = %d, %d bytes (want %d)", got.status, len(got.body), len(body)) + } + + // Byte totals come from Sweep, not Snapshot: SessionView.BytesAccepted is + // lastSweptBytes and only the sweep folds live observations into it. + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + session := snapshot.Sessions[0] + if session.SessionID != "sid-telemetry" { + t.Fatalf("session id = %q", session.SessionID) + } + if session.Subject != streamtelemetry.UserSubject(42) || session.ProfileID != "profile-1" { + t.Fatalf("subject = %+v profile = %q", session.Subject, session.ProfileID) + } + if session.MediaFileID != 77 { + t.Fatalf("media file id = %d, want 77", session.MediaFileID) + } + if session.PlayMethod != "direct" { + t.Fatalf("play method = %q", session.PlayMethod) + } + if session.StartedAtSource != streamtelemetry.StartedAtSourceSession { + t.Fatalf("started source = %q, want the ABS session's own start", session.StartedAtSource) + } + // ABS public tracks carry no signed stream token; claiming a verified + // issued-at would be a fabrication. + if session.TokenIssuedAtSources[streamtelemetry.TokenIssuedAtSourceNone] != 1 { + t.Fatalf("token sources = %+v", session.TokenIssuedAtSources) + } + if len(session.Routes) != 1 || session.Routes[0].Role != streamtelemetry.RoleViewerEgress || !session.Routes[0].CapRelevant { + t.Fatalf("routes = %+v", session.Routes) + } + if session.Routes[0].BytesAccepted != int64(len(body)) { + t.Fatalf("bytes = %d, want %d", session.Routes[0].BytesAccepted, len(body)) + } + if len(session.ClientVariants) != 1 || session.ClientVariants[0].Name != "Silo Audiobooks" || session.ClientVariants[0].Version != "3.1.0" { + t.Fatalf("client variants = %+v", session.ClientVariants) + } +} + +func TestMountedABSRouterPublicTrackEdgeCases(t *testing.T) { + body := []byte("\xff\xfb\x00\x00audio-bytes") + + t.Run("head counts zero bytes but one request", func(t *testing.T) { + registry := telemetryRegistry(t) + server := absTelemetryServer(t, registry, absPublicTrackDeps(t, "sid-head", "book-1", "42", body)) + got := getWithHeaders(t, server.Client(), http.MethodHead, server.URL+"/public/session/sid-head/track/1", nil) + if got.status != http.StatusOK || len(got.body) != 0 { + t.Fatalf("HEAD = %d, %d bytes", got.status, len(got.body)) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + if snapshot.Sessions[0].BytesAccepted != 0 || snapshot.Sessions[0].RequestCount != 1 { + t.Fatalf("HEAD accounting = bytes %d requests %d", + snapshot.Sessions[0].BytesAccepted, snapshot.Sessions[0].RequestCount) + } + }) + + t.Run("range is byte exact", func(t *testing.T) { + large := bytes.Repeat([]byte("abcdefghij"), 300_000) + registry := telemetryRegistry(t) + server := absTelemetryServer(t, registry, absPublicTrackDeps(t, "sid-range", "book-1", "42", large)) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + got := getWithHeaders(t, client, http.MethodGet, server.URL+"/public/session/sid-range/track/1", + map[string]string{"Range": "bytes=1000-2999"}) + if got.status != http.StatusPartialContent { + t.Fatalf("status = %d, want 206", got.status) + } + if want := fmt.Sprintf("bytes 1000-2999/%d", len(large)); got.header.Get("Content-Range") != want { + t.Fatalf("Content-Range = %q, want %q", got.header.Get("Content-Range"), want) + } + if !bytes.Equal(got.body, large[1000:3000]) { + t.Fatalf("range body mismatch: %d bytes", len(got.body)) + } + if snapshot := registry.Sweep(); len(snapshot.Sessions) != 1 || snapshot.Sessions[0].BytesAccepted != 2000 { + t.Fatalf("range accounting = %+v", snapshot.Sessions) + } + }) + + // A rejected request must create no LOGICAL activity. It does not produce an + // empty snapshot: the observer still saw the request and the error body, and + // that is deliberately retained in the unattributed counters. + for _, test := range []struct { + name string + sid string + closed bool + wantStatus int + }{ + {"unknown session", "sid-missing", false, http.StatusNotFound}, + {"closed session", "sid-closed", true, http.StatusGone}, + } { + t.Run(test.name+" creates no logical activity", func(t *testing.T) { + registry := telemetryRegistry(t) + deps := absPublicTrackDeps(t, "sid-real", "book-1", "42", body) + if test.closed { + closedAt := time.Now() + store := deps.PlaybackSessionStore.(*fakePlaybackSessionStore) + _ = store.InsertPlaybackSession(context.Background(), ABSPlaybackSession{ + ID: test.sid, UserID: "42", ContentID: "book-1", ClosedAt: &closedAt, + }) + } + server := absTelemetryServer(t, registry, deps) + got := getWithHeaders(t, server.Client(), http.MethodGet, + server.URL+"/public/session/"+test.sid+"/track/1", nil) + if got.status != test.wantStatus { + t.Fatalf("status = %d, want %d", got.status, test.wantStatus) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("rejected request created logical activity: %+v %+v", snapshot.Sessions, snapshot.Transfers) + } + if snapshot.UnattributedObservations == 0 { + t.Fatal("rejected request was not counted as unattributed") + } + }) + } +} + +// feedMediaStore resolves one media file by id for the RSS feed-file route. +type feedMediaStore struct { + noopMediaStore + file *models.MediaFile +} + +func (f *feedMediaStore) GetMediaFileByID(_ context.Context, id int) (*models.MediaFile, error) { + if f.file == nil || f.file.ID != id { + return nil, ErrNotFound + } + return f.file, nil +} + +type feedStore struct { + feed RSSFeed +} + +func (f *feedStore) GetFeedBySlug(_ context.Context, slug string) (RSSFeed, error) { + if f.feed.Slug != slug { + return RSSFeed{}, ErrNotFound + } + return f.feed, nil +} +func (f *feedStore) CreateFeed(context.Context, RSSFeed) error { return nil } +func (f *feedStore) CloseFeed(context.Context, string) error { return nil } +func (f *feedStore) GetFeed(context.Context, string) (RSSFeed, error) { + return RSSFeed{}, ErrNotFound +} +func (f *feedStore) ListUserFeeds(context.Context, string, string) ([]RSSFeed, error) { + return nil, nil +} + +// §4.2b: the RSS feed route has no authenticated caller — the slug is the +// capability — so the transfer must be attributed to the feed's owner. +func TestMountedABSRouterFeedFileResolvesOwner(t *testing.T) { + body := []byte(strings.Repeat("feed-audio", 200)) + path := filepath.Join(t.TempDir(), "feed.mp3") + if err := os.WriteFile(path, body, 0o644); err != nil { + t.Fatal(err) + } + registry := telemetryRegistry(t) + server := absTelemetryServer(t, registry, Dependencies{ + MediaStore: &feedMediaStore{file: &models.MediaFile{ID: 501, FilePath: path, ContentID: "book-9"}}, + RSSFeedStore: &feedStore{feed: RSSFeed{ID: "feed-1", UserID: "7", ProfileID: "profile-9", LibraryItemID: "book-9", Slug: "slug-9"}}, + }) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + + got := getWithHeaders(t, client, http.MethodGet, server.URL+"/feed/slug-9/file/501", nil) + if got.status != http.StatusOK || !bytes.Equal(got.body, body) { + t.Fatalf("GET = %d, %d bytes", got.status, len(got.body)) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 { + t.Fatalf("feed file created a logical session: %+v", snapshot.Sessions) + } + if len(snapshot.Transfers) != 1 { + t.Fatalf("transfers = %+v", snapshot.Transfers) + } + transfer := snapshot.Transfers[0] + if transfer.Subject != streamtelemetry.UserSubject(7) || transfer.ProfileID != "profile-9" { + t.Fatalf("feed transfer was not attributed to the feed owner: %+v", transfer) + } + if transfer.MediaFileID != 501 || transfer.BytesAccepted != int64(len(body)) { + t.Fatalf("transfer = %+v", transfer) + } +} + +// The family gate is the kill switch that makes enrolling a family sharing the +// API process reversible without losing all observation. +func TestMountedABSRouterFamilyGate(t *testing.T) { + body := []byte("\xff\xfb\x00\x00audio-bytes") + registry := telemetryRegistry(t, streamtelemetry.FamilyNative) + server := absTelemetryServer(t, registry, absPublicTrackDeps(t, "sid-gated", "book-1", "42", body)) + got := getWithHeaders(t, server.Client(), http.MethodGet, server.URL+"/public/session/sid-gated/track/1", nil) + if got.status != http.StatusOK || len(got.body) == 0 { + t.Fatalf("gated-out family broke serving: %d, %d bytes", got.status, len(got.body)) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || snapshot.UnattributedObservations != 0 { + t.Fatalf("gated-out family still observed: %+v", snapshot) + } +} diff --git a/internal/audiobooks/abs/testdata/media_routes.txt b/internal/audiobooks/abs/testdata/media_routes.txt index 48b7d7401..032b1024c 100644 --- a/internal/audiobooks/abs/testdata/media_routes.txt +++ b/internal/audiobooks/abs/testdata/media_routes.txt @@ -19,11 +19,11 @@ GET /abs/api/healthcheck non-media GET /abs/api/init non-media GET /abs/api/items/{id} non-media GET /abs/api/items/{id}/cover non-media -GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false true PATCH /abs/api/items/{id}/ebook/{fileid}/status non-media GET /abs/api/items/{id}/similar non-media -GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false -GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false true +GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false true POST /abs/api/items/{libraryItemId}/play non-media POST /abs/api/items/{libraryItemId}/play/{episodeId} non-media GET /abs/api/libraries non-media @@ -86,8 +86,8 @@ GET /abs/auth-settings non-media GET /abs/healthcheck non-media GET /abs/init non-media GET /abs/ping non-media -GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false -HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true true +HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true true GET /api/auth-settings non-media POST /api/auth/refresh non-media POST /api/authorize non-media @@ -108,11 +108,11 @@ GET /api/healthcheck non-media GET /api/init non-media GET /api/items/{id} non-media GET /api/items/{id}/cover non-media -GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false true PATCH /api/items/{id}/ebook/{fileid}/status non-media GET /api/items/{id}/similar non-media -GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false -GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false true +GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false true POST /api/items/{libraryItemId}/play non-media POST /api/items/{libraryItemId}/play/{episodeId} non-media GET /api/libraries non-media @@ -173,14 +173,14 @@ POST /api/session/{sid}/sync non-media POST /auth/refresh non-media GET /feed/{slug} non-media GET /feed/{slug}.xml non-media -GET /feed/{slug}/file/{ino} media transfer viewer_egress false false +GET /feed/{slug}/file/{ino} media transfer viewer_egress false true GET /healthcheck non-media GET /init non-media POST /login non-media POST /logout non-media GET /ping non-media -GET /public/session/{sid}/track/{idx} media playback viewer_egress true false -HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /public/session/{sid}/track/{idx} media playback viewer_egress true true +HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true true GET /status non-media # fixture 2 POST /abs/api/auth/logout non-media @@ -203,11 +203,11 @@ GET /abs/api/healthcheck non-media GET /abs/api/init non-media GET /abs/api/items/{id} non-media GET /abs/api/items/{id}/cover non-media -GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +GET /abs/api/items/{id}/ebook/{fileid} media transfer viewer_egress false true PATCH /abs/api/items/{id}/ebook/{fileid}/status non-media GET /abs/api/items/{id}/similar non-media -GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false -GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +GET /abs/api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false true +GET /abs/api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false true POST /abs/api/items/{libraryItemId}/play non-media POST /abs/api/items/{libraryItemId}/play/{episodeId} non-media GET /abs/api/libraries non-media @@ -270,8 +270,8 @@ GET /abs/auth-settings non-media GET /abs/healthcheck non-media GET /abs/init non-media GET /abs/ping non-media -GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false -HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /abs/public/session/{sid}/track/{idx} media playback viewer_egress true true +HEAD /abs/public/session/{sid}/track/{idx} media playback viewer_egress true true GET /api/auth-settings non-media POST /api/auth/refresh non-media POST /api/authorize non-media @@ -292,11 +292,11 @@ GET /api/healthcheck non-media GET /api/init non-media GET /api/items/{id} non-media GET /api/items/{id}/cover non-media -GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false false +GET /api/items/{id}/ebook/{fileid} media transfer viewer_egress false true PATCH /api/items/{id}/ebook/{fileid}/status non-media GET /api/items/{id}/similar non-media -GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false false -GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false false +GET /api/items/{libraryItemId}/file/{ino} media transfer viewer_egress false true +GET /api/items/{libraryItemId}/file/{ino}/download media transfer viewer_egress false true POST /api/items/{libraryItemId}/play non-media POST /api/items/{libraryItemId}/play/{episodeId} non-media GET /api/libraries non-media @@ -357,12 +357,12 @@ POST /api/session/{sid}/sync non-media POST /auth/refresh non-media GET /feed/{slug} non-media GET /feed/{slug}.xml non-media -GET /feed/{slug}/file/{ino} media transfer viewer_egress false false +GET /feed/{slug}/file/{ino} media transfer viewer_egress false true GET /healthcheck non-media GET /init non-media POST /login non-media POST /logout non-media GET /ping non-media -GET /public/session/{sid}/track/{idx} media playback viewer_egress true false -HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true false +GET /public/session/{sid}/track/{idx} media playback viewer_egress true true +HEAD /public/session/{sid}/track/{idx} media playback viewer_egress true true GET /status non-media diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 38e304ce9..d2d284b5f 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -626,6 +626,11 @@ func (h *PlaybackHandler) HandleCapabilitiesFull(w http.ResponseWriter, r *http. // HandleBitrateTest returns a small binary payload for clients that probe bandwidth. func (h *PlaybackHandler) HandleBitrateTest(w http.ResponseWriter, r *http.Request) { + // Jellyfin's authenticated bandwidth probe: transfer-observed, cap-exempt + // (§4.2 "classify but exempt"). It resolves no play session, so the subject + // is all the identity there is; an unauthenticated probe attaches nothing and + // its bytes fall into Unattributed*. + attachCompatTransfer(r.Context(), SessionFromContext(r.Context()), 0) w.Header().Set("Content-Type", "application/octet-stream") w.WriteHeader(http.StatusOK) _, _ = w.Write(make([]byte, 1024*1024)) diff --git a/internal/jellycompat/media_routes.go b/internal/jellycompat/media_routes.go index e83196c97..79fef735e 100644 --- a/internal/jellycompat/media_routes.go +++ b/internal/jellycompat/media_routes.go @@ -1,8 +1,11 @@ package jellycompat import ( + "net" "net/http" + "time" + "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -24,7 +27,62 @@ var jellycompatMediaRoutes = []streamtelemetry.MediaRoute{ func compatRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyJellycompat, Method: method, Pattern: pattern, Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "compat_play_session", - CapRelevant: capRelevant, Enrolled: false} + CapRelevant: capRelevant, Enrolled: true, Capture: compatCapture(pattern)} +} + +// compatCapture records the §2.2 request-time set for a Jellyfin client. +// +// Identity comes from the MediaBrowser authorization header, not X-Silo-Client*: +// Jellyfin clients never send silo's own headers, and firstMediaBrowserAuthorizationValue +// is the parser the negotiation path already uses for DeviceId +// (handlers_playback.go:764), so telemetry reads the same value the play session +// was keyed on rather than a second interpretation of the header. +// +// Byte accounting note: the compat router mounts httpstream.CompressExcept +// globally (router.go:47) and skipCompatMediaCompression (router.go:278) exempts +// the bulk media routes. Wrapping per route puts observedWriter BELOW the +// compression writer, so BytesAccepted equals wire bytes on the exempt routes and +// is PRE-compression on any compat media route still compressed (subtitles) — +// the same rule P0b documents for the native subtitle and font routes. That is +// deliberate; do not "fix" it by moving the wrapper. +func compatCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { + return func(r *http.Request) streamtelemetry.CaptureSet { + viewerIP := clientip.FromContext(r.Context()) + if viewerIP == "" { + viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) + if viewerIP == "" { + viewerIP = r.RemoteAddr + } + } + return streamtelemetry.CaptureSet{ + Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, + DeviceID: stripCompatNUL(firstMediaBrowserAuthorizationValue(r, "DeviceId")), + Client: streamtelemetry.ClientVariant{ + Name: stripCompatNUL(firstMediaBrowserAuthorizationValue(r, "Client")), + Version: stripCompatNUL(firstMediaBrowserAuthorizationValue(r, "Version")), + }, + UserAgent: r.UserAgent(), ReceivedAt: time.Now(), + } + } } func declareJellycompatMediaRoutes() { streamtelemetry.DeclareRoutes(jellycompatMediaRoutes...) } + +func compatMediaRoute(method, pattern string) streamtelemetry.MediaRoute { + for _, route := range jellycompatMediaRoutes { + if route.Method == method && route.Pattern == pattern { + return route + } + } + panic("undeclared jellycompat media route: " + method + " " + pattern) +} + +// observeCompat wraps a compat media handler. The panic in compatMediaRoute is +// what makes a typo fail the build through the route-manifest test instead of +// silently un-observing a route. +func observeCompat(registry *streamtelemetry.Registry, method, pattern string, handler http.HandlerFunc) http.HandlerFunc { + if registry == nil { + return handler + } + return registry.Observe(compatMediaRoute(method, pattern))(handler).ServeHTTP +} diff --git a/internal/jellycompat/media_routes_test.go b/internal/jellycompat/media_routes_test.go index 3c8407e0b..7d22e1c09 100644 --- a/internal/jellycompat/media_routes_test.go +++ b/internal/jellycompat/media_routes_test.go @@ -41,9 +41,16 @@ func TestMediaRouteManifest(t *testing.T) { if string(want) != actual { t.Fatalf("route manifest changed; inspect it and run go test . -update-route-manifest") } + // Every declared compat route is enrolled and carries a capture hook. A route + // that is declared but not enrolled, or enrolled with a nil Capture, would + // fall back to genericCapture and quietly lose the Jellyfin client identity + // compatCapture reads from the MediaBrowser authorization header. for _, route := range jellycompatMediaRoutes { - if route.Enrolled { - t.Fatalf("jellycompat route enrolled: %s %s", route.Method, route.Pattern) + if !route.Enrolled { + t.Fatalf("jellycompat route not enrolled: %s %s", route.Method, route.Pattern) + } + if route.Capture == nil { + t.Fatalf("jellycompat route has no capture hook: %s %s", route.Method, route.Pattern) } } } diff --git a/internal/jellycompat/router.go b/internal/jellycompat/router.go index d81e8c855..f137ba232 100644 --- a/internal/jellycompat/router.go +++ b/internal/jellycompat/router.go @@ -235,7 +235,7 @@ func NewRouter(deps Dependencies) chi.Router { r.Get("/Sessions", HandleSessions) r.Post("/Sessions/Capabilities", playbackHandler.HandleCapabilitiesFull) r.Post("/Sessions/Capabilities/Full", playbackHandler.HandleCapabilitiesFull) - r.Get("/Playback/BitrateTest", playbackHandler.HandleBitrateTest) + r.Get("/Playback/BitrateTest", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Playback/BitrateTest", playbackHandler.HandleBitrateTest)) r.Get("/Items/{id}/PlaybackInfo", playbackHandler.HandlePlaybackInfo) r.Post("/Items/{id}/PlaybackInfo", playbackHandler.HandlePlaybackInfo) r.Get("/Users/{userId}/Items/{id}/PlaybackInfo", playbackHandler.HandlePlaybackInfo) @@ -254,18 +254,18 @@ func NewRouter(deps Dependencies) chi.Router { // (e.g. libmpv) that don't forward auth headers or query parameters. r.Group(func(r chi.Router) { r.Use(PlaybackSessionAuth(deps.SessionStore, deps.PlaybackStore, adminAPIKeyAuth)) - r.Method(http.MethodHead, "/Items/{id}/Download", http.HandlerFunc(playbackHandler.HandleDownload)) - r.Get("/Items/{id}/Download", playbackHandler.HandleDownload) - r.Method(http.MethodHead, "/Videos/{id}/stream", http.HandlerFunc(playbackHandler.HandleVideoStream)) - r.Get("/Videos/{id}/stream", playbackHandler.HandleVideoStream) - r.Method(http.MethodHead, "/Videos/{id}/stream.{container}", http.HandlerFunc(playbackHandler.HandleVideoStream)) - r.Get("/Videos/{id}/stream.{container}", playbackHandler.HandleVideoStream) - r.Get("/Videos/{id}/master.m3u8", playbackHandler.HandleMasterManifest) - r.Get("/Videos/{id}/hls/{playlistId}/stream.m3u8", playbackHandler.HandleHLSManifest) - r.Get("/Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer}", playbackHandler.HandleHLSSegment) - r.Get("/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat}", playbackHandler.HandleSubtitleStream) + r.Method(http.MethodHead, "/Items/{id}/Download", observeCompat(deps.StreamTelemetry, http.MethodHead, "/Items/{id}/Download", playbackHandler.HandleDownload)) + r.Get("/Items/{id}/Download", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Items/{id}/Download", playbackHandler.HandleDownload)) + r.Method(http.MethodHead, "/Videos/{id}/stream", observeCompat(deps.StreamTelemetry, http.MethodHead, "/Videos/{id}/stream", playbackHandler.HandleVideoStream)) + r.Get("/Videos/{id}/stream", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{id}/stream", playbackHandler.HandleVideoStream)) + r.Method(http.MethodHead, "/Videos/{id}/stream.{container}", observeCompat(deps.StreamTelemetry, http.MethodHead, "/Videos/{id}/stream.{container}", playbackHandler.HandleVideoStream)) + r.Get("/Videos/{id}/stream.{container}", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{id}/stream.{container}", playbackHandler.HandleVideoStream)) + r.Get("/Videos/{id}/master.m3u8", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{id}/master.m3u8", playbackHandler.HandleMasterManifest)) + r.Get("/Videos/{id}/hls/{playlistId}/stream.m3u8", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{id}/hls/{playlistId}/stream.m3u8", playbackHandler.HandleHLSManifest)) + r.Get("/Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer}", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer}", playbackHandler.HandleHLSSegment)) + r.Get("/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat}", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat}", playbackHandler.HandleSubtitleStream)) // Infuse probes external subtitles with an extra numeric path component before stream.{format}. - r.Get("/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat}", playbackHandler.HandleSubtitleStream) + r.Get("/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat}", observeCompat(deps.StreamTelemetry, http.MethodGet, "/Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat}", playbackHandler.HandleSubtitleStream)) }) r.Method(http.MethodHead, "/System/Info/Public", http.HandlerFunc(systemHandler.HandlePublicInfo)) diff --git a/internal/jellycompat/server.go b/internal/jellycompat/server.go index 59b41754f..94788bea3 100644 --- a/internal/jellycompat/server.go +++ b/internal/jellycompat/server.go @@ -17,6 +17,7 @@ import ( "github.com/Silo-Server/silo-server/internal/recommendations" "github.com/Silo-Server/silo-server/internal/scantrigger" "github.com/Silo-Server/silo-server/internal/secret" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/userstore" "github.com/Silo-Server/silo-server/internal/watchstate" @@ -36,13 +37,16 @@ type Dependencies struct { DB *pgxpool.Pool SecretCipher *secret.Cipher // at-rest credential cipher (required when DB is set) ClientIPResolver *clientip.Resolver - Now func() time.Time - TokenGenerator func() string - SessionStore *SessionStore - IDCodec *ResourceIDCodec - ImageCache *ImageCache - DeviceProfiles *DeviceProfileStore - PlaybackStore CompatPlaybackStore + // StreamTelemetry is the local observation-only registry shared with the + // native API process. May be nil, which makes every media route unobserved. + StreamTelemetry *streamtelemetry.Registry + Now func() time.Time + TokenGenerator func() string + SessionStore *SessionStore + IDCodec *ResourceIDCodec + ImageCache *ImageCache + DeviceProfiles *DeviceProfileStore + PlaybackStore CompatPlaybackStore // RecipeNodeStore hands remote-transcode reconstruction recipes to the // control-plane recipe store (Redis) so a restarted transcode node can rebuild // a jellycompat session. Optional; nil disables the handoff. diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index dd03d7834..780f1be2d 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -76,6 +76,7 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusBadRequest, "BadRequest", "Media source is required") return } + attachCompatStream(r.Context(), session, playSession, source.FileID) method := "direct" if !staticRequest && !source.SupportsDirectPlay { @@ -186,6 +187,9 @@ func (h *PlaybackHandler) HandleDownload(w http.ResponseWriter, r *http.Request) writeError(w, http.StatusNotFound, "NotFound", "Media file not found") return } + // §4.2b: a download has a user but no stable playback session, so it is a + // Transfer rather than a logical session. + attachCompatTransfer(r.Context(), session, version.FileID) w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+url.PathEscape(filepath.Base(file.FilePath))) _ = playback.ServeDirectPlay(w, r, file.FilePath) @@ -218,6 +222,11 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re writeError(w, http.StatusBadRequest, "BadRequest", "Media source is required") return } + // Attach BEFORE ensureUpstreamPlayback below: this route can start a + // transcode before it writes a byte, which is the whole reason §4.2 enrolls + // manifest routes. A cut has to be able to act here, not after the side + // effect. See the boundary note in streamtelemetry.go. + attachCompatStream(r.Context(), session, playSession, source.FileID) var err error if h.NodePlanner != nil && h.JWTSecret != "" { @@ -333,6 +342,8 @@ func (h *PlaybackHandler) HandleHLSManifest(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusBadRequest, "BadRequest", "Media source is required") return } + // Before ensureTranscodeManifest, for the same reason as the master manifest. + attachCompatStream(r.Context(), session, playSession, source.FileID) // Ensure the transcode process is running. manifest, err := h.ensureTranscodeManifest(r.Context(), session, playSession.ID, *source) @@ -379,6 +390,11 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques writeError(w, http.StatusNotFound, "NotFound", "Playback session not found") return } + segmentSourceFileID := 0 + if source := firstMediaSource(playSession); source != nil { + segmentSourceFileID = source.FileID + } + attachCompatStream(r.Context(), session, playSession, segmentSourceFileID) name := chiURLParam(r, "segmentId") ext := chiURLParam(r, "segmentContainer") @@ -567,7 +583,7 @@ func (h *PlaybackHandler) HandleSubtitleStream(w http.ResponseWriter, r *http.Re return } - _, source, err := h.resolvePlaybackRoute(r, session, chiURLParam(r, "routeMediaSourceId"), chiURLParam(r, "routeMediaSourceId")) + playSession, source, err := h.resolvePlaybackRoute(r, session, chiURLParam(r, "routeMediaSourceId"), chiURLParam(r, "routeMediaSourceId")) if err != nil || source == nil { writeError(w, http.StatusNotFound, "NotFound", "Playback session not found") return @@ -582,6 +598,10 @@ func (h *PlaybackHandler) HandleSubtitleStream(w http.ResponseWriter, r *http.Re writeError(w, http.StatusNotFound, "NotFound", "Media file not found") return } + // Identity is fully known here. The later 400/404 branches for a bad index or + // a missing subtitle then record an outcome on a real session, which is + // correct: they are failures by an already-authorized principal. + attachCompatStream(r.Context(), session, playSession, source.FileID) routeIndex := chiURLParam(r, "routeIndex") trackIndex, parseErr := strconv.Atoi(routeIndex) diff --git a/internal/jellycompat/streamtelemetry.go b/internal/jellycompat/streamtelemetry.go new file mode 100644 index 000000000..e44894392 --- /dev/null +++ b/internal/jellycompat/streamtelemetry.go @@ -0,0 +1,71 @@ +package jellycompat + +import ( + "context" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +// The attachment boundary for every compat media handler is AUTHORIZATION +// SUCCESS — the point where the handler has established who is asking and which +// play session or item they are entitled to. Requests rejected before that point +// (401, 403, and the 404s that stand in for "that play session is not yours") +// create no logical activity. A failure AFTER that point — a missing file, an +// upstream 502, an invalid subtitle index — still creates activity, because it is +// real traffic by an authorized principal and logicalSession.outcomes is what +// records how it ended. +// +// This matters most on HandleMasterManifest (streams.go:197): it finishes +// authorization at the CompatToken and media-source checks, then starts real work +// via ensureUpstreamPlayback and can still 404 below that. Design §4.2 enrolls +// manifest routes precisely because "a killed session that reaches an unenrolled +// manifest route can reconstruct or start ffmpeg before the next segment is ever +// cut", so the attach must land BEFORE that side effect for P1's cut to be able +// to prevent it. + +// attachCompatStream attributes a compat playback observation to its play +// session. Identity comes from the authenticated compat session, never from the +// request: Session.StreamAppUserID is the numeric silo account id, so compat +// sessions land in the same subject space as native and proxy and a per-user +// total sums across families. +func attachCompatStream(ctx context.Context, session *Session, play *PlaybackSession, mediaFileID int) { + if session == nil { + return + } + attachment := streamtelemetry.Attachment{ + Subject: streamtelemetry.UserSubject(session.StreamAppUserID), + ProfileID: session.ProfileID, + MediaFileID: mediaFileID, + // The compat token is a session token, not a signed stream token whose + // iat this path verifies. Recording "verified" would be a lie. + TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone, + StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, + } + if play != nil { + attachment.SessionID = play.ID + attachment.PlayMethod = play.UpstreamPlayMethod + if !play.CreatedAt.IsZero() { + // P0a established the top-level compat CreatedAt as the source of + // truth for a compat session's start time. + attachment.StartedAt = play.CreatedAt + attachment.StartedAtSource = streamtelemetry.StartedAtSourceSession + } + } + streamtelemetry.Attach(ctx, attachment) +} + +// attachCompatTransfer attributes a download-class pour. Per §4.2b these carry a +// user but no stable playback session, so they never get a SessionID or a play +// method and never participate in per-session ratio rules. +func attachCompatTransfer(ctx context.Context, session *Session, mediaFileID int) { + if session == nil { + return + } + streamtelemetry.Attach(ctx, streamtelemetry.Attachment{ + Subject: streamtelemetry.UserSubject(session.StreamAppUserID), + ProfileID: session.ProfileID, + MediaFileID: mediaFileID, + StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, + TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone, + }) +} diff --git a/internal/jellycompat/streamtelemetry_bench_test.go b/internal/jellycompat/streamtelemetry_bench_test.go new file mode 100644 index 000000000..96c620fcf --- /dev/null +++ b/internal/jellycompat/streamtelemetry_bench_test.go @@ -0,0 +1,44 @@ +package jellycompat + +import ( + "io" + "net/http" + "testing" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +// BenchmarkCompatStreamTelemetry pairs the enabled and disabled sub-benchmarks in +// one run so the comparison is not across process invocations, mirroring +// internal/proxy/streamtelemetry_bench_test.go. Run with -count=5: a single run +// of either side is inside run-to-run variance. +func BenchmarkCompatStreamTelemetry(b *testing.B) { + b.Run("direct_stream/disabled", func(b *testing.B) { benchmarkCompatStream(b, false) }) + b.Run("direct_stream/enabled", func(b *testing.B) { benchmarkCompatStream(b, true) }) +} + +func benchmarkCompatStream(b *testing.B, enabled bool) { + var registry *streamtelemetry.Registry + if enabled { + registry = compatTelemetryRegistry(b) + } + fixture := newCompatTelemetryServer(b, registry) + url := fixture.server.URL + "/Videos/" + fixture.itemID + "/stream.mp4?static=true&api_key=" + compatTelemetryToken + client := fixture.client + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := client.Get(url) + if err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + b.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b.Fatalf("status = %d", resp.StatusCode) + } + } +} diff --git a/internal/jellycompat/streamtelemetry_test.go b/internal/jellycompat/streamtelemetry_test.go new file mode 100644 index 000000000..baf2e66f7 --- /dev/null +++ b/internal/jellycompat/streamtelemetry_test.go @@ -0,0 +1,310 @@ +package jellycompat + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +const compatTelemetryToken = "compat-telemetry-token" + +// compatTelemetryRegistry builds an enabled registry observing the jellycompat +// family. Every test that starts a registry must Stop it: the package-level now +// seam in streamtelemetry races leaked collector goroutines otherwise. +func compatTelemetryRegistry(t testing.TB, families ...streamtelemetry.Family) *streamtelemetry.Registry { + t.Helper() + cfg := streamtelemetry.DefaultConfig("compat-test") + cfg.Enabled = true + cfg.Retention = time.Minute + if len(families) == 0 { + families = []streamtelemetry.Family{streamtelemetry.FamilyJellycompat} + } + cfg.Families = make(map[streamtelemetry.Family]bool, len(families)) + for _, family := range families { + cfg.Families[family] = true + } + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + return registry +} + +type compatTelemetryFixture struct { + server *httptest.Server + client *http.Client + registry *streamtelemetry.Registry + itemID string + body string + store CompatPlaybackStore +} + +// newCompatTelemetryServer mounts the real compat router — global compression, +// PlaybackSessionAuth, request logging and all — behind a real socket. A +// handler-level test would bypass the middleware under test. +func newCompatTelemetryServer(t testing.TB, registry *streamtelemetry.Registry) compatTelemetryFixture { + t.Helper() + const body = "0123456789abcdefghijklmnopqrstuvwxyz" + filePath := filepath.Join(t.TempDir(), "movie.mp4") + if err := os.WriteFile(filePath, []byte(body), 0o644); err != nil { + t.Fatalf("write media: %v", err) + } + cfg, err := config.LoadFromDB(map[string]string{}) + if err != nil { + t.Fatalf("LoadFromDB: %v", err) + } + store := NewSessionStore(time.Hour, nil) + if err := store.Put(Session{Token: compatTelemetryToken, StreamAppUserID: 91, ProfileID: "profile-7"}); err != nil { + t.Fatalf("put compat session: %v", err) + } + codec := NewResourceIDCodec() + const contentID = "telemetry-movie" + detail := &upstreamItemDetail{ + ContentID: contentID, Type: "movie", Title: "Telemetry Movie", + Versions: []catalog.FileVersion{{ + FileID: 42, FilePath: filePath, Container: "mp4", + Duration: 3600, FileSize: int64(len(body)), AddedAt: time.Now(), + }}, + } + playbackStore := NewPlaybackSessionStore(time.Hour, nil) + router := NewRouter(Dependencies{ + Config: cfg, + SessionStore: store, + IDCodec: codec, + ContentService: &stubContentService{detail: detail}, + FileResolver: testCompatFileResolver{file: &models.MediaFile{ID: 42, FilePath: filePath}}, + SessionMgr: &testCompatSessionManager{}, + PlaybackStore: playbackStore, + StreamTelemetry: registry, + }) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + client := &http.Client{Transport: &http.Transport{DisableCompression: true}} + t.Cleanup(client.CloseIdleConnections) + return compatTelemetryFixture{ + server: server, client: client, registry: registry, + itemID: codec.EncodeStringID(EncodedIDItem, contentID), body: body, store: playbackStore, + } +} + +// compatResponse carries just what the assertions need. Returning the live +// *http.Response would leak an unclosed body past this helper. +type compatResponse struct { + status int + body string +} + +func (f compatTelemetryFixture) get(t *testing.T, method, url string, headers map[string]string) compatResponse { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatal(err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := f.client.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + buf := make([]byte, 0, 1024) + chunk := make([]byte, 512) + for { + n, readErr := resp.Body.Read(chunk) + buf = append(buf, chunk[:n]...) + if readErr != nil { + break + } + } + return compatResponse{status: resp.StatusCode, body: string(buf)} +} + +func TestMountedCompatRouterAttributesDirectStream(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + mediaURL := fixture.server.URL + "/Videos/" + fixture.itemID + "/stream.mp4?static=true&api_key=" + compatTelemetryToken + + got := fixture.get(t, http.MethodGet, mediaURL, map[string]string{ + "X-Emby-Authorization": `MediaBrowser Client="Jellyfin Web", Device="Chrome", DeviceId="device-abc", Version="10.11.6"`, + }) + if got.status != http.StatusOK || got.body != fixture.body { + t.Fatalf("GET = %d %q", got.status, got.body) + } + + // Sweep, not Snapshot: BytesAccepted is lastSweptBytes. + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + session := snapshot.Sessions[0] + // Session.StreamAppUserID is the numeric silo account id, so compat lands in + // the same subject space as native and proxy and a per-user total sums. + if session.Subject != streamtelemetry.UserSubject(91) || session.ProfileID != "profile-7" { + t.Fatalf("identity = %+v", session) + } + if session.SessionID == "" { + t.Fatal("compat session has no canonical play-session id") + } + if session.MediaFileID != 42 { + t.Fatalf("media file id = %d", session.MediaFileID) + } + if len(session.Routes) != 1 || session.Routes[0].Role != streamtelemetry.RoleViewerEgress { + t.Fatalf("routes = %+v", session.Routes) + } + if session.Routes[0].BytesAccepted != int64(len(fixture.body)) { + t.Fatalf("bytes = %d, want %d", session.Routes[0].BytesAccepted, len(fixture.body)) + } + // The compat token is a session token, not a signed stream token with an iat + // this path verifies. + if session.TokenIssuedAtSources[streamtelemetry.TokenIssuedAtSourceNone] == 0 { + t.Fatalf("token sources = %+v", session.TokenIssuedAtSources) + } + // Client identity comes from the MediaBrowser authorization header — the same + // parser the negotiation path uses for DeviceId — not X-Silo-Client*. + if len(session.DeviceIDs) != 1 || session.DeviceIDs[0] != "device-abc" { + t.Fatalf("device ids = %+v", session.DeviceIDs) + } + if len(session.ClientVariants) != 1 || session.ClientVariants[0].Name != "Jellyfin Web" || session.ClientVariants[0].Version != "10.11.6" { + t.Fatalf("client variants = %+v", session.ClientVariants) + } +} + +func TestMountedCompatRouterDownloadIsATransfer(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + url := fixture.server.URL + "/Items/" + fixture.itemID + "/Download?api_key=" + compatTelemetryToken + + got := fixture.get(t, http.MethodGet, url, nil) + if got.status != http.StatusOK || got.body != fixture.body { + t.Fatalf("GET = %d %q", got.status, got.body) + } + snapshot := registry.Sweep() + // §4.2b: a download has a user but no stable playback session. + if len(snapshot.Sessions) != 0 { + t.Fatalf("download created a logical session: %+v", snapshot.Sessions) + } + if len(snapshot.Transfers) != 1 { + t.Fatalf("transfers = %+v", snapshot.Transfers) + } + transfer := snapshot.Transfers[0] + if transfer.Subject != streamtelemetry.UserSubject(91) || transfer.MediaFileID != 42 { + t.Fatalf("transfer = %+v", transfer) + } + if transfer.BytesAccepted != int64(len(fixture.body)) { + t.Fatalf("transfer bytes = %d", transfer.BytesAccepted) + } +} + +func TestMountedCompatRouterBitrateTestIsACapExemptTransfer(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + + got := fixture.get(t, http.MethodGet, + fixture.server.URL+"/Playback/BitrateTest?api_key="+compatTelemetryToken, nil) + if got.status != http.StatusOK || len(got.body) != 1024*1024 { + t.Fatalf("bitrate probe = %d, %d bytes", got.status, len(got.body)) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 1 { + t.Fatalf("bitrate probe activity = %+v %+v", snapshot.Sessions, snapshot.Transfers) + } + if snapshot.Transfers[0].Subject != streamtelemetry.UserSubject(91) { + t.Fatalf("bitrate probe subject = %+v", snapshot.Transfers[0].Subject) + } + // §4.2 "classify but exempt": transfer-observed, never cap-relevant. + if snapshot.Transfers[0].BytesAccepted != 1024*1024 { + t.Fatalf("bitrate probe bytes = %d", snapshot.Transfers[0].BytesAccepted) + } +} + +// Rejected requests create no logical activity. Where the rejection happens +// decides whether they are observed at all, and both halves are worth pinning: +// PlaybackSessionAuth is mounted as middleware OUTSIDE the per-route observer +// (router.go:256), so a 401 never reaches the wrapper and costs nothing, while a +// request that authenticates and then fails inside the handler is observed and +// counted as unattributed. +func TestMountedCompatRouterRejectedRequestsCreateNoLogicalActivity(t *testing.T) { + t.Run("middleware 401 is never observed", func(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + + got := fixture.get(t, http.MethodGet, + fixture.server.URL+"/Videos/"+fixture.itemID+"/stream.mp4?static=true&api_key=not-a-token", nil) + if got.status != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", got.status) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("401 created logical activity: %+v %+v", snapshot.Sessions, snapshot.Transfers) + } + if snapshot.UnattributedObservations != 0 || snapshot.UnattributedBytes != 0 { + t.Fatalf("401 rejected by middleware should never reach the observer: %+v", snapshot) + } + }) + + // This is the case that proves the attach sits after the play-session + // resolution rather than at the top of the handler: the caller is + // authenticated, so the observer runs, but no session is ever attached. + t.Run("authenticated request with no resolvable play session", func(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + unknownItem := NewResourceIDCodec().EncodeStringID(EncodedIDItem, "no-such-item") + + got := fixture.get(t, http.MethodGet, + fixture.server.URL+"/Videos/"+unknownItem+"/stream.mp4?static=true&api_key="+compatTelemetryToken, nil) + if got.status == http.StatusOK { + t.Fatal("unknown item served successfully") + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 { + t.Fatalf("unresolvable play session created logical activity: %+v %+v", snapshot.Sessions, snapshot.Transfers) + } + if snapshot.UnattributedObservations == 0 { + t.Fatal("an observed but never-attached request was not counted as unattributed") + } + }) +} + +func TestMountedCompatRouterHEADCountsZeroBytes(t *testing.T) { + registry := compatTelemetryRegistry(t) + fixture := newCompatTelemetryServer(t, registry) + mediaURL := fixture.server.URL + "/Videos/" + fixture.itemID + "/stream.mp4?static=true&api_key=" + compatTelemetryToken + + got := fixture.get(t, http.MethodHead, mediaURL, nil) + if got.status != http.StatusOK || got.body != "" { + t.Fatalf("HEAD = %d %q", got.status, got.body) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + if snapshot.Sessions[0].BytesAccepted != 0 || snapshot.Sessions[0].RequestCount != 1 { + t.Fatalf("HEAD accounting = bytes %d requests %d", + snapshot.Sessions[0].BytesAccepted, snapshot.Sessions[0].RequestCount) + } +} + +// The kill switch that makes enrolling a family which shares the API process +// with native reversible without losing all observation. +func TestMountedCompatRouterFamilyGate(t *testing.T) { + registry := compatTelemetryRegistry(t, streamtelemetry.FamilyNative) + fixture := newCompatTelemetryServer(t, registry) + mediaURL := fixture.server.URL + "/Videos/" + fixture.itemID + "/stream.mp4?static=true&api_key=" + compatTelemetryToken + + got := fixture.get(t, http.MethodGet, mediaURL, nil) + if got.status != http.StatusOK || got.body != fixture.body { + t.Fatalf("gated-out family broke serving: %d %q", got.status, got.body) + } + snapshot := registry.Sweep() + if len(snapshot.Sessions) != 0 || len(snapshot.Transfers) != 0 || snapshot.UnattributedObservations != 0 { + t.Fatalf("gated-out family still observed: %+v", snapshot) + } +} diff --git a/internal/jellycompat/testdata/media_routes.txt b/internal/jellycompat/testdata/media_routes.txt index 7ce1bd7c5..adf3aa54a 100644 --- a/internal/jellycompat/testdata/media_routes.txt +++ b/internal/jellycompat/testdata/media_routes.txt @@ -16,8 +16,8 @@ GET /Items/Filters2 non-media GET /Items/Latest non-media GET /Items/Suggestions non-media GET /Items/{id} non-media -GET /Items/{id}/Download media transfer viewer_egress false false -HEAD /Items/{id}/Download media transfer viewer_egress false false +GET /Items/{id}/Download media transfer viewer_egress false true +HEAD /Items/{id}/Download media transfer viewer_egress false true GET /Items/{id}/Images/{imageType} non-media GET /Items/{id}/Images/{imageType}/{index} non-media GET /Items/{id}/Intros non-media @@ -33,7 +33,7 @@ GET /MediaSegments/{id} non-media GET /Movies/Recommendations non-media GET /Movies/{id}/Similar non-media GET /Persons non-media -GET /Playback/BitrateTest media transfer viewer_egress false false +GET /Playback/BitrateTest media transfer viewer_egress false true GET /QuickConnect/Enabled non-media GET /Search/Hints non-media GET /Sessions non-media @@ -91,15 +91,15 @@ DELETE /Users/{userId}/PlayedItems/{itemId} non-media POST /Users/{userId}/PlayedItems/{itemId} non-media GET /Users/{userId}/Views non-media DELETE /Videos/ActiveEncodings non-media -GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true false -GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true false -GET /Videos/{id}/master.m3u8 media manifest viewer_egress true false -GET /Videos/{id}/stream media playback viewer_egress true false -HEAD /Videos/{id}/stream media playback viewer_egress true false -GET /Videos/{id}/stream.{container} media playback viewer_egress true false -HEAD /Videos/{id}/stream.{container} media playback viewer_egress true false -GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true false -GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true true +GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true true +GET /Videos/{id}/master.m3u8 media manifest viewer_egress true true +GET /Videos/{id}/stream media playback viewer_egress true true +HEAD /Videos/{id}/stream media playback viewer_egress true true +GET /Videos/{id}/stream.{container} media playback viewer_egress true true +HEAD /Videos/{id}/stream.{container} media playback viewer_egress true true +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true true +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true true GET /socket non-media GET /web non-media CONNECT /web/* non-media @@ -129,8 +129,8 @@ GET /Items/Filters2 non-media GET /Items/Latest non-media GET /Items/Suggestions non-media GET /Items/{id} non-media -GET /Items/{id}/Download media transfer viewer_egress false false -HEAD /Items/{id}/Download media transfer viewer_egress false false +GET /Items/{id}/Download media transfer viewer_egress false true +HEAD /Items/{id}/Download media transfer viewer_egress false true GET /Items/{id}/Images/{imageType} non-media GET /Items/{id}/Images/{imageType}/{index} non-media GET /Items/{id}/Intros non-media @@ -146,7 +146,7 @@ GET /MediaSegments/{id} non-media GET /Movies/Recommendations non-media GET /Movies/{id}/Similar non-media GET /Persons non-media -GET /Playback/BitrateTest media transfer viewer_egress false false +GET /Playback/BitrateTest media transfer viewer_egress false true GET /QuickConnect/Enabled non-media GET /Search/Hints non-media GET /Sessions non-media @@ -204,15 +204,15 @@ DELETE /Users/{userId}/PlayedItems/{itemId} non-media POST /Users/{userId}/PlayedItems/{itemId} non-media GET /Users/{userId}/Views non-media DELETE /Videos/ActiveEncodings non-media -GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true false -GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true false -GET /Videos/{id}/master.m3u8 media manifest viewer_egress true false -GET /Videos/{id}/stream media playback viewer_egress true false -HEAD /Videos/{id}/stream media playback viewer_egress true false -GET /Videos/{id}/stream.{container} media playback viewer_egress true false -HEAD /Videos/{id}/stream.{container} media playback viewer_egress true false -GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true false -GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true false +GET /Videos/{id}/hls/{playlistId}/stream.m3u8 media manifest viewer_egress true true +GET /Videos/{id}/hls/{playlistId}/{segmentId}.{segmentContainer} media playback viewer_egress true true +GET /Videos/{id}/master.m3u8 media manifest viewer_egress true true +GET /Videos/{id}/stream media playback viewer_egress true true +HEAD /Videos/{id}/stream media playback viewer_egress true true +GET /Videos/{id}/stream.{container} media playback viewer_egress true true +HEAD /Videos/{id}/stream.{container} media playback viewer_egress true true +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/stream.{routeFormat} media playback viewer_egress true true +GET /Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeDeliveryIndex}/stream.{routeFormat} media playback viewer_egress true true GET /socket non-media GET /web non-media CONNECT /web/* non-media diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index b26bfdead..edd2f1ed6 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -3,6 +3,7 @@ package streamtelemetry import ( "log/slog" "os" + "sort" "strconv" "strings" "time" @@ -11,6 +12,7 @@ import ( const ( enabledEnv = "SILO_STREAM_TELEMETRY_ENABLED" + familiesEnv = "SILO_STREAM_TELEMETRY_FAMILIES" sweepIntervalEnv = "SILO_STREAM_TELEMETRY_SWEEP_INTERVAL" retentionEnv = "SILO_STREAM_TELEMETRY_RETENTION" maxSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_SESSIONS" @@ -26,12 +28,31 @@ const ( maxMergedTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_MERGED_TRANSFERS" ) +// defaultObservedFamilies is the set observed when SILO_STREAM_TELEMETRY_FAMILIES +// is unset. It is deliberately NOT "every declared family": jellycompat and ABS +// share the API process with native, so defaulting them on would widen +// instrumentation across a live byte path on upgrade alone, which is exactly what +// §6's one-family-at-a-time rollout exists to prevent. Proxy and transcode node +// are separate processes, so their own SILO_STREAM_TELEMETRY_ENABLED already gates +// them and they stay in the default set. Name a family in the variable to observe +// it; move it in here once it has run in production, and delete this set when all +// five have. +var defaultObservedFamilies = map[Family]bool{ + FamilyNative: true, + FamilyProxy: true, + FamilyTranscodeNode: true, +} + type Config struct { Enabled bool NodeID string PublisherID string PublisherEpoch int64 Distributed bool + // Families narrows which route families are observed. Empty means + // defaultObservedFamilies. It is a kill switch as much as a rollout control: + // one misbehaving family can be dropped without losing all observation. + Families map[Family]bool SweepInterval time.Duration Retention time.Duration @@ -138,6 +159,13 @@ func ConfigFromEnv(nodeID string) Config { parseDistributedPositive(maxPublishersEnv, &cfg.MaxPublishers) parseDistributedPositive(maxMergedSessionsEnv, &cfg.MaxMergedSessions) parseDistributedPositive(maxMergedTransfersEnv, &cfg.MaxMergedTransfers) + if value := strings.TrimSpace(os.Getenv(familiesEnv)); value != "" { + if families, ok := parseFamilies(value); ok { + cfg.Families = families + } else { + coreInvalid = append(coreInvalid, familiesEnv) + } + } if value := os.Getenv(keyPrefixEnv); value != "" { if strings.TrimSpace(value) == "" || strings.IndexFunc(value, unicode.IsSpace) >= 0 { distributedInvalid = append(distributedInvalid, keyPrefixEnv) @@ -173,6 +201,53 @@ func ConfigFromEnv(nodeID string) Config { return cfg } +// ObservesFamily reports whether routes in this family are wrapped. It is read +// once per route at mount time, never on the hot path. +func (c Config) ObservesFamily(family Family) bool { + if len(c.Families) == 0 { + return defaultObservedFamilies[family] + } + return c.Families[family] +} + +// ObservedFamilies lists the observed families in a stable order, for the +// startup log that makes the resolved set visible. +func (c Config) ObservedFamilies() []string { + set := c.Families + if len(set) == 0 { + set = defaultObservedFamilies + } + names := make([]string, 0, len(set)) + for family, observed := range set { + if observed { + names = append(names, string(family)) + } + } + sort.Strings(names) + return names +} + +// parseFamilies decodes the comma-separated family list. An unrecognized name is +// a core-invalid setting rather than a distributed-only one: a typo that silently +// observed nothing would be worse than no telemetry at all. +func parseFamilies(value string) (map[Family]bool, bool) { + families := make(map[Family]bool) + for _, entry := range strings.Split(value, ",") { + name := strings.ToLower(strings.TrimSpace(entry)) + if name == "" { + continue + } + family := Family(name) + switch family { + case FamilyNative, FamilyJellycompat, FamilyProxy, FamilyABS, FamilyTranscodeNode: + families[family] = true + default: + return nil, false + } + } + return families, true +} + func envEnabled(value string) bool { value = strings.TrimSpace(strings.ToLower(value)) return value == "1" || value == "true" || value == "yes" || value == "on" diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index d03ca6c49..f9eef064b 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -128,7 +128,73 @@ func TestConfigFromEnvValidation(t *testing.T) { func clearConfigEnv(t *testing.T) { t.Helper() for _, name := range []string{enabledEnv, sweepIntervalEnv, retentionEnv, maxSessionsEnv, maxTransfersEnv, maxObservationsEnv, - distributedEnv, freshnessEnv, membershipTTLEnv, keyPrefixEnv, fullResyncEveryEnv, maxPublishersEnv, maxMergedSessionsEnv, maxMergedTransfersEnv} { + distributedEnv, freshnessEnv, membershipTTLEnv, keyPrefixEnv, fullResyncEveryEnv, maxPublishersEnv, maxMergedSessionsEnv, maxMergedTransfersEnv, + familiesEnv} { t.Setenv(name, "") } } + +// The family gate is what makes a shared-process family (jellycompat, ABS) safe +// to enroll: it is both the staged-rollout control §6 asks for and a kill switch +// for one misbehaving family that keeps the rest observing. +func TestConfigFamilyGate(t *testing.T) { + t.Run("unset observes the shipped set only", func(t *testing.T) { + clearConfigEnv(t) + cfg := ConfigFromEnv("node") + if len(cfg.Families) != 0 { + t.Fatalf("families = %+v, want unset", cfg.Families) + } + for _, family := range []Family{FamilyNative, FamilyProxy, FamilyTranscodeNode} { + if !cfg.ObservesFamily(family) { + t.Fatalf("%s not observed by default", family) + } + } + // Widening the default would instrument two more live byte paths in the + // API process on upgrade alone. That has to be an explicit opt-in. + for _, family := range []Family{FamilyJellycompat, FamilyABS} { + if cfg.ObservesFamily(family) { + t.Fatalf("%s observed by default", family) + } + } + }) + t.Run("explicit list narrows and widens", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(familiesEnv, " Native , jellycompat ,, ABS ") + cfg := ConfigFromEnv("node") + if !cfg.Enabled { + t.Fatalf("config = %+v", cfg) + } + for _, family := range []Family{FamilyNative, FamilyJellycompat, FamilyABS} { + if !cfg.ObservesFamily(family) { + t.Fatalf("%s not observed", family) + } + } + for _, family := range []Family{FamilyProxy, FamilyTranscodeNode} { + if cfg.ObservesFamily(family) { + t.Fatalf("%s observed despite an explicit list omitting it", family) + } + } + if got := cfg.ObservedFamilies(); len(got) != 3 || got[0] != "abs" || got[1] != "jellycompat" || got[2] != "native" { + t.Fatalf("observed families = %v", got) + } + }) + t.Run("unknown family disables telemetry", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(familiesEnv, "native,not_a_family") + cfg := ConfigFromEnv("node") + if cfg.Enabled { + t.Fatal("a typo in the family list must disable telemetry rather than silently observe nothing") + } + }) + t.Run("only whitespace falls back to the default set", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(familiesEnv, " , ") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.ObservesFamily(FamilyNative) || cfg.ObservesFamily(FamilyABS) { + t.Fatalf("config = %+v", cfg) + } + }) +} diff --git a/internal/streamtelemetry/writer.go b/internal/streamtelemetry/writer.go index db41c5f97..7b74213c7 100644 --- a/internal/streamtelemetry/writer.go +++ b/internal/streamtelemetry/writer.go @@ -14,7 +14,10 @@ const OutcomeUnknown httpstream.StreamOutcome = "unknown" func (r *Registry) Observe(route MediaRoute) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { - if r == nil || !r.cfg.Enabled || !route.Enrolled { + // Evaluated once at mount time — Observe returns the middleware, and the + // closure below is what runs per request — so the family gate costs + // nothing on the hot path. + if r == nil || !r.cfg.Enabled || !route.Enrolled || !r.cfg.ObservesFamily(route.Family) { return next } return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { diff --git a/internal/streamtelemetry/writer_test.go b/internal/streamtelemetry/writer_test.go index fea5ff7e3..637ec46e2 100644 --- a/internal/streamtelemetry/writer_test.go +++ b/internal/streamtelemetry/writer_test.go @@ -2,6 +2,8 @@ package streamtelemetry import ( "bufio" + "context" + "fmt" "io" "net" "net/http" @@ -141,3 +143,45 @@ func TestObservedWriterClassifiesTransportFailuresOnRelease(t *testing.T) { }) } } + +// A route in an unobserved family must get the handler back unchanged — not a +// wrapper that decides per request — so the gate costs nothing on the hot path +// and cannot half-observe. +func TestObserveSkipsUnobservedFamily(t *testing.T) { + cfg := testConfig() + cfg.Families = map[Family]bool{FamilyNative: true} + registry := NewRegistry(cfg, NewLocalStore(), nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + + body := []byte("audiobook-bytes") + serve := func(family Family) Snapshot { + route := MediaRoute{Family: family, Method: http.MethodGet, Pattern: "/gated", + Class: ClassPlayback, Role: RoleViewerEgress, CapRelevant: true, Enrolled: true} + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), Attachment{Subject: UserSubject(7), SessionID: "gated-" + string(family), + StartedAt: time.Unix(100, 0), StartedAtSource: StartedAtSourceSession}) + _, _ = w.Write(body) + }) + wrapped := registry.Observe(route)(inner) + if family != FamilyNative { + // An unobserved family must be handed back the very handler it passed in. + if fmt.Sprintf("%p", wrapped) != fmt.Sprintf("%p", http.Handler(inner)) { + t.Fatalf("%s was wrapped despite being outside the observed set", family) + } + } + recorder := httptest.NewRecorder() + wrapped.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/gated", nil)) + if recorder.Body.String() != string(body) { + t.Fatalf("%s body = %q", family, recorder.Body.String()) + } + return registry.Sweep() + } + + if snapshot := serve(FamilyABS); len(snapshot.Sessions) != 0 { + t.Fatalf("gated-out family produced sessions: %+v", snapshot.Sessions) + } + snapshot := serve(FamilyNative) + if len(snapshot.Sessions) != 1 || snapshot.Sessions[0].BytesAccepted != int64(len(body)) { + t.Fatalf("observed family sessions = %+v", snapshot.Sessions) + } +} From b6c9a1c79f4004be941c16abb3f516db254834ec Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:52:51 +0000 Subject: [PATCH 09/44] feat(streamtelemetry): add the P0d admin parity projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serves the merged global view beside both legacy live-session projections and the diff between them, at GET /api/v1/admin/stream-telemetry/parity. Read-only: no /api/v1 response changes, no migration, no Postgres or Redis write. This change compares; it does not cut over. §6 puts the repoint after parity is demonstrated, and there is nothing to demonstrate it with yet — telemetry is off in every deployment. The admin session payload is also a join rather than a swap: playbackSessionRow carries ~50 display fields (title, poster, season/episode, position, decisions, source codecs) that telemetry is explicitly not canonical for. Repointing belongs to the separate retirement change, which this endpoint exists to give evidence for. Closes the open item left by P0c. BuildGlobalView measured 347 ms at 50 000 sessions, so ViewCache serves it with bounded staleness. It is read-driven rather than a ticker: a ticker would pay the full rebuild on every server forever whether or not an admin is looking, while a TTL pays only when someone asks and single-flights however many readers arrive together. A reader holding a cached value never queues behind a rebuild. A failed refresh keeps the last good view and reports the error — going blind is worse than being visibly stale — and before the first build the view is reported unavailable rather than empty, which a consumer would read as "nothing is streaming". CompareLiveSessions is pure — no clock, no store, no logger — so every rule is tested in CI without Postgres or Redis, the same property that makes BuildGlobalView testable. Only a field both sides carry can disagree: a legacy row with no profile id is a gap in that projection, not a contradiction, and counting it as one would bury the real mismatches. Start times compare with one second of tolerance, because two independent writers cannot be expected to agree to the nanosecond and nothing downstream needs them to. Every list is capped with an explicit dropped count. The projection renders a play method only when the merged view has exactly one — §2.5 leaves the scalar unset when publishers disagree, and picking one here would reintroduce the arbitrary choice the merge refuses to make — and takes the node from the viewer-edge publisher only, so a relayed session does not claim a node that never served a viewer. The view's completeness travels with the diff. A degraded view is missing sessions by construction, so a report built on one is evidence of blindness rather than disagreement; that is the distinction P0c built the flag for. A source that cannot be read reports itself unavailable with a reason instead of being omitted, which would read as "nothing to compare against". Planned, implemented and reviewed with Claude (Opus 5). Unlike the two enrolment commits before it, this one had NO cross-model adversarial review — the Codex side of the relay hit its usage limit partway through the session. The project's own gates were run in full. Part of #135 --- cmd/silo/main.go | 13 + .../api/handlers/stream_telemetry_parity.go | 211 ++++++++++++++++ .../handlers/stream_telemetry_parity_test.go | 135 ++++++++++ internal/api/router.go | 137 ++++++----- internal/api/testdata/media_routes.txt | 1 + internal/nodesessions/reader.go | 69 ++++++ internal/streamtelemetry/config.go | 17 +- internal/streamtelemetry/config_test.go | 2 +- internal/streamtelemetry/global.go | 8 +- internal/streamtelemetry/parity.go | 230 ++++++++++++++++++ internal/streamtelemetry/parity_test.go | 182 ++++++++++++++ internal/streamtelemetry/store.go | 2 + internal/streamtelemetry/viewcache.go | 184 ++++++++++++++ internal/streamtelemetry/viewcache_test.go | 172 +++++++++++++ 14 files changed, 1291 insertions(+), 72 deletions(-) create mode 100644 internal/api/handlers/stream_telemetry_parity.go create mode 100644 internal/api/handlers/stream_telemetry_parity_test.go create mode 100644 internal/nodesessions/reader.go create mode 100644 internal/streamtelemetry/parity.go create mode 100644 internal/streamtelemetry/parity_test.go create mode 100644 internal/streamtelemetry/viewcache.go create mode 100644 internal/streamtelemetry/viewcache_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index aee3b0a04..3c0125383 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -197,6 +197,16 @@ func newStreamTelemetryRegistry(ctx context.Context, nodeID string, redisClient return streamtelemetry.NewRegistry(streamTelemetryConfig, store, slog.Default()) } +// newStreamTelemetryViewCache builds the bounded-staleness cache the admin +// parity endpoint reads. It shares one cached view across every reader so the +// merged rebuild is paid at most once per TTL, not once per request. +func newStreamTelemetryViewCache(registry *streamtelemetry.Registry, nodeID string) *streamtelemetry.ViewCache { + if registry == nil { + return nil + } + return streamtelemetry.NewViewCache(registry, streamtelemetry.ConfigFromEnv(nodeID).ViewTTL, slog.Default()) +} + func resolvePluginCacheDir() string { if v := strings.TrimSpace(os.Getenv("SILO_PLUGIN_CACHE_DIR")); v != "" { return v @@ -720,6 +730,7 @@ func main() { appCtx, appCancel := context.WithCancel(ctx) defer appCancel() var streamTelemetryRegistry *streamtelemetry.Registry + var streamTelemetryViewCache *streamtelemetry.ViewCache restartReqCh := make(chan struct{}, 1) var restartRequested atomic.Bool @@ -887,6 +898,7 @@ func main() { if mode == "" || mode == "integrated" || mode == "api" { streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, apiRedisClient) streamTelemetryRegistry.Start(appCtx) + streamTelemetryViewCache = newStreamTelemetryViewCache(streamTelemetryRegistry, nodeID) } // Assigned below once the trusted-proxy config is seeded; captured by the @@ -906,6 +918,7 @@ func main() { RedisBootstrapAvailable: redisBootstrapAvailable, AppContext: appCtx, StreamTelemetry: streamTelemetryRegistry, + StreamTelemetryViewCache: streamTelemetryViewCache, DB: pool, SecretCipher: dataCipher, EventBus: eventBus, diff --git a/internal/api/handlers/stream_telemetry_parity.go b/internal/api/handlers/stream_telemetry_parity.go new file mode 100644 index 000000000..ff2851139 --- /dev/null +++ b/internal/api/handlers/stream_telemetry_parity.go @@ -0,0 +1,211 @@ +package handlers + +import ( + "context" + "net/http" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +// parityScanLimit bounds each legacy read. It sits above the merged-session cap +// so a normal fleet is never truncated, while a runaway table still cannot blow +// the handler up. +const parityScanLimit = 60_000 + +// StreamTelemetryParityHandler serves P0d's admin parity projection: the merged +// stream-telemetry view beside both legacy live-session projections, and the +// diff between them. +// +// It compares; it does not cut over. The design puts the repoint after parity is +// demonstrated, and legacy retirement is its own project — see the design note +// for why the admin session payload is a join rather than a swap. +type StreamTelemetryParityHandler struct { + Registry *streamtelemetry.Registry + ViewCache *streamtelemetry.ViewCache + Pool *pgxpool.Pool + Redis *redis.Client +} + +type parityViewResponse struct { + Available bool `json:"available"` + BuiltAt string `json:"built_at,omitempty"` + AgeMS int64 `json:"age_ms"` + Stale bool `json:"stale"` + BuildTookMS int64 `json:"build_took_ms"` + Refreshes int64 `json:"refreshes"` + Failures int64 `json:"failures"` + LastError string `json:"last_error,omitempty"` + Complete bool `json:"complete"` + IncompleteReasons []string `json:"incomplete_reasons"` + MissingPublishers []string `json:"missing_publishers"` + ClockSkewSuspected bool `json:"clock_skew_suspected"` + Publishers []string `json:"publishers"` + SessionCount int `json:"session_count"` + TransferCount int `json:"transfer_count"` +} + +type paritySourceResponse struct { + Source string `json:"source"` + Available bool `json:"available"` + Error string `json:"error,omitempty"` + Notes []string `json:"notes,omitempty"` + Report *streamtelemetry.ParityReport `json:"report,omitempty"` +} + +type parityResponse struct { + Enabled bool `json:"enabled"` + Reason string `json:"reason,omitempty"` + View parityViewResponse `json:"view"` + Sources []paritySourceResponse `json:"sources"` +} + +// HandleGetStreamTelemetryParity handles +// GET /api/v1/admin/stream-telemetry/parity. +func (h *StreamTelemetryParityHandler) HandleGetStreamTelemetryParity(w http.ResponseWriter, r *http.Request) { + if h == nil || h.Registry == nil || !h.Registry.Enabled() { + // The honest answer is "there is nothing to compare", not an empty + // report that reads as agreement. + writeJSON(w, http.StatusOK, parityResponse{ + Reason: "stream telemetry is disabled on this process", + Sources: []paritySourceResponse{}, + }) + return + } + + ctx := r.Context() + view, status := h.ViewCache.View(ctx) + response := parityResponse{Enabled: true, View: describeView(view, status), Sources: []paritySourceResponse{}} + if !status.Available { + response.Reason = "the global view has not been built yet" + writeJSON(w, http.StatusOK, response) + return + } + + telemetry := streamtelemetry.LiveSessionsFromGlobalView(view) + response.Sources = append(response.Sources, + h.comparePostgres(ctx, telemetry), + h.compareNodeSessions(ctx, telemetry), + ) + writeJSON(w, http.StatusOK, response) +} + +// describeView surfaces the completeness flag alongside the diff on purpose. A +// degraded view is missing sessions by construction, so a parity report built on +// one is not evidence of disagreement — it is evidence of blindness, and P0c +// built the flag precisely so the two cannot be confused. +func describeView(view streamtelemetry.GlobalMonitoringView, status streamtelemetry.ViewCacheStatus) parityViewResponse { + response := parityViewResponse{ + Available: status.Available, AgeMS: status.Age.Milliseconds(), Stale: status.Stale, + BuildTookMS: status.BuildTook.Milliseconds(), Refreshes: status.Refreshes, + Failures: status.Failures, LastError: status.LastError, + Complete: view.Complete, ClockSkewSuspected: view.ClockSkewSuspected, + IncompleteReasons: view.IncompleteReasons, MissingPublishers: []string{}, + Publishers: []string{}, SessionCount: len(view.Sessions), TransferCount: len(view.Transfers), + } + if response.IncompleteReasons == nil { + response.IncompleteReasons = []string{} + } + if !view.BuiltAt.IsZero() { + response.BuiltAt = view.BuiltAt.UTC().Format(time.RFC3339Nano) + } + for _, publisher := range view.MissingPublishers { + response.MissingPublishers = append(response.MissingPublishers, publisher.PublisherID) + } + for _, publisher := range view.Publishers { + response.Publishers = append(response.Publishers, publisher.PublisherID+"="+string(publisher.State)) + } + return response +} + +func (h *StreamTelemetryParityHandler) comparePostgres(ctx context.Context, telemetry []streamtelemetry.LiveSession) paritySourceResponse { + const source = "playback_sessions_sync" + if h.Pool == nil { + return paritySourceResponse{Source: source, Error: "database not configured"} + } + // Only the parity columns: the enriched admin query joins media, series and + // episode metadata telemetry cannot express, so reading it here would cost + // far more and compare nothing extra. + rows, err := h.Pool.Query(ctx, ` + SELECT session_id, user_id, COALESCE(profile_id, ''), COALESCE(media_file_id, 0), + COALESCE(play_method, ''), COALESCE(reporting_node, ''), started_at + FROM playback_sessions_sync + LIMIT $1`, parityScanLimit) + if err != nil { + return paritySourceResponse{Source: source, Error: err.Error()} + } + defer rows.Close() + + legacy := make([]streamtelemetry.LiveSession, 0) + for rows.Next() { + var ( + sessionID, profileID, playMethod, node string + userID, mediaFileID int + startedAt time.Time + ) + if err := rows.Scan(&sessionID, &userID, &profileID, &mediaFileID, &playMethod, &node, &startedAt); err != nil { + return paritySourceResponse{Source: source, Error: err.Error()} + } + session := streamtelemetry.LiveSession{ + SessionID: sessionID, ProfileID: profileID, MediaFileID: mediaFileID, + PlayMethod: playMethod, Node: node, StartedAt: startedAt, + } + if userID > 0 { + session.Subject = streamtelemetry.UserSubject(userID) + } + legacy = append(legacy, session) + } + if err := rows.Err(); err != nil { + return paritySourceResponse{Source: source, Error: err.Error()} + } + + report := streamtelemetry.CompareLiveSessions(source, telemetry, legacy, streamtelemetry.DefaultParityLimit) + return paritySourceResponse{Source: source, Available: true, Report: &report} +} + +func (h *StreamTelemetryParityHandler) compareNodeSessions(ctx context.Context, telemetry []streamtelemetry.LiveSession) paritySourceResponse { + const source = "node_sessions_redis" + if h.Redis == nil { + return paritySourceResponse{Source: source, Error: "redis not configured"} + } + result, err := nodesessions.ListAll(ctx, h.Redis, parityScanLimit) + if err != nil { + return paritySourceResponse{Source: source, Error: err.Error()} + } + + legacy := make([]streamtelemetry.LiveSession, 0, len(result.Sessions)) + for _, info := range result.Sessions { + session := streamtelemetry.LiveSession{ + SessionID: info.SessionID, ProfileID: info.ProfileID, + MediaFileID: info.MediaFileID, Node: info.NodeName, + } + if info.AuthUserID > 0 { + session.Subject = streamtelemetry.UserSubject(info.AuthUserID) + } + // The node record carries both a formatted timestamp and the immutable + // nanosecond stamp P0a added. Prefer the nanos: the formatted value is + // second-resolution and is what a mixed-version node may not have. + if info.StartedAtUnixNano > 0 { + session.StartedAt = time.Unix(0, info.StartedAtUnixNano) + } else if parsed, parseErr := time.Parse(time.RFC3339, info.StartedAt); parseErr == nil { + session.StartedAt = parsed + } + legacy = append(legacy, session) + } + + response := paritySourceResponse{Source: source, Available: true} + if result.Undecodable > 0 { + response.Notes = append(response.Notes, "undecodable records skipped") + } + if result.Truncated { + // Never let a capped read pass as a complete one. + response.Notes = append(response.Notes, "scan truncated at the record limit; the report is partial") + } + report := streamtelemetry.CompareLiveSessions(source, telemetry, legacy, streamtelemetry.DefaultParityLimit) + response.Report = &report + return response +} diff --git a/internal/api/handlers/stream_telemetry_parity_test.go b/internal/api/handlers/stream_telemetry_parity_test.go new file mode 100644 index 000000000..098f531bb --- /dev/null +++ b/internal/api/handlers/stream_telemetry_parity_test.go @@ -0,0 +1,135 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtelemetry" +) + +func decodeParity(t *testing.T, recorder *httptest.ResponseRecorder) parityResponse { + t.Helper() + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var response parityResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode: %v; body = %s", err, recorder.Body.String()) + } + return response +} + +func serveParity(t *testing.T, handler *StreamTelemetryParityHandler) parityResponse { + t.Helper() + recorder := httptest.NewRecorder() + handler.HandleGetStreamTelemetryParity(recorder, httptest.NewRequest(http.MethodGet, "/admin/stream-telemetry/parity", nil)) + return decodeParity(t, recorder) +} + +// With telemetry off the endpoint must say so rather than return an empty +// report, which a reader would take as "the two projections agree". +func TestStreamTelemetryParityReportsDisabled(t *testing.T) { + t.Run("nil handler fields", func(t *testing.T) { + response := serveParity(t, &StreamTelemetryParityHandler{}) + if response.Enabled || response.Reason == "" { + t.Fatalf("response = %+v", response) + } + if len(response.Sources) != 0 { + t.Fatalf("disabled telemetry still produced comparisons: %+v", response.Sources) + } + }) + t.Run("disabled registry", func(t *testing.T) { + cfg := streamtelemetry.DefaultConfig("parity-test") + registry := streamtelemetry.NewRegistry(cfg, streamtelemetry.NewLocalStore(), nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + response := serveParity(t, &StreamTelemetryParityHandler{Registry: registry}) + if response.Enabled { + t.Fatalf("disabled registry reported enabled: %+v", response) + } + }) +} + +func enabledParityHandler(t *testing.T) *StreamTelemetryParityHandler { + t.Helper() + cfg := streamtelemetry.DefaultConfig("parity-test") + cfg.Enabled = true + cfg.Retention = time.Minute + store := streamtelemetry.NewLocalStore() + registry := streamtelemetry.NewRegistry(cfg, store, nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + // Publish one snapshot so the merged view has a publisher and is buildable. + if err := store.Publish(context.Background(), registry.Snapshot()); err != nil { + t.Fatal(err) + } + return &StreamTelemetryParityHandler{ + Registry: registry, + ViewCache: streamtelemetry.NewViewCache(registry, time.Minute, nil), + } +} + +// A source that cannot be read must report itself unavailable with a reason. +// Omitting it would read as "there was nothing to compare against". +func TestStreamTelemetryParityReportsUnreadableSources(t *testing.T) { + response := serveParity(t, enabledParityHandler(t)) + if !response.Enabled { + t.Fatalf("response = %+v", response) + } + if len(response.Sources) != 2 { + t.Fatalf("sources = %+v", response.Sources) + } + wantSources := map[string]bool{"playback_sessions_sync": false, "node_sessions_redis": false} + for _, source := range response.Sources { + if _, known := wantSources[source.Source]; !known { + t.Fatalf("unexpected source %q", source.Source) + } + wantSources[source.Source] = true + if source.Available { + t.Fatalf("%s reported available with no backing store", source.Source) + } + if source.Error == "" { + t.Fatalf("%s reported unavailable with no reason", source.Source) + } + if source.Report != nil { + t.Fatalf("%s produced a report it could not have computed", source.Source) + } + } + for name, seen := range wantSources { + if !seen { + t.Fatalf("source %q was omitted from the response entirely", name) + } + } +} + +// The completeness flag has to travel with the diff: a degraded view is missing +// sessions by construction, so a parity report built on one is evidence of +// blindness rather than disagreement. +func TestStreamTelemetryParitySurfacesViewCompleteness(t *testing.T) { + response := serveParity(t, enabledParityHandler(t)) + if !response.View.Available { + t.Fatalf("view = %+v", response.View) + } + if response.View.IncompleteReasons == nil { + t.Fatal("incomplete_reasons must be present, not null, so a client can read it unconditionally") + } + if response.View.MissingPublishers == nil || response.View.Publishers == nil { + t.Fatalf("publisher lists must be present: %+v", response.View) + } + if response.View.Refreshes != 1 { + t.Fatalf("refreshes = %d, want exactly one build for one request", response.View.Refreshes) + } +} + +// The endpoint must not rebuild the merged view per request: it measured ~347 ms +// at the 50 000-session cap. +func TestStreamTelemetryParityReusesTheCachedView(t *testing.T) { + handler := enabledParityHandler(t) + for i := 0; i < 4; i++ { + if response := serveParity(t, handler); response.View.Refreshes != 1 { + t.Fatalf("request %d rebuilt the view: refreshes = %d", i, response.View.Refreshes) + } + } +} diff --git a/internal/api/router.go b/internal/api/router.go index 3db42277c..a9fba3439 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -98,68 +98,71 @@ type Dependencies struct { DB *pgxpool.Pool SecretCipher *secret.Cipher // at-rest credential cipher (required when DB is set) FrontendFS fs.FS - S3Public *s3client.Client // public assets bucket client (may be nil) - S3Private *s3client.Client // private internal bucket client (may be nil) - S3UserDB *s3client.Client // user-db bucket client (may be nil) - BrandingService *branding.Service // white-label branding (nil when DB unavailable) - FolderRepo *catalog.FolderRepository // media folder repository (may be nil) - FileRepo *scanner.FileRepository // media file repository (may be nil) - Scanner *scanner.Scanner // scanner instance (may be nil) - LibraryIngester *libraryingest.Executor // shared library ingest executor (may be nil) - ProbeEnsurer handlers.PlaybackProbeEnsurer // on-demand probe repair for playback/detail (may be nil) - UserStoreProvider userstore.UserStoreProvider // user store provider (may be nil) - SessionMgr *playback.SessionManager // playback session manager (may be nil) - StreamTelemetry *streamtelemetry.Registry // local observation-only stream telemetry (may be nil) - SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) - StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) - MovieMatchQueueRepo *metadata.MovieMatchQueueRepository - SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository - Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) - NodeRepo *nodepool.Repository // stream node repository (may be nil) - ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) - TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) - NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) - SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger - EventBus cache.EventBus - AdminStatsProvider handlers.AdminStatsSource - Recommender recommendations.Recommender // nil when disabled - RecWorker *recommendations.Worker // nil when disabled - CatalogSearchVectorizer catalog.CatalogSearchQueryVectorizer - RatingsRepo *catalog.RatingsRepo - PersonRepo *catalog.PersonRepository - PersonRefreshQueue handlers.PersonRefreshQueue - PersonRefresher handlers.PersonRefresher - RateLimitMW *ratelimit.Middleware - ClientIPResolver *clientip.Resolver - NodeID string - LogStreamHub *logstream.Hub - RealtimeHub *notifications.Hub - Notifications *notifications.System // user-facing release notifications (may be nil) - PolicySystem *policy.System // policy engine lifecycle (may be nil) - EventsHub *evt.Hub - ScanRegistry *evt.ScanRegistry - LibraryScanQueue *scanqueue.Service - ActivityLogWriter activitylog.Writer - ActivityLogRepo *activitylog.Repo - OpsLogRepo *opslog.Repo - FFmpegLogSink playback.FFmpegLogSink - RedisClient *redis.Client // for session listing (may be nil) - TaskManager *taskmanager.TaskManager // task manager (may be nil) - ArtifactManager *downloads.ArtifactManager // download prepare-to-file pipeline (may be nil) - AdminJobCancelRegistry *adminjob.CancelRegistry - IntroRepository *intromarkers.Repository - IntroAnalyzer *intromarkers.Analyzer - MarkerRegistry *markers.Registry - MarkerResolver markers.ExternalIDResolver - MarkerProviderConfig *markers.ProviderConfigStore - MarkerContributionStore *markers.ContributionStore - MarkerContributionService *markers.ContributionService - WatchProviderService handlers.WatchProviderService - WatchCompletionObserver watchstate.CompletionObserver - PluginService *plugins.Service - PluginHTTPProxy *plugins.HTTPProxy - PluginUserConfig *plugins.UserConfigStore - AuthProviders []auth.RegisteredProvider + S3Public *s3client.Client // public assets bucket client (may be nil) + S3Private *s3client.Client // private internal bucket client (may be nil) + S3UserDB *s3client.Client // user-db bucket client (may be nil) + BrandingService *branding.Service // white-label branding (nil when DB unavailable) + FolderRepo *catalog.FolderRepository // media folder repository (may be nil) + FileRepo *scanner.FileRepository // media file repository (may be nil) + Scanner *scanner.Scanner // scanner instance (may be nil) + LibraryIngester *libraryingest.Executor // shared library ingest executor (may be nil) + ProbeEnsurer handlers.PlaybackProbeEnsurer // on-demand probe repair for playback/detail (may be nil) + UserStoreProvider userstore.UserStoreProvider // user store provider (may be nil) + SessionMgr *playback.SessionManager // playback session manager (may be nil) + StreamTelemetry *streamtelemetry.Registry // local observation-only stream telemetry (may be nil) + // StreamTelemetryViewCache serves the merged global view with bounded + // staleness so the admin parity endpoint never rebuilds it per request. + StreamTelemetryViewCache *streamtelemetry.ViewCache + SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil) + StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil) + MovieMatchQueueRepo *metadata.MovieMatchQueueRepository + SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository + Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil) + NodeRepo *nodepool.Repository // stream node repository (may be nil) + ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil) + TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil) + NodePlanner *nodepool.Planner // group/cap-aware node selection (may be nil) + SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger + EventBus cache.EventBus + AdminStatsProvider handlers.AdminStatsSource + Recommender recommendations.Recommender // nil when disabled + RecWorker *recommendations.Worker // nil when disabled + CatalogSearchVectorizer catalog.CatalogSearchQueryVectorizer + RatingsRepo *catalog.RatingsRepo + PersonRepo *catalog.PersonRepository + PersonRefreshQueue handlers.PersonRefreshQueue + PersonRefresher handlers.PersonRefresher + RateLimitMW *ratelimit.Middleware + ClientIPResolver *clientip.Resolver + NodeID string + LogStreamHub *logstream.Hub + RealtimeHub *notifications.Hub + Notifications *notifications.System // user-facing release notifications (may be nil) + PolicySystem *policy.System // policy engine lifecycle (may be nil) + EventsHub *evt.Hub + ScanRegistry *evt.ScanRegistry + LibraryScanQueue *scanqueue.Service + ActivityLogWriter activitylog.Writer + ActivityLogRepo *activitylog.Repo + OpsLogRepo *opslog.Repo + FFmpegLogSink playback.FFmpegLogSink + RedisClient *redis.Client // for session listing (may be nil) + TaskManager *taskmanager.TaskManager // task manager (may be nil) + ArtifactManager *downloads.ArtifactManager // download prepare-to-file pipeline (may be nil) + AdminJobCancelRegistry *adminjob.CancelRegistry + IntroRepository *intromarkers.Repository + IntroAnalyzer *intromarkers.Analyzer + MarkerRegistry *markers.Registry + MarkerResolver markers.ExternalIDResolver + MarkerProviderConfig *markers.ProviderConfigStore + MarkerContributionStore *markers.ContributionStore + MarkerContributionService *markers.ContributionService + WatchProviderService handlers.WatchProviderService + WatchCompletionObserver watchstate.CompletionObserver + PluginService *plugins.Service + PluginHTTPProxy *plugins.HTTPProxy + PluginUserConfig *plugins.UserConfigStore + AuthProviders []auth.RegisteredProvider // PublicURL is the externally-reachable origin (scheme + host) for this // silo instance. Used to build redirect_uri values handed to OAuth // IdPs. Empty disables the /oauth/{install_id}/{init,callback} routes. @@ -2822,6 +2825,16 @@ func NewRouter(deps Dependencies) chi.Router { } r.Get("/sessions", adminHandler.HandleListSessions) + // P0d parity projection: the merged telemetry view beside + // both legacy live-session projections and their diff. It + // compares only — the repoint is the separate retirement + // change, which this endpoint exists to give evidence for. + r.Get("/stream-telemetry/parity", (&handlers.StreamTelemetryParityHandler{ + Registry: deps.StreamTelemetry, + ViewCache: deps.StreamTelemetryViewCache, + Pool: deps.DB, + Redis: deps.RedisClient, + }).HandleGetStreamTelemetryParity) r.Get("/sessions/capabilities", adminHandler.HandleGetSessionsCapabilities) r.Get("/playback-history", adminHandler.HandleListPlaybackHistory) r.Get("/unmatched", adminHandler.HandleListUnmatched) diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt index 26be420fb..5ca72f78d 100644 --- a/internal/api/testdata/media_routes.txt +++ b/internal/api/testdata/media_routes.txt @@ -149,6 +149,7 @@ GET /api/v1/admin/settings/sensitive-status non-media GET /api/v1/admin/settings/{key} non-media PUT /api/v1/admin/settings/{key} non-media GET /api/v1/admin/stats non-media +GET /api/v1/admin/stream-telemetry/parity non-media GET /api/v1/admin/subtitle-providers/ non-media PUT /api/v1/admin/subtitle-providers/{provider}/ non-media POST /api/v1/admin/subtitle-providers/{provider}/test non-media diff --git a/internal/nodesessions/reader.go b/internal/nodesessions/reader.go new file mode 100644 index 000000000..73ac582bf --- /dev/null +++ b/internal/nodesessions/reader.go @@ -0,0 +1,69 @@ +package nodesessions + +import ( + "context" + "encoding/json" + + "github.com/redis/go-redis/v9" +) + +// ListResult is the outcome of reading every node's live session records. +type ListResult struct { + Sessions []SessionInfo + // Undecodable counts keys whose value did not parse as a SessionInfo. A + // mixed-version fleet can legitimately write a record this binary cannot + // read; surfacing the count keeps that visible instead of quietly shrinking + // the result. + Undecodable int + // Truncated reports that the scan stopped at limit. A caller that ignores + // this would read a partial answer as a complete one. + Truncated bool +} + +// ListAll reads the live session records every proxy and transcode node +// publishes under silo:sessions:{nodeHash}:{sessionID}. +// +// It exists here rather than in a handler because this package owns the key +// format and the record shape. Note that internal/api/handlers/nodes.go +// deliberately keeps its own scan: that endpoint passes the stored JSON through +// opaquely so an older node's extra fields survive the round trip, which a +// decode-and-re-encode reader would silently drop. +func ListAll(ctx context.Context, rdb *redis.Client, limit int) (ListResult, error) { + var result ListResult + if rdb == nil { + return result, nil + } + if limit <= 0 { + limit = 50_000 + } + + var cursor uint64 + for { + keys, next, err := rdb.Scan(ctx, cursor, keyPrefix+"*", 200).Result() + if err != nil { + return result, err + } + for _, key := range keys { + if len(result.Sessions) >= limit { + result.Truncated = true + return result, nil + } + value, err := rdb.Get(ctx, key).Result() + if err != nil { + // A key that expired between the SCAN and the GET is normal: + // these records carry a 60s TTL. + continue + } + var info SessionInfo + if err := json.Unmarshal([]byte(value), &info); err != nil || info.SessionID == "" { + result.Undecodable++ + continue + } + result.Sessions = append(result.Sessions, info) + } + cursor = next + if cursor == 0 { + return result, nil + } + } +} diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index edd2f1ed6..730529758 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -26,6 +26,7 @@ const ( maxPublishersEnv = "SILO_STREAM_TELEMETRY_MAX_PUBLISHERS" maxMergedSessionsEnv = "SILO_STREAM_TELEMETRY_MAX_MERGED_SESSIONS" maxMergedTransfersEnv = "SILO_STREAM_TELEMETRY_MAX_MERGED_TRANSFERS" + viewTTLEnv = "SILO_STREAM_TELEMETRY_VIEW_TTL" ) // defaultObservedFamilies is the set observed when SILO_STREAM_TELEMETRY_FAMILIES @@ -54,11 +55,15 @@ type Config struct { // one misbehaving family can be dropped without losing all observation. Families map[Family]bool - SweepInterval time.Duration - Retention time.Duration - Freshness time.Duration - MembershipTTL time.Duration - KeyPrefix string + SweepInterval time.Duration + Retention time.Duration + Freshness time.Duration + MembershipTTL time.Duration + KeyPrefix string + // ViewTTL bounds how stale a served merged view may be. It gates a rebuild + // that measured ~347 ms at the 50 000-session cap, so it is a cost control + // rather than a freshness preference. + ViewTTL time.Duration FullResyncEvery int MaxPublishers int MaxMergedSessions int @@ -82,6 +87,7 @@ func DefaultConfig(nodeID string) Config { return Config{ NodeID: nodeID, SweepInterval: time.Second, Retention: 5 * time.Minute, Freshness: 5 * time.Second, MembershipTTL: time.Minute, KeyPrefix: "silo:stelem", + ViewTTL: DefaultViewTTL, FullResyncEvery: 60, MaxPublishers: 256, MaxMergedSessions: 50_000, MaxMergedTransfers: 50_000, MaxSessions: 10_000, MaxTransfers: 10_000, MaxObservations: 50_000, MaxObservationsPerSession: 64, MaxViewerIPsPerSession: 32, @@ -155,6 +161,7 @@ func ConfigFromEnv(nodeID string) Config { parsePositive(maxObservationsEnv, &cfg.MaxObservations) parseDistributedDuration(freshnessEnv, &cfg.Freshness) parseDistributedDuration(membershipTTLEnv, &cfg.MembershipTTL) + parseDistributedDuration(viewTTLEnv, &cfg.ViewTTL) parseDistributedPositive(fullResyncEveryEnv, &cfg.FullResyncEvery) parseDistributedPositive(maxPublishersEnv, &cfg.MaxPublishers) parseDistributedPositive(maxMergedSessionsEnv, &cfg.MaxMergedSessions) diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index f9eef064b..6a3b30af5 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -129,7 +129,7 @@ func clearConfigEnv(t *testing.T) { t.Helper() for _, name := range []string{enabledEnv, sweepIntervalEnv, retentionEnv, maxSessionsEnv, maxTransfersEnv, maxObservationsEnv, distributedEnv, freshnessEnv, membershipTTLEnv, keyPrefixEnv, fullResyncEveryEnv, maxPublishersEnv, maxMergedSessionsEnv, maxMergedTransfersEnv, - familiesEnv} { + familiesEnv, viewTTLEnv} { t.Setenv(name, "") } } diff --git a/internal/streamtelemetry/global.go b/internal/streamtelemetry/global.go index 03b692c73..a416aa10a 100644 --- a/internal/streamtelemetry/global.go +++ b/internal/streamtelemetry/global.go @@ -469,9 +469,9 @@ func mergeSession(id string, contributions []sessionContribution, params ViewPar if winningRank == 1 || len(winningTimes) > 1 { result.StartedAtDegraded = true } - applyIdentity(&result, "subject", subjectValues) + applyIdentity(&result, identityFieldSubject, subjectValues) applyIdentity(&result, identityFieldProfileID, profileValues) - applyIdentity(&result, "media_file_id", mediaValues) + applyIdentity(&result, identityFieldMediaFileID, mediaValues) result.ViewerIPs, result.ViewerIPsOverflowed = cappedStrings(viewerIPs, params.MaxViewerIPsPerSession, result.ViewerIPsOverflowed) result.DeviceIDs, result.DeviceIDsOverflowed = cappedStrings(deviceIDs, params.MaxDeviceIDsPerSession, result.DeviceIDsOverflowed) result.UserAgents, result.UserAgentsOverflowed = cappedStrings(userAgents, params.MaxUserAgentsPerSession, result.UserAgentsOverflowed) @@ -529,12 +529,12 @@ func applyIdentity(result *GlobalSessionView, field string, values map[string][] if len(values) == 1 { for value := range values { switch field { - case "subject": + case identityFieldSubject: parts := strings.SplitN(value, "\x00", 2) result.Subject = Subject{Kind: SubjectKind(parts[0]), ID: parts[1]} case identityFieldProfileID: result.ProfileID = value - case "media_file_id": + case identityFieldMediaFileID: result.MediaFileID, _ = strconv.Atoi(value) } } diff --git a/internal/streamtelemetry/parity.go b/internal/streamtelemetry/parity.go new file mode 100644 index 000000000..1d1676879 --- /dev/null +++ b/internal/streamtelemetry/parity.go @@ -0,0 +1,230 @@ +package streamtelemetry + +import ( + "sort" + "strconv" + "time" +) + +// DefaultParityLimit bounds every list in a ParityReport. An admin endpoint that +// returned 50 000 session ids would be its own outage. +const DefaultParityLimit = 50 + +// parityStartedAtTolerance is how far apart two projections' start times may be +// before it counts as a disagreement. +// +// Postgres stores a timestamp and telemetry keeps nanoseconds, and the two are +// written by independent processes, so sub-second skew is normal rather than a +// parity failure. It is also below the resolution any consumer acts on: the +// design's victim ordering is (startedAtUnixNano, sessionID), which only has to +// be a total order, not agree across stores. +const parityStartedAtTolerance = time.Second + +// LiveSession is one live streaming session reduced to the fields every +// projection can express. +// +// It is deliberately small. Telemetry has no media title, poster or playback +// position, and the legacy projections have no byte counts or viewer-edge +// publisher. Comparing a field only one side can express would manufacture +// mismatches and bury the real ones. +type LiveSession struct { + SessionID string + Subject Subject + ProfileID string + MediaFileID int + PlayMethod string + Node string + StartedAt time.Time +} + +// LiveSessionsFromGlobalView projects the merged view onto the comparable core, +// sorted by session id. +// +// Node comes from the session's viewer-edge publisher only: a session a node +// merely relayed is not a session that node served a viewer from, and claiming +// otherwise would disagree with the legacy projections for the wrong reason. +func LiveSessionsFromGlobalView(view GlobalMonitoringView) []LiveSession { + sessions := make([]LiveSession, 0, len(view.Sessions)) + for _, session := range view.Sessions { + live := LiveSession{ + SessionID: session.SessionID, Subject: session.Subject, ProfileID: session.ProfileID, + MediaFileID: session.MediaFileID, StartedAt: session.StartedAt, + } + if len(session.PlayMethods) == 1 { + // A merged scalar play method is deliberately absent when publishers + // disagree (§2.5); a single unioned value is the only unambiguous one. + live.PlayMethod = session.PlayMethods[0] + } + for _, publisher := range session.ViewerEdgePublishers { + if publisher.NodeID != "" { + live.Node = publisher.NodeID + break + } + } + sessions = append(sessions, live) + } + sort.Slice(sessions, func(i, j int) bool { return sessions[i].SessionID < sessions[j].SessionID }) + return sessions +} + +// ParityMismatch is one field two projections disagree about for a session they +// both know. +type ParityMismatch struct { + SessionID string `json:"session_id"` + Field string `json:"field"` + Telemetry string `json:"telemetry"` + Legacy string `json:"legacy"` +} + +// ParityReport is the diff between the telemetry projection and one legacy +// projection. Every list is capped, with an explicit count of what the cap +// dropped: silent truncation would read as "covered everything". +type ParityReport struct { + Source string `json:"source"` + TelemetryCount int `json:"telemetry_count"` + LegacyCount int `json:"legacy_count"` + InBoth int `json:"in_both"` + // Agrees means the two projections describe the same set of sessions and + // disagree on no field they both express. It deliberately does NOT account + // for FieldsAbsent: a field only one side carries is a different question + // from a field they contradict each other about, and folding the two + // together would make this flag permanently false — legacy rows carry no + // value for several of these — and therefore useless. Read FieldsAbsent as + // well before treating agreement as clearance to cut over. + Agrees bool `json:"agrees"` + TelemetryOnly []string `json:"telemetry_only"` + TelemetryMore int `json:"telemetry_only_truncated"` + LegacyOnly []string `json:"legacy_only"` + LegacyMore int `json:"legacy_only_truncated"` + Mismatches []ParityMismatch `json:"mismatches"` + MismatchesMore int `json:"mismatches_truncated"` + // FieldsAbsent counts, per field, sessions both projections know where one + // side carries no value at all. That is a gap in a projection, not a + // disagreement between them, and counting it as a mismatch would bury the + // real ones. + FieldsAbsent map[string]int `json:"fields_absent,omitempty"` +} + +// CompareLiveSessions diffs two projections of the same live activity, keyed by +// session id. It is a pure function — no clock, no store, no logger — so every +// rule below is unit-testable in CI on a machine with neither Postgres nor +// Redis, which is the same reason BuildGlobalView is pure. +func CompareLiveSessions(source string, telemetry, legacy []LiveSession, limit int) ParityReport { + if limit <= 0 { + limit = DefaultParityLimit + } + report := ParityReport{ + Source: source, TelemetryCount: len(telemetry), LegacyCount: len(legacy), + TelemetryOnly: []string{}, LegacyOnly: []string{}, Mismatches: []ParityMismatch{}, + FieldsAbsent: map[string]int{}, + } + + legacyByID := make(map[string]LiveSession, len(legacy)) + for _, session := range legacy { + legacyByID[session.SessionID] = session + } + telemetryByID := make(map[string]LiveSession, len(telemetry)) + for _, session := range telemetry { + telemetryByID[session.SessionID] = session + } + + telemetryOnly := make([]string, 0) + mismatches := make([]ParityMismatch, 0) + for _, session := range telemetry { + counterpart, ok := legacyByID[session.SessionID] + if !ok { + telemetryOnly = append(telemetryOnly, session.SessionID) + continue + } + report.InBoth++ + mismatches = append(mismatches, compareSession(session, counterpart, report.FieldsAbsent)...) + } + legacyOnly := make([]string, 0) + for _, session := range legacy { + if _, ok := telemetryByID[session.SessionID]; !ok { + legacyOnly = append(legacyOnly, session.SessionID) + } + } + + sort.Strings(telemetryOnly) + sort.Strings(legacyOnly) + sort.Slice(mismatches, func(i, j int) bool { + if mismatches[i].SessionID == mismatches[j].SessionID { + return mismatches[i].Field < mismatches[j].Field + } + return mismatches[i].SessionID < mismatches[j].SessionID + }) + + report.Agrees = len(telemetryOnly) == 0 && len(legacyOnly) == 0 && len(mismatches) == 0 + report.TelemetryOnly, report.TelemetryMore = capStrings(telemetryOnly, limit) + report.LegacyOnly, report.LegacyMore = capStrings(legacyOnly, limit) + if len(mismatches) > limit { + report.MismatchesMore = len(mismatches) - limit + mismatches = mismatches[:limit] + } + report.Mismatches = mismatches + if len(report.FieldsAbsent) == 0 { + report.FieldsAbsent = nil + } + return report +} + +func compareSession(telemetry, legacy LiveSession, absent map[string]int) []ParityMismatch { + mismatches := make([]ParityMismatch, 0, 4) + add := func(field, left, right string) { + // Only a field both sides carry can disagree. + if left == "" || right == "" { + if left != right { + absent[field]++ + } + return + } + if left != right { + mismatches = append(mismatches, ParityMismatch{ + SessionID: telemetry.SessionID, Field: field, Telemetry: left, Legacy: right, + }) + } + } + add(identityFieldSubject, subjectKey(telemetry.Subject), subjectKey(legacy.Subject)) + add(identityFieldProfileID, telemetry.ProfileID, legacy.ProfileID) + add(identityFieldMediaFileID, positiveInt(telemetry.MediaFileID), positiveInt(legacy.MediaFileID)) + add("play_method", telemetry.PlayMethod, legacy.PlayMethod) + add("node", telemetry.Node, legacy.Node) + + switch { + case telemetry.StartedAt.IsZero() || legacy.StartedAt.IsZero(): + if telemetry.StartedAt.IsZero() != legacy.StartedAt.IsZero() { + absent["started_at"]++ + } + default: + if delta := telemetry.StartedAt.Sub(legacy.StartedAt); delta > parityStartedAtTolerance || delta < -parityStartedAtTolerance { + mismatches = append(mismatches, ParityMismatch{ + SessionID: telemetry.SessionID, Field: "started_at", + Telemetry: telemetry.StartedAt.UTC().Format(time.RFC3339Nano), + Legacy: legacy.StartedAt.UTC().Format(time.RFC3339Nano), + }) + } + } + return mismatches +} + +func subjectKey(subject Subject) string { + if subject.Kind == "" || subject.ID == "" { + return "" + } + return string(subject.Kind) + ":" + subject.ID +} + +func positiveInt(value int) string { + if value <= 0 { + return "" + } + return strconv.Itoa(value) +} + +func capStrings(values []string, limit int) ([]string, int) { + if len(values) <= limit { + return values, 0 + } + return values[:limit], len(values) - limit +} diff --git a/internal/streamtelemetry/parity_test.go b/internal/streamtelemetry/parity_test.go new file mode 100644 index 000000000..cee398b7b --- /dev/null +++ b/internal/streamtelemetry/parity_test.go @@ -0,0 +1,182 @@ +package streamtelemetry + +import ( + "testing" + "time" +) + +func liveSession(id string, mutate ...func(*LiveSession)) LiveSession { + session := LiveSession{ + SessionID: id, Subject: UserSubject(7), ProfileID: "profile-1", + MediaFileID: 42, PlayMethod: "direct", Node: "node-a", + StartedAt: time.Unix(1_700_000_000, 0), + } + for _, apply := range mutate { + apply(&session) + } + return session +} + +func TestCompareLiveSessionsAgreesOnIdenticalSets(t *testing.T) { + sessions := []LiveSession{liveSession("a"), liveSession("b")} + report := CompareLiveSessions("legacy", sessions, sessions, 0) + if !report.Agrees { + t.Fatalf("identical sets disagreed: %+v", report) + } + if report.InBoth != 2 || report.TelemetryCount != 2 || report.LegacyCount != 2 { + t.Fatalf("counts = %+v", report) + } + if len(report.Mismatches) != 0 || len(report.TelemetryOnly) != 0 || len(report.LegacyOnly) != 0 { + t.Fatalf("report = %+v", report) + } +} + +func TestCompareLiveSessionsReportsOnlySides(t *testing.T) { + telemetry := []LiveSession{liveSession("a"), liveSession("only-telemetry")} + legacy := []LiveSession{liveSession("a"), liveSession("only-legacy")} + report := CompareLiveSessions("legacy", telemetry, legacy, 0) + if report.Agrees { + t.Fatal("differing sets agreed") + } + if len(report.TelemetryOnly) != 1 || report.TelemetryOnly[0] != "only-telemetry" { + t.Fatalf("telemetry only = %+v", report.TelemetryOnly) + } + if len(report.LegacyOnly) != 1 || report.LegacyOnly[0] != "only-legacy" { + t.Fatalf("legacy only = %+v", report.LegacyOnly) + } + if report.InBoth != 1 { + t.Fatalf("in both = %d", report.InBoth) + } +} + +func TestCompareLiveSessionsFieldRules(t *testing.T) { + t.Run("subject disagreement is a mismatch", func(t *testing.T) { + telemetry := []LiveSession{liveSession("a")} + legacy := []LiveSession{liveSession("a", func(s *LiveSession) { s.Subject = UserSubject(9) })} + report := CompareLiveSessions("legacy", telemetry, legacy, 0) + if len(report.Mismatches) != 1 || report.Mismatches[0].Field != "subject" { + t.Fatalf("mismatches = %+v", report.Mismatches) + } + if report.Mismatches[0].Telemetry != "user:7" || report.Mismatches[0].Legacy != "user:9" { + t.Fatalf("mismatch values = %+v", report.Mismatches[0]) + } + }) + + // A value only one projection carries is a gap in that projection, not a + // disagreement between them. Counting it as a mismatch would bury the real + // ones under every field the older projection never populated. + t.Run("absence is not a mismatch", func(t *testing.T) { + telemetry := []LiveSession{liveSession("a")} + legacy := []LiveSession{liveSession("a", func(s *LiveSession) { + s.ProfileID = "" + s.MediaFileID = 0 + s.Node = "" + })} + report := CompareLiveSessions("legacy", telemetry, legacy, 0) + if len(report.Mismatches) != 0 { + t.Fatalf("absence produced mismatches: %+v", report.Mismatches) + } + // Agrees covers set membership and real contradiction only. Folding + // absences in would make it permanently false, since legacy rows carry + // no value for several of these fields, and therefore useless — so the + // absences are reported on their own axis instead. + if !report.Agrees { + t.Fatalf("absence alone was reported as disagreement: %+v", report) + } + for _, field := range []string{"profile_id", "media_file_id", "node"} { + if report.FieldsAbsent[field] != 1 { + t.Fatalf("fields absent = %+v", report.FieldsAbsent) + } + } + }) + + // Two independent writers stamping the same session cannot be expected to + // agree to the nanosecond. + t.Run("sub-second start skew is tolerated", func(t *testing.T) { + telemetry := []LiveSession{liveSession("a")} + legacy := []LiveSession{liveSession("a", func(s *LiveSession) { + s.StartedAt = s.StartedAt.Add(900 * time.Millisecond) + })} + if report := CompareLiveSessions("legacy", telemetry, legacy, 0); !report.Agrees { + t.Fatalf("900ms of start skew was reported as a mismatch: %+v", report.Mismatches) + } + }) + + t.Run("multi-second start skew is a mismatch", func(t *testing.T) { + telemetry := []LiveSession{liveSession("a")} + legacy := []LiveSession{liveSession("a", func(s *LiveSession) { + s.StartedAt = s.StartedAt.Add(-5 * time.Second) + })} + report := CompareLiveSessions("legacy", telemetry, legacy, 0) + if len(report.Mismatches) != 1 || report.Mismatches[0].Field != "started_at" { + t.Fatalf("mismatches = %+v", report.Mismatches) + } + }) +} + +// Truncation must be visible. A capped list with no count would read as +// "covered everything". +func TestCompareLiveSessionsCapsEveryList(t *testing.T) { + telemetry := make([]LiveSession, 0, 10) + legacy := make([]LiveSession, 0, 10) + for i := 0; i < 10; i++ { + telemetry = append(telemetry, liveSession(string(rune('a'+i))+"-telemetry")) + legacy = append(legacy, liveSession(string(rune('a'+i))+"-legacy")) + } + // Sessions present in both, disagreeing on subject, to exercise the mismatch cap. + for i := 0; i < 10; i++ { + id := "shared-" + string(rune('a'+i)) + telemetry = append(telemetry, liveSession(id)) + legacy = append(legacy, liveSession(id, func(s *LiveSession) { s.Subject = UserSubject(99) })) + } + + report := CompareLiveSessions("legacy", telemetry, legacy, 3) + if len(report.TelemetryOnly) != 3 || report.TelemetryMore != 7 { + t.Fatalf("telemetry only = %d (+%d)", len(report.TelemetryOnly), report.TelemetryMore) + } + if len(report.LegacyOnly) != 3 || report.LegacyMore != 7 { + t.Fatalf("legacy only = %d (+%d)", len(report.LegacyOnly), report.LegacyMore) + } + if len(report.Mismatches) != 3 || report.MismatchesMore != 7 { + t.Fatalf("mismatches = %d (+%d)", len(report.Mismatches), report.MismatchesMore) + } +} + +func TestLiveSessionsFromGlobalView(t *testing.T) { + view := GlobalMonitoringView{Sessions: []GlobalSessionView{ + { + SessionID: "b", Subject: UserSubject(7), ProfileID: "profile-1", MediaFileID: 42, + StartedAt: time.Unix(1_700_000_000, 0), + PlayMethods: []string{"direct"}, + ViewerEdgePublishers: []PublisherRef{ + {PublisherID: "p1", NodeID: ""}, + {PublisherID: "p2", NodeID: "node-b"}, + }, + }, + { + SessionID: "a", Subject: UserSubject(8), + // Two publishers disagreed about the play method, so §2.5 leaves the + // merged scalar unset and the projection must not invent one. + PlayMethods: []string{"direct", "transcode"}, + // Relay-only: no viewer edge, so this session claims no node. + Publishers: []PublisherRef{{PublisherID: "node", NodeID: "node-c"}}, + }, + }} + + sessions := LiveSessionsFromGlobalView(view) + if len(sessions) != 2 || sessions[0].SessionID != "a" || sessions[1].SessionID != "b" { + t.Fatalf("projection not sorted by session id: %+v", sessions) + } + if sessions[1].Node != "node-b" { + t.Fatalf("node = %q, want the first viewer-edge publisher with a node id", sessions[1].Node) + } + if sessions[1].PlayMethod != "direct" { + t.Fatalf("play method = %q", sessions[1].PlayMethod) + } + if sessions[0].PlayMethod != "" { + t.Fatalf("a disputed play method was rendered as %q; §2.5 forbids picking one", sessions[0].PlayMethod) + } + if sessions[0].Node != "" { + t.Fatalf("a relay-only session claimed node %q", sessions[0].Node) + } +} diff --git a/internal/streamtelemetry/store.go b/internal/streamtelemetry/store.go index dbed960e1..deb08e928 100644 --- a/internal/streamtelemetry/store.go +++ b/internal/streamtelemetry/store.go @@ -13,7 +13,9 @@ const ( publisherReasonMetaMissing = "meta_missing" publisherReasonIdentityMismatch = "identity_mismatch" publisherReasonCountMismatch = "count_mismatch" + identityFieldSubject = "subject" identityFieldProfileID = "profile_id" + identityFieldMediaFileID = "media_file_id" ) type SnapshotStore interface { diff --git a/internal/streamtelemetry/viewcache.go b/internal/streamtelemetry/viewcache.go new file mode 100644 index 000000000..48fa9cea3 --- /dev/null +++ b/internal/streamtelemetry/viewcache.go @@ -0,0 +1,184 @@ +package streamtelemetry + +import ( + "context" + "log/slog" + "sync" + "time" +) + +// DefaultViewTTL bounds how stale a served global view may be before a read +// rebuilds it. +const DefaultViewTTL = 5 * time.Second + +// ViewCacheStatus describes the served view's freshness. It travels with every +// read so a consumer can tell a fresh answer from a stale one instead of +// guessing. +type ViewCacheStatus struct { + // Available is false only before the first successful build. A zero view + // with Available false is explicitly "we do not know yet", which a consumer + // must not read as "no sessions are live". + Available bool + RefreshedAt time.Time + Age time.Duration + // Stale is true when the served value is older than the TTL, which happens + // when a refresh failed or one is in flight and a cached value was served + // rather than making the reader wait. + Stale bool + // BuildTook is the cost of the last successful rebuild. This is the number + // to watch: BuildGlobalView measured 347 ms at 50 000 sessions. + BuildTook time.Duration + Refreshes int64 + Failures int64 + LastError string +} + +// ViewCache serves the merged global view from a bounded-staleness cache rather +// than rebuilding it per read. +// +// BuildGlobalView is ~347 ms at the 50 000-session cap, so an admin endpoint +// that rebuilt per request would be a self-inflicted denial of service. The +// refresh is driven by reads rather than a ticker: a ticker would pay that cost +// on every server forever whether or not anyone is looking, while a read-driven +// TTL pays only when someone asks and never more than once per interval however +// many readers arrive at once. +// +// P1 needs a periodically refreshed view for its evaluator and can add a +// background ticker to this type; the read path will not have to change. +type ViewCache struct { + registry *Registry + ttl time.Duration + logger *slog.Logger + + mu sync.Mutex + building bool + buildDone chan struct{} + + view GlobalMonitoringView + status ViewCacheStatus + refreshes int64 + failures int64 +} + +// NewViewCache returns a cache over registry's global view. A non-positive ttl +// falls back to DefaultViewTTL. +func NewViewCache(registry *Registry, ttl time.Duration, logger *slog.Logger) *ViewCache { + if ttl <= 0 { + ttl = DefaultViewTTL + } + if logger == nil { + logger = slog.Default() + } + return &ViewCache{registry: registry, ttl: ttl, logger: logger} +} + +// TTL reports the configured staleness bound. +func (c *ViewCache) TTL() time.Duration { + if c == nil { + return 0 + } + return c.ttl +} + +// View returns the merged global view and its freshness. It never panics on a +// nil cache, a nil registry, or a registry with telemetry disabled: those report +// Available false rather than an empty-but-complete view, which a consumer could +// mistake for "nothing is streaming". +func (c *ViewCache) View(ctx context.Context) (GlobalMonitoringView, ViewCacheStatus) { + if c == nil || c.registry == nil || !c.registry.Enabled() { + return GlobalMonitoringView{}, ViewCacheStatus{} + } + + for { + c.mu.Lock() + age := now().Sub(c.status.RefreshedAt) + if c.status.Available && age <= c.ttl { + view, status := c.snapshotLocked(age, false) + c.mu.Unlock() + return view, status + } + if c.building { + // A refresh is already in flight. A reader that already has a value + // takes it rather than queueing behind a rebuild that may take + // hundreds of milliseconds; only a reader with nothing at all waits. + if c.status.Available { + view, status := c.snapshotLocked(age, true) + c.mu.Unlock() + return view, status + } + wait := c.buildDone + c.mu.Unlock() + select { + case <-wait: + continue + case <-ctx.Done(): + return GlobalMonitoringView{}, ViewCacheStatus{} + } + } + c.building = true + c.buildDone = make(chan struct{}) + done := c.buildDone + c.mu.Unlock() + + view, err := c.build(ctx) + + c.mu.Lock() + if err != nil { + // Keep the last good view. Going blind is worse than being visibly + // stale, and Stale plus LastError says exactly which one this is. + c.failures++ + c.status.Failures = c.failures + c.status.LastError = err.Error() + } else { + c.refreshes++ + c.view = view.clone() + c.status.Available = true + c.status.RefreshedAt = now() + c.status.Refreshes = c.refreshes + c.status.LastError = "" + } + c.building = false + close(done) + served, status := c.snapshotLocked(now().Sub(c.status.RefreshedAt), err != nil) + c.mu.Unlock() + return served, status + } +} + +func (c *ViewCache) build(ctx context.Context) (GlobalMonitoringView, error) { + start := now() + view, err := c.registry.GlobalView(ctx) + if err != nil { + return GlobalMonitoringView{}, err + } + c.mu.Lock() + c.status.BuildTook = now().Sub(start) + c.mu.Unlock() + return view, nil +} + +// snapshotLocked must be called with c.mu held. +func (c *ViewCache) snapshotLocked(age time.Duration, forceStale bool) (GlobalMonitoringView, ViewCacheStatus) { + status := c.status + status.Age = age + status.Stale = forceStale || age > c.ttl + if !status.Available { + status.Age = 0 + return GlobalMonitoringView{}, status + } + return c.view.clone(), status +} + +// clone deep-copies the slices a caller could otherwise mutate through the +// cached value. The maps and slices inside each session are shared with the +// cached copy, so consumers must treat the result as read-only — which every +// consumer of a monitoring view already does. +func (v GlobalMonitoringView) clone() GlobalMonitoringView { + out := v + out.IncompleteReasons = append([]string(nil), v.IncompleteReasons...) + out.Publishers = append([]PublisherStatus(nil), v.Publishers...) + out.MissingPublishers = append([]PublisherRef(nil), v.MissingPublishers...) + out.Sessions = append([]GlobalSessionView(nil), v.Sessions...) + out.Transfers = append([]GlobalTransferView(nil), v.Transfers...) + return out +} diff --git a/internal/streamtelemetry/viewcache_test.go b/internal/streamtelemetry/viewcache_test.go new file mode 100644 index 000000000..1a7dba49f --- /dev/null +++ b/internal/streamtelemetry/viewcache_test.go @@ -0,0 +1,172 @@ +package streamtelemetry + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// countingStore wraps LocalStore so a test can count how many times the cache +// actually rebuilt the view, and force a build failure. +type countingStore struct { + mu sync.Mutex + inner *LocalStore + loads int + failed error +} + +func newCountingStore() *countingStore { return &countingStore{inner: NewLocalStore()} } + +func (s *countingStore) Publish(ctx context.Context, snapshot Snapshot) error { + return s.inner.Publish(ctx, snapshot) +} +func (s *countingStore) Load(ctx context.Context) (Snapshot, error) { return s.inner.Load(ctx) } +func (s *countingStore) Leave(ctx context.Context) error { return s.inner.Leave(ctx) } +func (s *countingStore) LoadAll(ctx context.Context) (PublisherSet, error) { + s.mu.Lock() + s.loads++ + failed := s.failed + s.mu.Unlock() + if failed != nil { + return PublisherSet{}, failed + } + return s.inner.LoadAll(ctx) +} +func (s *countingStore) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.loads +} +func (s *countingStore) fail(err error) { + s.mu.Lock() + s.failed = err + s.mu.Unlock() +} + +func newCacheFixture(t *testing.T, ttl time.Duration) (*ViewCache, *Registry, *countingStore) { + t.Helper() + store := newCountingStore() + registry := NewRegistry(testConfig(), store, nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + // Publish one snapshot so the merged view has a publisher to report. + if err := store.Publish(context.Background(), registry.Snapshot()); err != nil { + t.Fatal(err) + } + return NewViewCache(registry, ttl, nil), registry, store +} + +func TestViewCacheServesWithinTTLAndRebuildsAfter(t *testing.T) { + cache, _, store := newCacheFixture(t, time.Minute) + + if _, status := cache.View(context.Background()); !status.Available || status.Stale { + t.Fatalf("first read = %+v", status) + } + if store.count() != 1 { + t.Fatalf("builds after first read = %d, want 1", store.count()) + } + + // Inside the TTL the cached value is served without touching the store. + for i := 0; i < 5; i++ { + if _, status := cache.View(context.Background()); status.Stale { + t.Fatalf("read %d was stale inside the TTL: %+v", i, status) + } + } + if store.count() != 1 { + t.Fatalf("builds inside TTL = %d, want 1 — the cache rebuilt per read", store.count()) + } + + // Past the TTL a read rebuilds. Move the package clock rather than sleeping. + restore := now + now = func() time.Time { return restore().Add(2 * time.Minute) } + t.Cleanup(func() { now = restore }) + if _, status := cache.View(context.Background()); !status.Available { + t.Fatalf("read after TTL = %+v", status) + } + if store.count() != 2 { + t.Fatalf("builds after TTL = %d, want 2", store.count()) + } +} + +// Many admins refreshing at once must not stampede a 347 ms rebuild. +func TestViewCacheSingleFlights(t *testing.T) { + cache, _, store := newCacheFixture(t, time.Minute) + + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, status := cache.View(context.Background()); !status.Available { + t.Errorf("concurrent read had no view") + } + }() + } + wg.Wait() + if got := store.count(); got != 1 { + t.Fatalf("builds under 16 concurrent readers = %d, want 1", got) + } +} + +// A failed refresh must keep the last good view rather than going blind: a +// consumer that saw an empty view would read it as "nothing is streaming". +func TestViewCacheKeepsLastGoodViewOnFailure(t *testing.T) { + cache, _, store := newCacheFixture(t, time.Minute) + if _, status := cache.View(context.Background()); !status.Available { + t.Fatal("first read failed") + } + + store.fail(errors.New("redis is down")) + restore := now + now = func() time.Time { return restore().Add(2 * time.Minute) } + t.Cleanup(func() { now = restore }) + + view, status := cache.View(context.Background()) + if !status.Available { + t.Fatal("a failed refresh discarded the last good view") + } + if !status.Stale { + t.Fatalf("a failed refresh was not reported stale: %+v", status) + } + if status.LastError == "" || status.Failures != 1 { + t.Fatalf("failure not reported: %+v", status) + } + if view.Epoch == "" && len(view.Publishers) == 0 { + t.Fatalf("served view is empty: %+v", view) + } +} + +func TestViewCacheReportsUnavailableWithoutTelemetry(t *testing.T) { + t.Run("nil cache", func(t *testing.T) { + var cache *ViewCache + if _, status := cache.View(context.Background()); status.Available { + t.Fatalf("nil cache reported available: %+v", status) + } + }) + t.Run("disabled registry", func(t *testing.T) { + cfg := testConfig() + cfg.Enabled = false + registry := NewRegistry(cfg, NewLocalStore(), nil) + t.Cleanup(func() { _ = registry.Stop(context.Background()) }) + cache := NewViewCache(registry, time.Minute, nil) + if _, status := cache.View(context.Background()); status.Available { + t.Fatalf("disabled registry reported available: %+v", status) + } + }) + t.Run("nil registry", func(t *testing.T) { + cache := NewViewCache(nil, time.Minute, nil) + if _, status := cache.View(context.Background()); status.Available { + t.Fatalf("nil registry reported available: %+v", status) + } + }) +} + +func TestViewCacheDefaultsTTL(t *testing.T) { + if got := NewViewCache(nil, 0, nil).TTL(); got != DefaultViewTTL { + t.Fatalf("TTL = %s, want %s", got, DefaultViewTTL) + } + if got := NewViewCache(nil, -time.Second, nil).TTL(); got != DefaultViewTTL { + t.Fatalf("negative TTL = %s, want %s", got, DefaultViewTTL) + } +} From f6c8b04c5421cd3a0d6c969d289699610e4a63e6 Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:34:45 +0000 Subject: [PATCH 10/44] docs(streamtelemetry): one working document for what shipped, one appendix for the rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ten documents on this branch — an eight-revision design, three per-phase documents, four verbatim prior-art copies and an HTML walkthrough, 4,517 lines in all — are replaced by two, and the working document is scoped to what this branch actually built. docs/design/2026-08-17-stream-telemetry.md is the working document. It states P0 as built rather than as planned: all five route families enrolled with their route counts, the family gate and its rollout procedure, the merge and completeness contract, the parity endpoint and why P0d deliberately stopped at comparison, the measured hot-path cost, the Redis transport as implemented, and one configuration table for every variable. It carries three diagrams and opens with a plain-language summary of what the system does and does not do. The enforcement design (former section 3) and the rules design (former section 5) are moved out to the appendix. Both were written and reviewed before any traffic had been observed, and every threshold in them is a guess until the measurements this branch produces exist. Keeping them in the working document implied a commitment the branch does not make: monitoring is what ships here, enforcement is designed afterwards against real distributions. Sections 3 and 5 remain as stubs rather than being renumbered away, because Go comments cite 2.2, 2.5, 4.2, 4.2b, 4.4, 6 and 7.1 directly and renumbering would break them. Nothing in the tree cites 3.x or 5.x; references from the surviving text are retargeted at the appendix. The document is also corrected against production. It previously stated that telemetry had never run in a deployment and that parity had never been observed; both were true when written and are not now. An 18-hour soak (185 samples, native and jellycompat) is recorded in section 6 with its cost numbers, the families it did not exercise, and the legacy-store defect the parity projection surfaced (#666). docs/design/2026-08-17-stream-telemetry-appendix.md holds what the working document sheds: approaches abandoned during implementation with the measurement that killed each, eight revisions of design positions abandoned under adversarial review, the prior-art trail, a glossary resolving the inherited identifiers, the verification and review record for P0, and now the deferred P1+ design. The streaming write-deadline document keeps its own file. It predates this branch, is referenced independently, and is where someone editing CopyChunked will look. --- .../2026-07-09-streaming-write-deadline.md | 69 +- .../2026-08-17-stream-telemetry-appendix.md | 520 ++++++++++ docs/design/2026-08-17-stream-telemetry.md | 969 ++++++++++++++++++ 3 files changed, 1552 insertions(+), 6 deletions(-) create mode 100644 docs/design/2026-08-17-stream-telemetry-appendix.md create mode 100644 docs/design/2026-08-17-stream-telemetry.md diff --git a/docs/design/2026-07-09-streaming-write-deadline.md b/docs/design/2026-07-09-streaming-write-deadline.md index 7483ade68..84980b1f8 100644 --- a/docs/design/2026-07-09-streaming-write-deadline.md +++ b/docs/design/2026-07-09-streaming-write-deadline.md @@ -86,12 +86,21 @@ Key points: - Bumps are rate-limited (`step`, ~15s) so we do one `SetWriteDeadline` per ~15s of wall time, not one per 32 KB chunk. - **`ReadFrom` must be implemented** and delegate to the underlying writer in bounded - slices (e.g. `io.CopyN(underlying, r, 64<<20)` per iteration, bumping the deadline - between iterations). Rationale: `http.ServeContent` → `io.Copy` uses the - `io.ReaderFrom` fast path on the response writer (sendfile for `*os.File`); a naive - wrapper without `ReadFrom` silently forfeits sendfile and burns CPU copying 15 GB - through userspace. Bounded-slice delegation keeps sendfile *and* the rolling - deadline. + slices, bumping the deadline between iterations. Rationale: `http.ServeContent` → + `io.Copy` uses the `io.ReaderFrom` fast path on the response writer (sendfile for + `*os.File`); a naive wrapper without `ReadFrom` silently forfeits sendfile and burns + CPU copying 15 GB through userspace. Bounded-slice delegation keeps sendfile *and* + the rolling deadline. + - **Slice size is a correctness constraint, not a tuning knob.** The deadline is an + *absolute* time, so a write attempted after it fails immediately — which means the + slice size divided by the window is a hard floor on the sustained client rate. The + original 64 MiB slice against the 180s window implied ~3 Mbit/s: any slower client + had its deadline expire part-way through a single slice and was reaped despite + making continuous progress. The slice is now `httpstream.ReadFromChunkDefault` + (4 MiB, ~186 kbit/s floor). Raising it re-introduces the reap. + - The shared helpers live in `internal/httpstream/readfrom.go` (`ReaderFromOf`, + `CopyChunked`, `WriterOnly`) and are reused by every response-writer wrapper on a + media chain, not just this one — see "Writer-chain conformance" below. - First bump happens in `New` (covers the header write + first body bytes). - `window` default 180s, overridable via env `SILO_STREAM_WRITE_STALL_TIMEOUT` (seconds). 180s > the Apple client's longest observed benign backpressure park @@ -125,6 +134,54 @@ Explicitly **unchanged**: `cmd/silo/main.go:2374` itself stays at `WriteTimeout: 120s`. That is the point: the global guard remains, streaming handlers opt out per-response with a better contract. +### Writer-chain conformance (added 2026-08-16) + +This wrapper is not the only `http.ResponseWriter` on a media route, and the ones +above it in the chain can silently defeat it. Two rules now apply to *every* wrapper +mounted on a path that serves media: + +- **Forward `ReadFrom`.** `io.Copy` discovers `io.ReaderFrom` by direct type assertion + and never consults `Unwrap()`, so a single wrapper without `ReadFrom` disables + sendfile for everything below it. Wrappers that count bytes must transfer in bounded + slices and credit each one, or a large transfer lands in a single accounting bucket. +- **Implement `Unwrap()`.** Without it, `http.ResponseController` dead-ends at that + wrapper and `SetWriteDeadline` fails, which degrades this writer to a plain + pass-through — the deadline is silently gone. Preserve `Hijacker` too wherever a + wrapper could sit over an upgradable route (ABS socket.io, the playback control + websocket). + +chi's `middleware.Compress` is the one that cannot be repaired: `compressResponseWriter` +implements `Unwrap`/`Flush`/`Hijack`/`Push` but **not** `ReadFrom`, and its handler wraps +unconditionally — the encoder is selected later, so even a non-compressible content type +still gets a sendfile-killing wrapper. It is therefore bypassed on exact bulk-media +routes via `httpstream.CompressExcept`, matching only the registered GET/HEAD methods +with exact segment counts and exact casing. A *blanket* bypass would be wrong: subtitle +font bundles are JSON served under the same global compressor, and bypassing them would +drop `Content-Encoding`/`Vary` and change the wire contract. + +Because handler-level tests bypass exactly the middleware this concerns, conformance is +verified by driving the **mounted routers over real sockets** (`internal/api`, +`internal/jellycompat`, `internal/audiobooks/abs`, `internal/proxy`), covering GET/HEAD, +single and multi-range, conditional responses, `Accept-Encoding` present and absent, +HTTP/2, the proxy→node hop, and the socket.io upgrade. + +Stream telemetry adds `streamtelemetry.observedWriter` between an enrolled route +handler's `RollingDeadlineWriter` and the real response writer, on every enrolled route +family. It follows the same conformance contract: bounded `ReadFrom` forwarding +preserves sendfile, `Unwrap` preserves deadline traversal, and `Flush`, `Hijack`, and +`Push` retain their optional-interface behavior. The outer compressor still bypasses +only exact bulk routes; subtitle-font JSON remains compressible. + +**Forwarding `ReadFrom` is necessary but not sufficient.** Go's kernel sendfile path +unwraps exactly one `io.LimitedReader` before it looks for the `*os.File`, and +`http.ServeContent`'s `io.CopyN` already contributes that one — so an accounting layer +that hands down a *freshly nested* limiter forfeits sendfile even though it forwards +`ReadFrom` correctly. `CopyChunked` therefore slices the caller's limiter over the same +underlying reader instead of nesting a new one. If you change it, re-run the +`strace -f -e trace=sendfile` comparison over a mounted router: a byte-exact body and a +correct `Range` status prove HTTP correctness, not sendfile. See +[stream telemetry §4.4](2026-08-17-stream-telemetry.md). + ## Server tests 1. **Unit — rolling behavior:** wrapper on a fake ResponseController records deadline diff --git a/docs/design/2026-08-17-stream-telemetry-appendix.md b/docs/design/2026-08-17-stream-telemetry-appendix.md new file mode 100644 index 000000000..a1321609a --- /dev/null +++ b/docs/design/2026-08-17-stream-telemetry-appendix.md @@ -0,0 +1,520 @@ +# Stream Telemetry — Appendix: approaches tried and discarded + +Companion to [the stream telemetry design](2026-08-17-stream-telemetry.md). That +document says what the system *is*; this one says what it deliberately is **not**, and +why. Every entry below was proposed, argued for, and rejected on evidence — several of +them twice. + +Read this before proposing a simplification. The working document is short because the +reasoning was moved here, not because the reasoning is absent. Section references +(`§2.5`, `§4.4`, …) point at the working document. + +--- + +## A. Approaches abandoned during implementation + +Found by building, measuring or testing — not by review. + +| Abandoned approach | Why it failed | +|---|---| +| **Nesting `io.LimitReader` in every accounting layer** | Go's kernel sendfile path unwraps exactly one limiter, and `http.ServeContent`'s `io.CopyN` already contributes it — so the *first* accounting layer silently forfeits sendfile. Measured with `strace`: 0 syscalls through the mounted proxy chain, 6 after slicing the limiter instead of nesting it (§4.4). This had been "settled by reasoning" twice, wrongly, before anyone ran `strace`. | +| **P0a's first sendfile fix** (`79493512`) | It restored `ReadFrom` forwarding but kept the nested limiter, so it did not actually restore sendfile. A byte-exact body and a correct `Range` status prove HTTP correctness, not sendfile. | +| **ORing `StartedAtDegraded` across every publisher** | A relay-only contribution carries a publisher-local first-seen stamp, so correlating a transcode node degraded an otherwise authoritative viewer-edge session. Start-time authority is viewer-edge-owned (§2.5). Caught by adversarial plan review before it shipped. | +| **"401/403/404 create no logical activity" as a blanket rule** | Conflicts with why manifest routes are enrolled at all: the compat master manifest finishes authorization, *then* starts a transcode, and can still 404. The boundary is authorization success, not response status (§4.2). | +| **A family flag defaulting to every family** | Would widen instrumentation across two more live byte paths inside the API process on upgrade alone. The default must be what already shipped; "set the variable before deploying" is a runbook, not a safe default (§6). | +| **Folding "field absent on one side" into a parity mismatch** | Legacy rows carry no value for several compared fields, so `agrees` would be permanently false and the flag useless. Absence is its own axis, counted in `fields_absent` (§6/P0d). Settled by writing the test and finding the assertion wrong, not the code. | +| **Asserting byte totals from `Registry.Snapshot()`** | `BytesAccepted` there is `lastSweptBytes`; only `Sweep()` folds live observations. Five tests, one full remediation round (§2.2). | +| **Repointing admin reads onto telemetry in P0d** | The payload is a join onto ~50 display fields telemetry is not canonical for, and parity has never been observed. The design orders comparison before repoint for a reason (§6/P0d). | +| **A ticker-refreshed global view** | `BuildGlobalView` costs ~347 ms at the 50,000-session cap. A ticker pays that on every server forever whether or not an admin is looking. Replaced by a read-driven TTL cache with single-flight refresh (§6/P0d). | +| **A blanket compression bypass on media routes** | Subtitle font bundles are JSON and sit below global compression; bypassing would drop `Content-Encoding`/`Vary` and change the wire contract. Bypass only non-compressible bulk paths (§4.4). | +| **`r.Use` middleware for ABS telemetry** | The ABS group is shared with socket.io by `Mount`, so a group-level wrapper sits over the websocket upgrade. Wrapped per route instead, with the mounted-router socket test running telemetry both off and on (§4.2). | +| **Reusing the compat `nodes.go` Redis scan for parity** | That endpoint passes stored session JSON through opaquely so an older node's extra fields survive the round trip; a decoding reader would drop them. A second, decoding reader exists for the parity path only (§6/P0d). | + +--- + +## B. Positions abandoned during design review + +The design went through four adversarial review rounds by Codex `gpt-5.6-sol` at high +reasoning effort plus four maintainer scope corrections. Roughly one review finding in +five was itself wrong, so every one was verified against the code before acceptance. +Condensed, newest first. + +### Revision 8 — maintainer direction, after P0a shipped + +| Was | Became | +|---|---| +| Identity disagreement **quarantines the row** | **Record and surface both values** with a prominent admin warning. Quarantining hides exactly the case the system exists to catch: a publisher conflict is itself a possible abuse signal. Monitoring records and surfaces only; decision logic is deferred to P1+ (§2.5). | + +### Revision 7 — maintainer direction + +| Was | Became | +|---|---| +| `LibraryHarvest` is the primary rip signal | **`DeliveryRate` ships first** — no storage, catches the single-pass rip. Harvest demoted to the patient case and re-evaluated before P2 (§5.3). | +| Identity = ids + route + method + IP | Adds the **request-time capture set**: client name/version/build/channel/UA, device id, outcome, token age, request count. Unrecoverable if not captured at request time (§2.2). | +| Downstream restream and aggregate node load are tracked gaps | **Explicit non-goals.** Not gaps, not deferred (§5.2). | + +### Revision 6 — after review round 4 + +| Was | Became | +|---|---| +| "Only a permanent ban is durable" | **Contradicted the sanction table.** Durable state is *sanctions*; durability and permanence are separate axes (§1, §3.2). | +| `EvaluationInput` carries `Usage` + `Jobs` | **Fields with no producer.** P1 ships `View` + `Limits` only (§5.1). | +| "Five rules ship" | Two enforce; the rest deferred without a phase (§5.1). | +| "Download flood: enforced" | **Not covered** — re-downloading one title never grows the distinct set (§5.2). | +| The view carries a "monotonic version" | It is an **opaque epoch**; a fingerprint has no order (§2.5). | +| `complete` = freshness | Freshness cannot say *which* publishers are required. Membership added (§2.5). | +| Stop-on-lease-loss = fenced | **Not fencing.** A paused leader can resume; needs a monotonic fence token validated at every mutation (§2.5). | +| Sanction cache invalidated by pub/sub | **Pub/sub is lossy and at-most-once.** Now a hint over a generation counter reconciled against Postgres (§3.4). | +| Harvest merges served intervals | **No producer exists** — no offsets on `Observation`/`Transfer`, multipart ranges give no usable `Content-Range` (§5.3). | +| ABS "records an empty address" | **False.** It falls back to `RemoteAddr`; the real defect is proxy-peer attribution (§4.3). | + +### Revision 5 — after review round 3 + +| Was | Became | +|---|---| +| `LibraryHarvest` is ledger-free and cheap | **Withdrawn.** A byte *sum* is not byte *coverage* — serving the first 10% nine times reads as 90%. Needs a bounded, pruned coverage store; idempotence holds only *after* completion is established (§5.3). | +| Harvest denominator = 90% of `FileSize` | **Representation-specific**, and `file_size` is nullable. Transcode and remux declared **undetectable** by this rule rather than silently missed (§5.3). | +| `stream_bans` carries suspend | **Cannot** — no `action`, no lift metadata, no active state. Replaced by `stream_sanctions` (§3.4). | +| Suspend "blocks all streams" | Four unspecified behaviours, now defined: gate, in-flight fan-out, remote job stop, and what a lift clears (§3.4). | +| `StartedAt` authoritative from session creation | **No carrier existed** — no creation time in token or card, so reconstruction stamped `time.Now()`. Added in P0a (§4.3). | +| Viewer identity owned by the outermost edge | **The compat proxy could not supply it** — `buildProxyRedirectURL` omitted uid/pid/mfid entirely. Fixed in P0a (§4.3). | +| Redis failure ⇒ every replica evaluates | **Split-brain**, and worse than useless: replicas could issue *durable* suspensions from divergent local views. Fenced lease; a degraded view stops global evaluation (§2.5). | +| Violation counter per evaluation pass | A steady condition escalated to suspension in three ticks. Now per *incident transition* (§3.2). | +| Eight writer wrappers | **Nine** — chi compression is mounted globally on both routers and has no `ReadFrom` (§4.4). | +| Legacy retirement inside P0 | **Its own project** — nine named consumers including health payloads and two `/api/v1` endpoints (§6). | +| P0 ships as one phase | **Split P0a–P0d** (§6). | + +### Revision 4 — after review round 2 + +| Was | Became | +|---|---| +| "One Snapshot, no secondary data paths" | **Asserted, not built.** `Snapshot` was per-publisher while rules needed a global view. The merge contract, ownership per field and freshness were defined (§2.5). | +| Deterministic ordering removes the need for election | **Wrong.** A total order only agrees on identical inputs. Election restored; ordering demoted to intra-epoch tie-break (§2.5). | +| `Rule(Snapshot)` | **Self-contradictory** — `OverCap` needs limits, ratio needs file sizes, volume needs history. Replaced by an assembled `EvaluationInput` (§5.1). | +| Transcode-node segments/artifacts = `viewer_egress` | **Wrong, and a repeat of a round-1 bug.** Every node route sits behind `requireBearer`; counting them re-created relay double-counting (§4.2). | +| Transcode ratio via target bitrate × runtime | **No such denominator exists.** `TargetBitrateKbps` is max *video* only and often 0; no runtime is stored; segments are re-servable (§5.3). | +| Per-session ratio catches ripping | **Wrong.** A rip reads the source once, at ~1× coverage. Demoted to alert-only secondary (§5.3). | +| Download-class pours fit the session model | **They don't** — proxy downloads mint a new session id per redirect. Split into `Transfer` (§4.2b). | +| `UsageSink` seam in P0 = "one migration, one file" | **Wrong** — multi-replica double counting, epochs, idempotency. In-memory sink dropped; the ledger is unscheduled until its contract is designed (§5.4). | +| Escalation ends in an automatic ban | **Softened.** Policy table with `suspend` as the aggressive default; `ban` stays an admin action (§3.2). | +| `clientip` missing on the proxy only | **Also missing on standalone ABS** (§4.3). | +| "No annotation fails registration" | Not implementable in chi — it cannot tell whether an arbitrary `r.Get` serves media. Replaced by a route-manifest diff test (§4.1). | + +### Revision 3 — maintainer scope correction + +| Was | Became | +|---|---| +| Volume and byte budget "deliberately not in v1" | **Reversed.** Concurrency caps count streams and therefore cannot bound a rip; volume is first-class (§5.2, §5.3). | +| Usage persistence unaddressed | Checked all 76 existing tables: none stores bytes (§5.4). | + +### Revision 2 — after the first adversarial review + +The review returned "do not implement as written", and its root finding became the +spine of the design: **an in-flight HTTP transfer is not a playback session.** + +| Was | Became | +|---|---| +| Per-request meter is the unit; "no merge exists, therefore merge bugs cannot" | **Wrong.** Short transfers vanish between sweeps; relays double-count; native and remote ids genuinely differ. Aggregation is unavoidable — the fix is homogeneous role-tagged observations folded into one accumulator (§2.1, §2.2). | +| Cuts need no durability, re-derived each tick | **Wrong.** A cut that works destroys its own evidence, so it lapses and the still-valid token reconstructs. That is an exploitable duty cycle, not self-healing (§3.1). | +| Meter flag replaces `SetWriteDeadline`; lands in ~5–6s | **Wrong.** A flag is cooperative and cannot interrupt a blocked write; the real fallback is a 180s stall window with `WriteTimeout: 0` behind it (§3.3). | +| `nil` transport ⇒ "one code path" | **Overstated** — true of the rule API only. Replaced by an explicit `SnapshotStore` plus per-process publisher identity (§2.3). | +| Bytes are the only liveness clock | **Too reductive.** Misses buffer-ahead, producer activity and never-served requests (§2.4). | +| `clientip` is free at the edge | **Wrong.** Not mounted on the proxy or ABS routers (§4.3). | +| The meter preserves sendfile via `Unwrap`/`ReadFrom` | **Wrong.** `io.Copy` uses a direct type assertion and never consults `Unwrap` (§4.4). | +| Enrolment list complete | **Incomplete** — missing all manifest routes, native subtitles and fonts, and the compat bandwidth probe (§4.2). | +| P0 is zero-risk | **Wrong.** No policy risk is not no playback risk (§6). | +| Node-blob Redis encoding | **Partly kept**, refined to per-instance hash + deltas + heartbeat (§8). A blob TTL would tie publisher health to node liveness. | +| Group-merged limit resolution | **Confirmed correct**, hardened with caching and a degraded-mode metric (§5.3). | + +--- + +## C. Prior art: the `feat/sauron-async-enforcer` branch + +This design supersedes an earlier attempt — 26 commits, ~12.5k lines, based on a commit +70 branches behind `main`. That branch was **mined, not rebased**, and still exists as +`origin/feat/sauron-async-enforcer` (`d01b5de3`). Its four planning documents were +copied verbatim into this branch for context and have been removed again now that their +conclusions are recorded here. They remain readable at: + +| Document | Path on `origin/feat/sauron-async-enforcer` | +|---|---| +| Original plan + as-built deltas | `docs/superpowers/plans/2026-07-04-stream-monitoring-and-kill-switch.md` | +| The widened architecture decision | `docs/superpowers/plans/2026-07-07-abuse-cold-enforcer-architecture.md` | +| Path × monitoring × kill coverage matrix | `docs/architecture/playback-paths-monitoring-kill-matrix.md` | +| Adversarial abuse scoring, corrections 1–9, decisions A1–A8 | `docs/architecture/stream-abuse-matrix.md` | + +**Kept from it — the ideas, not the code:** server-observed existence rather than +client-reported liveness; every reason collapsing to a small set of enforcement actions; +the hot path paying at most one in-memory lookup; the `Route` dimension; and the finding +that *every* byte-serving surface must be enrolled or the picture lies. + +**Kept after review, having first been deleted along with the code:** reason-scoped +monotonic verdict expiry; per-process publisher identity distinct from node URL; a +bounded transfer registry; first-seen logical `StartedAt`; a non-extending cut latch; +and mounted-router/real-socket tests. + +**Deliberately not repeated:** the two-tier lease; the durable `stream_revocations` +table; durable tombstones; startup resurrection machinery; the Postgres session log; and +`mergeStreams`-style reconciliation of two incompatible models. Those are consequences +of choices this design does not make. + +### Inherited identifiers + +Older decision and gap ids still appear in commit messages and code comments. Recorded +here so they resolve after the prior-art copies were deleted. + +| Id | Meaning | Status in this design | +|---|---|---| +| **A1** | Over-cap enforcement uses a long revocation matching the token's reconstructable lifetime | Retained as the sticky-verdict argument (§3.1). | +| **A2** | Durable tombstones for revocation state | **Not repeated** — replaced by `stream_sanctions` (§3.4). | +| **A3** | Credential identity for a user cutoff comes from presented credential time | Superseded by the explicit creation-time claim (§4.3). | +| **A4** | ABS bare files and ebook/comic/PDF reads are observed and killable but **cap-exempt** | Retained (§4.2, §4.2b). | +| **A5** | Liveness is server-observed only; paused sessions holding an open realtime connection stay exempt (issue #243) | Retained (§2.4). | +| **A6** | Publish every stream to Redis keyed by per-process instance id; one evaluator per tick | Retained and hardened — the lease must be *fenced* (§2.3, §2.5). | +| **A7** | Registry saturation fails closed for download-class pours, plus a per-user concurrent-transfer cap | **Deferred to P1.** P0 serves through and reports truncation (§2.2, §9). | +| **A8** | Split the work along dependency order rather than shipping one batch | Retained as P0a–P0d (§6). | +| **GAP-10** | ABS access-log wrapper lacks `Unwrap`, so in-flight cuts no-op | **Fixed** (§4.4). | +| **GAP-11** | Ebook routes not wired into the kill switch | Enrolled as cap-exempt transfers (§4.2). | +| **GAP-12** | The rolling deadline re-arms a cut socket | Cut latch retained in the P1 design (§3.3). | +| **GAP-13** | `mergeStreams` discards edge `BytesServed` | **Not applicable** — no two-model merge exists here (§2.5). | +| **GAP-14** | Registry saturation blinds monitoring | Open, owned by P1 (§9). | +| **GAP-15** | Edge transcode visibility created before proxying | Superseded by the attachment boundary (§4.2). | +| **D18** | Jellycompat download quota hole | Out of scope — an enforcement gap in another subsystem (§5.5). | +| **E25–E28** | Compat and `auth/refresh` API flood limiting | Out of scope (§5.5). | +| **E29** | Per-node concurrent-transcode cap | Out of scope — node admission, not telemetry (§5.2, §5.5). | + +--- + +## D. Verification and review record for P0 + +**Gates run on the implementation host** across the whole branch: `gofmt`, +`go build ./...`, `go vet ./...` clean; `go test ./...` with one pre-existing failure +(`TestResolveCopySeekAnchorMatchesRealLongGOPHEVC`, which needs ffmpeg ≥ 5.x); +`golangci-lint run --new-from-merge-base=origin/main` reporting 0 issues; +`make verify-local-paths` passing; `go test -race -count=3` green on +`streamtelemetry`, `proxy`, `transcodenode`, `httpstream`, `jellycompat`, +`audiobooks/...`, `api` and `nodesessions`. The web gates were not run — `pnpm` is +absent on that host — but no `web/` file was touched. + +**Cross-model review coverage was uneven, and two commits have none.** The Codex side of +the relay hit its usage limit partway through the final session. + +| Commit | Plan | Plan review | Implementation | Result review | +|---|---|---|---|---| +| proxy + transcode-node enrolment | Claude | **Codex** — 8 findings | **Codex** | Claude; 3 defects, fixed by Codex | +| jellycompat + ABS enrolment | Claude | **Codex** — 9 findings, 7 accepted | Claude | Claude | +| P0d admin parity | Claude | **none** | Claude | Claude | + +**The P0d parity commit is the one with no second opinion at all**, and it is also the +commit that decides what "agrees" means — the semantics the legacy-retirement project +will lean on. If anything gets re-reviewed, it is that. + +**What the review rounds were worth.** Codex's plan reviews caught the sendfile defect, +the start-time degradation defect, the family-gate default, and an ABS client-info +helper that had been claimed not to exist — all before any of it shipped. Roughly one +finding in five was still wrong in one direction or the other, so each was checked +against the code before being acted on. + +**What worked, recorded because it is repeatable:** + +- Adversarial plan review *before* writing code. Two rounds, seventeen findings, and + the two that mattered most would both have shipped. +- Measuring instead of arguing. The sendfile question had been settled by reasoning + twice, wrongly; `strace` settled it in one command. +- Driving mounted routers over real sockets. Handler-level tests bypass the middleware + under test. +- Making the merge and the comparison pure functions. No Redis, no Postgres, every rule + tested in CI. +- Writing the test before trusting the semantics. The `agrees`-versus-absence question + was settled by a failing test — and the code turned out to be right. +- Running the gates directly rather than trusting the implementer's report. Codex + reported the socket tests as impossible in its sandbox; they run fine on the host. + +--- + +## E. Deferred P1+ design, retained for reference + +Pulled out of the main document when P0 shipped. This is the enforcement and rules +design as reviewed, preserved so the thinking is not lost — **not** a commitment to +build it as written. Every threshold below was chosen before any production traffic +had been observed through the telemetry path, which is precisely why it was deferred. + +## 3. Enforcement — P1, designed not built + +### 3.1 Why a cut must be sticky + +A violation that successfully stops its victim destroys the evidence it was derived +from. Cut lands → the meter disappears → the snapshot is clean → the cut lapses → the +still-valid token reconstructs (tokens stay reconstructable for ~24h, and loading an +existing session does not rerun fresh admission) → serves seconds of bytes → is cut +again. That is an exploitable duty cycle, not self-healing. + +### 3.2 Four actions, chosen by a policy table + +The infrastructure worth building is observing accurately and applying a deny. Which +rule triggers which response is tuning, and must never require a code change. + +| Action | Effect | Reversible | Durable | +|---|---|---|---| +| `alert` | admin notification only | n/a | no | +| `cut` | stop this session; sticky so it cannot immediately resume | expires | no | +| `suspend` | block this user from all streams until lifted | yes | yes | +| `ban` | permanent denial | admin only | yes | + +Default posture leans aggressive: rules escalate to **`suspend`**, not `ban`. A suspend +stops abuse immediately and is fully reversible; `ban` stays a deliberate admin action. + +**A violation is an incident, not an evaluation pass.** One continuous over-cap +condition observed on three consecutive ticks is *one* violation. Count a new violation +only on a condition *transition*, or after the previous incident has cleared — +otherwise a steady state escalates to suspension in fifteen seconds. + +| Violation | Sticky for | Durable? | +|---|---|---| +| 1st | 10 min cut | no | +| 2nd within the window | 30 min cut | no | +| 3rd | the rule's configured action (default `suspend`) | yes | +| Admin terminate | 30 min cut | no | +| Admin suspend / ban | until lifted / `expires_at` | yes | + +**Sticky verdicts are the load-bearing part; escalation is not.** If the counter proves +fiddly, a single sticky verdict per subject still closes the oscillator. Expiry merges +monotonically and is reason-scoped: a later 10-minute over-cap verdict must never +shorten a live 30-minute admin termination. Postgres is written **only** when a +sanction is created, lifted or expires. + +### 3.3 Cutting: cooperative flag *and* interrupt + +A flag is only checked when execution reaches the next write, so it cannot interrupt a +`Write` already blocked on an unreading client, a `ReadFrom` blocked inside the +underlying writer, a remux goroutine blocked reading ffmpeg stdout, a segment request +waiting on production, or a relay blocked reading upstream. The real fallback is the +180s stall window, and the standalone streaming servers run `WriteTimeout: 0`, so there +is no server-level guard behind it. A cut handle therefore owns three things: + +1. **An atomic flag** — stops a fast-draining pour at the next application write. This + is the rip case. +2. **A request-scoped `CancelFunc` plus route-specific closers** — cancels upstream + response bodies and closes ffmpeg-fed pipes, reaching blocked reads a deadline + cannot. +3. **An immediate response write deadline**, with a **cut latch** so the rolling + deadline's periodic bump cannot re-arm a cut socket. + +Several handlers currently normalize or ignore write/copy errors, so ending the body +does not universally close the connection, especially for chunked and HTTP/2 responses. +Each enrolled route must be audited for error propagation, not assumed. + +Honest bound: a cut lands within one enforcer tick for a draining stream and within the +interrupt path's latency for a blocked one. It is **not** a uniform 5s guarantee. + +### 3.4 The sanction ledger + +One Goose migration. The table models the *sanction*, not the ban, because `suspend` is +durable and reversible: + +```sql +CREATE TABLE stream_sanctions ( + id BIGSERIAL PRIMARY KEY, + subject_kind TEXT NOT NULL, -- 'user' | 'profile' | 'ip' + subject_id TEXT NOT NULL, + action TEXT NOT NULL, -- 'suspend' | 'ban' + status TEXT NOT NULL, -- 'active' | 'expired' | 'lifted' + rule TEXT NOT NULL, -- which rule fired, or 'admin' + evidence JSONB, -- snapshot rows + violation history + created_by INTEGER, -- NULL = automatic + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ, -- NULL = indefinite + lifted_at TIMESTAMPTZ, + lifted_by INTEGER +); +CREATE UNIQUE INDEX ... ON stream_sanctions (subject_kind, subject_id) WHERE status = 'active'; +CREATE INDEX ... ON stream_sanctions (expires_at) WHERE status = 'active'; +``` + +One active sanction per subject, enforced by the partial unique index. Lifting sets +`status`/`lifted_*` rather than deleting, preserving audit history. `evidence` is what +makes an automatic sanction reviewable. + +**What a suspend must actually do**, because "blocks the user from all streams" is four +distinct behaviours: + +1. Deny every future viewer-facing media request for that subject. +2. Fan out cuts to every in-flight session *and* transfer for that subject. +3. Stop correlated remote jobs — otherwise a node keeps encoding for its full idle + window for a suspended user. +4. Define what an admin lift clears: the sanction always; sticky session cuts and the + violation counter **also**, so a lifted user is not immediately re-suspended by + stale state. + +`users.enabled` is not a substitute: it blocks the whole account rather than streaming, +and does not stop already-issued proxy stream tokens or ABS public-session +capabilities. + +**Cache coherence blocks P1.** The set loads into memory at boot, but Redis pub/sub is +lossy and at-most-once, so a replica that misses a message would deny or allow +indefinitely — unacceptable in both directions for a durable deny cache. The cache +needs a generation counter reconciled against Postgres on a bounded interval, with +pub/sub used only to make the common case fast. + +--- + +## 5. Rules — P1 and later + +### 5.1 One assembled input + +`Rule = func(EvaluationInput) []Sanction`, where the input is built once per evaluation +pass: + +```text +// P1 ships exactly this: +EvaluationInput { + View GlobalMonitoringView // §2.5 — merged, epoch-stamped + Limits EffectiveLimits // cached LimitResolver +} + +// P2 adds, for LibraryHarvest only: + Harvest HarvestFacts // §5.3 +``` + +Rules stay pure consumers; every policy, catalog and ledger lookup happens once, +centrally, cached. This is what makes "one place to look" true rather than asserted. A +field with no producer is an invitation to build one badly, so fields appear when their +stores exist, not before. + +### 5.2 The abuse surface, and which signal catches it + +| Abuse | Signal | Phase | Confidence | +|---|---|---|---| +| Account sharing | session count per `uid` | P1 | **enforced** — measured fact | +| **Fast pull / naive rip** | **delivery rate ÷ media bitrate** | **P1** | **enforced — cheapest signal here** | +| Looping / wasteful re-pull | session bytes ÷ file size | P1 | alert-only, secondary | +| Library harvest / patient rip | distinct titles fully retrieved per window | P2 | alert-only first | +| Re-stream, naive token sharing | distinct viewer IPs per session | P3 | alert-only | +| Sustained volume | bytes per user per window | later | deferred — needs a durable ledger | +| Download flood | — | — | **not covered** — re-downloading one title never grows the distinct set | +| Token mint / hoard / replay | distinct sessions per token | deferred | not covered | +| Low-byte, high-work storms | requests per session per window | deferred | not covered | +| Transcode exhaustion, one user | starts + seconds per window | deferred | not covered | + +Deferred rows are work without meaningful bytes, so a byte-oriented engine is the wrong +instrument for them; the §2.2 capture set records what a later probe would need without +building the probe now. + +**Two explicit non-goals** — not gaps, not deferred, not tracked: + +- **Downstream re-streaming.** A service that pulls once at a normal rate and fans out + on its own infrastructure is invisible by construction, and is accepted as such. + (`RestreamFanout` still catches one token used from several addresses, which is a + different and much dumber attack.) +- **Aggregate transcode load across all users.** Per-node exhaustion control is a node + admission concern. + +### 5.3 Rip detection + +**`DeliveryRate` — sustained delivery rate ÷ media bitrate. Ships first.** + +A real player pulls at roughly **1× realtime**: it buffers ahead, then throttles to +match playback. A ripper pulls at whatever the link allows — routinely 10–20×. Both +inputs already exist (`MediaFile.Bitrate`, `MediaFile.Duration`, and the byte deltas +the collector already computes), so the rule needs **no storage at all**. It is the +cheapest thing in this design and it catches the single-pass rip. + +- **Playback sessions only.** Full-speed delivery is expected for downloads, so + `Transfer` is exempt. +- **Sustained, not instantaneous.** HLS buffer-ahead legitimately spikes for the first + minute and after a seek. +- Requires a non-zero bitrate; fall back to `FileSize ÷ Duration`, else skip the + session. +- **The threshold needs real traffic.** Read the distribution before picking a + multiplier. + +**`OverCap`** — count cap-relevant viewer-egress sessions per `uid` against the +**group-merged effective** limit (`access.EffectivePolicyForUser` → `strictestPositive`, +as the admission closure already does) — never raw `users.max_streams`, which is 0 for +standard users. Victim selection is deterministic, ordered by +`(logicalStartedAtUnixNano, canonicalSessionID)`, as a tie-break *within* an epoch and +not a substitute for election. Limits resolve through a shared cached `LimitResolver`; +on provider failure enforce the cached value and emit a degraded-mode metric. Fail open +only with no trustworthy cached value. + +**`LibraryHarvest` — distinct titles fully retrieved per window. Alert-only first.** + +**A byte sum is not byte coverage:** serving the first 10% of a file nine times sums to +90% of `FileSize` while revealing 10%. So this needs a real `HarvestProgressStore` +merging **non-overlapping served byte intervals** keyed by +`(user, canonical_item, representation, window)`, with one idempotent completion fact +written when union coverage crosses the threshold. + +**It is blocked on a producer that does not exist.** Neither `Observation` nor +`Transfer` carries representation offsets; the direct path derives a single `rangeStart` +only after `ServeContent`, and a multipart range response yields no usable global +`Content-Range` at all. Release facts must carry exact served intervals, and multipart +or unknown-offset responses are excluded from coverage rather than guessed at. Window +semantics and the canonical title/part key are also undefined. + +Denominators are representation-specific: `MediaFile.FileSize` for direct play (and it +is nullable — no size, no rule), artifact output size for prepared downloads, converted +`stat.Size` for ebooks. **Remux and transcode are explicitly undetectable by this +rule**, not quietly missed. + +Known evasions and false positives, stated rather than discovered later: pulling 89% of +every title is invisible; re-downloading one title forever does not grow the distinct +set; transcoding the whole library is invisible; episodes are distinct items so a +legitimate binge completes dozens weekly; **household profiles share one `user_id`** so +several family members aggregate into one subject; public RSS downloads attribute to +the **feed owner**, so legitimate subscribers can make an owner look like a harvester; +and treating a 30-second extra, one comic chapter and a four-hour film as one "title" +is implausible without media-class-specific policy. + +**`SessionOverConsumption`** — session bytes ÷ source file size. Catches only the +*wasteful* abuser who pulls the same file repeatedly inside one session. Kept because +it is nearly free, explicitly demoted to secondary alert-only so it is not mistaken for +rip protection, and applies to playback sessions only. + +**`RestreamFanout`** — distinct viewer IPs per canonical session, sustained over a +window, from the bounded per-session IP set. Alert-only by default; stronger action +behind an operator setting. Privacy posture stated explicitly: this retains +short-window per-viewer IP history. + +**`AdminTerminate` is not a rule.** It is operator-initiated and immediate — a command +that writes a sticky verdict directly, not something that arrives through the +evaluation loop. + +**Transcode over-consumption does not ship.** There is no usable denominator: +`TargetBitrateKbps` is max *video* bitrate and is frequently zero, jellycompat local +transcodes never set it, no job start time or accumulated runtime is stored, wall-clock +is not media time, and segments are re-servable across replans. + +### 5.4 The usage ledger, deferred honestly + +A per-user byte budget ("500 GB / 30 days") is desirable and not in the shipping set, +because it is harder than one migration and one file: multi-replica double counting +(every replica flushing its own view multiplies the total), publisher epoch and stable +identity across restarts, cumulative vs incremental values and idempotency keys, UTC +bucket boundaries and late deltas, restart reconciliation and retention. + +**No interface is frozen in P0.** When the ledger lands it uses an explicitly +idempotent contract — `(publisher_id, publisher_epoch, sequence, subject, metric, +cumulative_value, observed_at)` — with only the publisher that observed the viewer +egress writing usage, a persisted checkpoint, and increments computed transactionally. +A process-local in-memory sink cannot satisfy a multi-replica rule and would invite +callers to depend on semantics the durable version cannot honour. + +No existing table could carry it: of the current 76, none stores bytes. + +### 5.5 Still out of scope + +The jellycompat download quota hole, the per-node concurrent-transcode cap, and +compat/`auth.refresh` rate limiting remain deferred. They are enforcement gaps in +*other* subsystems, not monitoring gaps, and none blocks this design. + +--- + +--- + +## AI-use disclosure + +Written with AI assistance (Claude Opus 5), consolidating the revision history of eight +design revisions, three phase documents and four prior-art documents. The review +findings summarised in section B were produced by Codex `gpt-5.6-sol` at high reasoning +effort and were verified against the cited code before acceptance. diff --git a/docs/design/2026-08-17-stream-telemetry.md b/docs/design/2026-08-17-stream-telemetry.md new file mode 100644 index 000000000..7b0703ea5 --- /dev/null +++ b/docs/design/2026-08-17-stream-telemetry.md @@ -0,0 +1,969 @@ +# Stream Telemetry + +> **Status.** P0 (a–d) is **built** on `feat/stream-telemetry-enforcer`, cut from `main` +> @ `edd919c5`. It observes only: nothing is blocked, throttled, cut or banned. +> Enforcement (P1), harvest (P2) and heuristics (P3) are **deferred** — see §3 and §5. +> +> **Validated in production.** An 18-hour soak on a live deployment (185 samples, +> `native` + `jellycompat` enabled) held at 183/183 complete views, zero build failures, +> zero stale views, and a merge cost of 3 ms median / 10 ms p95 / 52 ms max. See §6. +> +> **Section numbers are load-bearing.** Go comments cite `§2.2`, `§2.5`, `§4.2`, +> `§4.2b`, `§4.4`, `§6` and `§7.1` directly. Renumbering breaks those references. +> §3 and §5 are intentionally kept as stubs rather than renumbered away. +> +> **Companion:** [stream-telemetry appendix](2026-08-17-stream-telemetry-appendix.md) — +> approaches tried and discarded, revision history, and the prior-art trail. Read it +> before proposing anything that looks like a simplification; most simplifications +> here have already been tried and have a recorded reason for failing. +> +> **Related:** [streaming write deadline](2026-07-09-streaming-write-deadline.md) — +> the writer-chain conformance rules §4.4 depends on. + +--- + +## In plain language + +**The problem.** The server can count how many streams a user has open, but not how +many *bytes* they pull, from which addresses, at what rate, or through which node. A +concurrency cap therefore cannot see a ripper: one session downloading the whole +library at link speed looks exactly like one person watching a film. There is also no +single place to look — proxy health, admin sessions, node sessions and playback stats +each answer a different part of the question from a different store. + +**What P0 built.** Every byte-serving route in every process now reports what it +served, to whom, and how fast, into one merged picture — asynchronously, off the hot +path, and without trusting anything the client says. Five router families across three +kinds of process publish into Redis; a pure function merges them; an admin endpoint +serves the result and diffs it against the two projections admins read today. + +**What P0 deliberately does not do.** It makes no decisions. No request is denied, +delayed or cut, and no existing admin read has been repointed onto it. Judgement — +which rate is a rip, which session to cut, who to suspend — is deferred on purpose: +a threshold set before the traffic has been measured is a guess, and this is the thing +that does the measuring. Monitoring becomes first-class here; enforcement is built on +top of it afterwards, against real distributions. + +--- + +## Architecture at a glance + +### Figure 1 — five families, three processes, one merge + +```mermaid +flowchart LR + subgraph API["silo API process"] + N["native · 18 routes
viewer_egress"] --> RA["streamtelemetry.Registry
sweep → Snapshot"] + J["jellycompat · 24
viewer_egress"] --> RA + A["abs · 22
viewer_egress"] --> RA + end + subgraph PX["proxy node process"] + P["proxy · 22
viewer_egress"] --> RP["Registry
key = verified stream token"] + end + subgraph TN["transcode node process"] + T["transcode_node · 8
internal_relay"] --> RT["Registry
correlation key only"] + end + RA -- "publish snapshot" --> R[("Redis
silo:stelem:snap:PUBLISHER
silo:stelem:roster")] + RP -- "publish snapshot" --> R + RT -- "publish snapshot" --> R + R -- "reads roster + all snapshots" --> M["BuildGlobalView — pure
complete | degraded"] + M --> V["ViewCache
read-driven TTL, single-flight"] + V --> E["GET /api/v1/admin/stream-telemetry/parity
CompareLiveSessions"] + L["legacy projections, still authoritative
playback_sessions in Postgres
silo:sessions:* in Redis"] -. "compared field-by-field, never replaced" .-> E +``` + +Every arrow crosses a process or store boundary; nothing on the left reads anything on +the right. The dashed edge is what P0d deliberately did **not** cut over — the parity +endpoint reads those two stores only to diff against them (§6). + +### Figure 2 — what one observed request touches + +```mermaid +flowchart TB + subgraph REQ["1 · request in"] + C["viewer
GET /api/v1/stream/SESSION"] --> MW["base middleware
clientip · logger · metrics · compress"] + MW --> OB["Registry.Observe route
provisional Observation"] + OB --> H["handler
resolve + authorize"] + end + H -- "on authorization success only" --> AT["streamtelemetry.Attach
subject · profile · session id · media file"] + AT --> REG["Registry
sharded by session id
release folds the final byte total"] + subgraph RES["2 · response bytes out"] + F["os.File"] --> SC["http.ServeContent
io.CopyN adds limiter 1"] + SC --> WC["writer chain — each layer forwards ReadFrom
requestlog · metrics · activitylog · observedWriter"] + WC --> K["kernel sendfile → socket"] + end + WC -. "bytes accepted" .-> REG +``` + +The observer is one `http.Handler` wrapper plus one `ResponseWriter`; everything else +the registry does happens on a sweep goroutine. Two properties of this picture are +load-bearing and were each got wrong once before: + +- **The attach seam is authorization success, not response status** (§4.2). It lands + *before* a manifest handler starts a transcode, which is what will let a P1 cut + prevent that side effect instead of cleaning up after it. +- **`CopyChunked` slices the caller's limiter rather than nesting a new one** (§4.4). + Go's kernel `sendfile` path unwraps exactly one `io.LimitedReader`, so nesting one + per accounting layer silently forfeits sendfile for the whole chain. + +### Figure 3 — where the code went + +5,405 added production lines across 64 files. Two thirds are in one package, and the +categories most likely to read as scope creep are the two that are not telemetry. + +| Destination | Files | Lines | Share | +|---|---:|---:|---| +| `internal/streamtelemetry` | 16 | 3,495 | `███████████████████` 65% | +| route wiring & call sites | ~22 | 637 | `███` 12% | +| per-family declarations | 10 | 637 | `███` 12% | +| admin read path | 3 | 294 | `█` 5% | +| writer-chain conformance | 10 | 282 | `█` 5% | +| identity prerequisites | 2 | 60 | ` ` 1% | + +Per-family cost is ~64 lines: one route table and one identity-capture function. The +declarations cannot move into the core package without it importing all five router +families. Writer-chain conformance and identity prerequisites are not telemetry at all +— they are §4.4 and §6/P0a, and both fix defects that predate this work. + +--- + +## 0. What this builds on + +Two things on `main` are load-bearing here: + +- **`internal/httpstream.RollingDeadlineWriter`** — stall detection, outcome + classification, and a `ReadFrom` that preserves sendfile. Note *how*: a direct + `s.w.(io.ReaderFrom)` assertion. `io.Copy` discovers `io.ReaderFrom` the same way and + **never consults `Unwrap()`**, so any wrapper that does not itself implement + `ReadFrom` drops the whole chain to userspace copying. This governs §4.4. +- **`internal/clientip`** — a trusted-proxy boundary resolver. Before P0a it was + mounted on the native and jellycompat routers **only**, so `clientip.FromContext` + returned nothing at exactly the edges where viewer fan-out has to be observed. + +`main` contains none of the earlier `feat/sauron-async-enforcer` work. What was kept +from it is ideas, not code: server-observed existence rather than client-reported +liveness; every reason collapsing to a small set of enforcement actions; the hot path +paying at most one in-memory lookup; the `Route` dimension; and the finding that +*every* byte-serving surface must be enrolled or the picture lies. What was +deliberately not repeated is listed in the appendix. + +--- + +## 1. Requirements + +1. Every playback session reports into a central picture: bytes served, to which IP, at + what rate, direct/remux/transcode, which node. +2. Reporting is **off the hot path** — bytes go out first, telemetry follows. +3. **Never trust the client.** Liveness and volume are server-observed. +4. **One implementation** for integrated and multi-node, not two code paths. +5. **No full restart resiliency yet.** Durable state is limited to *sanctions* (appendix E, former §3.4) + — both `suspend` and `ban`. Durability and permanence are separate axes: a suspend + is durable *and* reversible. +6. Keep Postgres off the hot path and out of high-volume enforcement work. +7. **Bound volume, not just concurrency.** A stream cap counts sessions and is blind to + how many bytes flow through them (appendix E, former §5.2). +8. **One place to look.** All of the above resolves against a single monitoring + picture, not a per-feature side channel. + +--- + +## 2. The unit of accounting: logical sessions, not requests + +### 2.1 Why a request is not a session + +Making the in-flight HTTP transfer the unit fails in both directions, and no tuning +fixes it: + +- **Short transfers vanish.** An HLS segment, a subtitle, a small Range or a LAN-speed + download can begin and complete entirely between two collector sweeps. +- **One pour is counted many times.** Concurrent Range requests open several meters; a + proxied segment is metered at proxy egress and again at transcode-node egress. +- **Identities genuinely differ.** Native logical session ids and remote transcode + transport ids are deliberately distinct in current code. + +Aggregation is therefore unavoidable. The fix is to make every observation +**homogeneous and role-tagged**, then fold them into one canonical accumulator — +rather than reconciling two incompatible models after the fact. + +### 2.2 Three-level model — as built + +``` +Observation (per HTTP request) → LogicalSession / Transfer → Snapshot +role: viewer_egress keyed by canonical sid per publisher + | internal_relay first-seen StartedAt instance + | producer folded byte total + liveness signals +``` + +**`Observation`** — one per in-flight request, carrying an identity resolved once at +entry and an explicit `Role`. It counts bytes but creates no logical activity; a +request that never attaches is reported only through the unattributed counters. + +The **request-time capture set** exists because anything sampled at request time cannot +be recovered later — the request is gone. Anything derivable from stored data (geo/ASN +from IP, rates from bytes and timestamps) can wait. + +| Field | Why | +|---|---| +| `ClientName`, `ClientVersion`, `ClientBuild`, `ClientChannel`, `ClientUserAgent` | Separates a real player from `curl` or a scraper. Client-side anomalies are usually meaningless until grouped by user agent. | +| `DeviceID` | One account across forty devices is a strong sharing signal. | +| `Outcome` (`completed` / `stalled_reap` / `client_gone`) | A ripper **completes**; a browser **aborts**. Already classified by the rolling deadline writer — free to record. | +| `TokenIssuedAt` | Enables "how many distinct sessions is one token driving" — the cheap probe for token hoard/replay. | +| `RequestCount` (per session, per window) | Low-byte, high-work abuse (manifest/seek/replan storms) is invisible to a byte counter. | + +Each family supplies its own capture function, because identity lives in a different +place per protocol: native reads `X-Silo-*` headers, jellycompat parses the +`MediaBrowser` authorization header (the same parser the negotiation path uses, so +telemetry reads the value the play session was keyed on), ABS carries a numeric account +id as a string, and the transcode node has none of these. + +**Roles** are assigned at route declaration, never inferred. This single field is what +stops relay double-counting: + +| Role | Meaning | Counts toward cap? | Counts toward bytes? | +|---|---|---|---| +| `viewer_egress` | bytes leaving to an actual viewer | yes (if cap-relevant route) | yes | +| `internal_relay` | proxy→node, artifact fetch, server-to-server | **no** | no (correlated only) | +| `producer` | transcode job activity, no viewer bytes | no | no | + +**`LogicalSession`** — the canonical unit, keyed by canonical session id. It owns +`StartedAt` (first seen, never rewritten), a monotonic `BytesServed` accumulator, the +liveness signals of §2.4, the bounded viewer-IP set, and route/method/node attribution. +**Every observation folds its final total in on release**, so a transfer that lives and +dies between sweeps is still counted; the sweep reads in-flight deltas, and release is +what makes it lossless. A retention window after the last observation ends keeps the +session alive across the gaps between HLS segment requests. + +> **Trap, cost one full remediation round:** `Registry.Snapshot()` reports +> `lastSweptBytes`, not live bytes — only `Sweep()` folds live observations. Any test +> asserting a byte total must call `Sweep()` first. Both methods carry a doc comment +> saying so. + +**`Snapshot`** — one publisher instance's complete set of sessions and transfers at an +instant, plus `capturedAt` and a publisher heartbeat. + +**Bounds.** The registry has exact CAS-reserved global limits plus per-session limits. +When a limit is reached the request still proceeds; the snapshot becomes `Truncated`, +monotonic dropped counters grow, and the registry emits at most one warning per minute. +Bounded sets drop the newest value and expose their own overflow flag. Saturation +serving through is a P0 decision — a fail-closed download policy belongs to P1. + +### 2.3 Transport: a `SnapshotStore` in every mode, never `nil` + +An explicit `SnapshotStore` interface with a **local in-process implementation** rather +than `nil`, plus a Redis implementation. Every publisher — each API replica, each +proxy, each transcode node — publishes under a **random process-unique instance id**, +with logical node identity carried as a separate field. A node hash derived from +`nodeURL` is not process identity: instances sharing a public URL collide. + +Stated plainly here and in the code: **synchronous admission remains per-process; this +is a cross-replica backstop, not a replacement for it.** + +### 2.4 Liveness: bytes are primary, not sole + +Bytes alone miss HLS buffer-ahead (a client can legitimately make no request for +minutes), seek discontinuity, a paused stream, a transcode still burning CPU after the +last served byte, and a request in flight that has not yet served a byte. Three +**separately server-observed** signals live on the `LogicalSession`. None is +client-reported and none is a state machine, so this does not become a second lifecycle +model competing with `playback.SessionManager`: + +- `openObservations` — how many observations are in flight right now. +- `lastByteAccepted` — the byte clock; authority for volume and viewer activity. +- `realtimeConnectionAlive` — an open, ping-checked WebSocket, used only for the + paused-session exemption (issue #243). An *observed connection*, not a reported + position. + +Cap relevance is derived at evaluation time from these, never stored as a state. No +client progress timestamp enters enforcement. + +**Producer activity does not live on the `LogicalSession`.** A transcode job counts +toward neither the cap nor viewer bytes. A `JobView` is the natural carrier but is +**deferred without a phase** (§6): a live job snapshot cannot count jobs *started* over +a window, and the node request carries only a transport session id with no canonical +owner. + +### 2.5 The global view — as built + +Rules never read snapshots; they read a `GlobalMonitoringView` built by merging every +fresh publisher snapshot. `BuildGlobalView` is a **pure function** — its input carries +the roster, decoded snapshots, errors, the build time and all bounds; it reads neither +a clock nor Redis. That is what lets every rule below be unit-tested in CI with neither +Postgres nor Redis, and it is the property to copy for P1's evaluator. + +| Field | Merge rule | +|---|---| +| Session id | Canonical join key. Remote transport ids correlate to it explicitly; they are never treated as sessions. | +| Subject, profile id, media file id | **Only viewer-egress publishers contribute.** Populated disagreements retain all attributed values, flag a conflict, and leave the scalar zero. | +| `StartedAt` | Highest source rank (`claim`, `session`, `issued_at`, `first_seen`), then earliest value at that rank. **Degradation is viewer-edge-owned** — a relay's publisher-local first-seen stamp must not degrade an authoritative viewer-edge session. | +| Viewer bytes | Sum **only** viewer-egress routes. Never `SessionView.BytesAccepted`, which includes every role. | +| Relay bytes | Summed separately, for correlation only. | +| Open observations, requests | Saturating sum. | +| Liveness timestamps | Latest. Realtime connection is logical OR. | +| Viewer IPs and captured sets | Bounded, naturally sorted **unions** with overflow flags — never latest-wins, which would collapse the fan-out signal to one address. | +| Routes | Union by method, pattern and role; counters use saturating sums. | +| Play method | Union plus per-publisher values. **There is no invented scalar winner.** | +| Transfers | Never merged across publishers; keyed by publisher id plus transfer id. | +| Stale publisher | Dropped wholesale past the freshness bound; its heartbeat distinguishes a stalled publisher from a dead node. | + +**Identity disagreement records and surfaces *both* values.** It never picks an edge +arbitrarily and never quarantines the row. Two frontends can reconstruct the same +session and disagree; the conflicting values are carried side by side and flagged for +the admin view with a prominent warning, because a disagreement is itself a possible +abuse signal and hiding it defeats the purpose of the system. **Monitoring records and +surfaces only — no automated reaction, no resolution logic.** What a conflict *means* +is deferred to P1 or later. Media-file disagreement is both a conflict and a union, +because a legitimate replan can touch more than one file. + +**The view epoch** is the first 16 bytes of SHA-256 over the sorted publisher +`(publisherID, epoch, sequence)` tuples. It is an idempotency key, not an ordering: +epochs are equal or different, never earlier or later. + +**Completeness needs a membership contract, not just freshness.** A heartbeat says a +publisher is alive; it cannot say which publishers *must* be present before a decision +is safe. A publisher whose heartbeat is older than the membership TTL has departed and +does not block completeness; a roster member without a usable fresh snapshot is stale, +excluded, named in `MissingPublishers`, and does block it. `Complete` is true only when +all four hold: + +1. no publisher is missing or stale; +2. no merged snapshot is truncated; +3. the reader hit no publisher/session/transfer cap; +4. no publisher has decode errors, a count mismatch, or an oversized hash. + +**What the flag buys is telling blindness apart from absence.** A session that leaves +the view because the viewer closed the player, one whose bytes stop growing because the +client is buffered ahead, and one that disappears because its publisher stalled are +three different facts producing the same observation. Only the flag separates them. + +The damaging case is not a strange rate but an aggregate that under-counts and so fails +open: a per-user stream total sums across publishers, so losing one publisher makes six +streams read as two and a cap derived from it admits more. `BytesServed` is monotonic +**within a publisher**; the merged sum carries no such guarantee, because two +publishers can contribute viewer bytes to the same canonical session once the proxy and +ABS families are enrolled. **Consumers must not derive per-user aggregates, caps or +deltas from an incomplete view.** + +**A publisher that cannot reach Redis never joins the roster.** It is invisible to the +view rather than degrading it; P1 must not read that as safe completeness. + +**Election, when P1 needs it, must be fenced.** A total order only agrees on identical +inputs, so deterministic victim ordering is a tie-break *within* an epoch and not a +substitute for election. Stopping is not fencing either: a paused leader can resume +after its lease expired and write against a stale view. The lease must issue a +**monotonically increasing fence token** validated atomically at every Redis verdict +write and every durable sanction write. The epoch fingerprint cannot serve — it has no +order. **No such primitive exists in `internal/cache`; it is new work and it blocks +P1.** On lease loss or a degraded view, stop global evaluation: cached durable denies, +local admission and telemetry publishing keep running, so enforcement degrades to what +one process can safely decide rather than splitting the brain. + +--- + +## 3. Enforcement — deferred to P1 + +Nothing in this branch enforces anything. No request is denied, delayed, cut or +throttled, and no user is sanctioned. + +The enforcement design (sticky cuts, the policy table, cooperative-flag plus interrupt, +the sanction ledger) was written and reviewed, then deliberately pulled out of this +document: every threshold in it was a guess made before any traffic had been observed. +It is retained verbatim in appendix section E and will be reworked against the +measurements this branch produces, rather than shipped on assumption. + + +## 4. Enrolment — as built + +The recurring failure mode of the earlier attempt was an unenrolled byte path being +both invisible and unkillable, rediscovered four separate times. A hand-maintained list +is the wrong artefact. + +### 4.1 Typed declaration plus a manifest test + +Every media route is declared as a typed `MediaRoute` carrying family, method, pattern, +class, role, canonical session key, cap relevance and a capture function. The wrapper +is derived from the declaration, and a mount-site typo panics rather than silently +un-observing a route. + +"A media route with no annotation fails registration" is not implementable — chi cannot +tell whether an arbitrary `r.Get` serves media. The enforceable form is a **route +manifest test** per family: walk the mounted routers, collect every `(method, pattern)` +and diff against a checked-in manifest. Every walked route must be either a typed media +declaration or an explicit non-media line, and their union must cover every +declaration. A new media route fails the build until it is classified. + +**94 of 1,003 declared route entries are observed**; the other 909 are pinned as +non-media by the fixtures. When adding a route, classify it in the family's +`media_routes.go`, inspect the manifest diff, then regenerate: + +```bash +go test . -run TestMediaRouteManifest -update-route-manifest +``` + +**The attachment boundary, stated once and applied everywhere:** + +> A logical session or transfer is created at **authorization success** — the point +> where the handler has established *who* is asking and *which* session or item they +> are entitled to. Requests rejected before that point create nothing. A failure +> *after* it — a missing file, an upstream 502, an invalid subtitle index — still +> creates activity, because it is real traffic by an authorized principal, and the +> outcome records how it ended. + +Response status is **not** the boundary: the compat master manifest finishes +authorization, then starts a transcode, and can still 404. +`internal/jellycompat/streamtelemetry_test.go` pins this. + +### 4.2 The route inventory as enrolled + +| Family | Routes | Role | Canonical session key | +|---|---:|---|---| +| `native` | 18 | `viewer_egress` | handler attachment | +| `proxy` | 22 | `viewer_egress` | verified stream token | +| `transcode_node` | 8 | **`internal_relay`** | forwarded token, else `node-transport:` | +| `jellycompat` | 24 | `viewer_egress` | compat play session | +| `abs` | 22 | `viewer_egress` | ABS session id / abs user / feed owner | + +Classes are `playback`, `manifest` and `transfer`. Cap relevance is per route, not per +family: streams, segments and manifests are cap-relevant; downloads, ebook reads, ABS +files and the Jellyfin bandwidth probe are observed but cap-exempt. + +**Manifest routes are enrolled and load-bearing.** A killed session that reaches an +unenrolled manifest route can reconstruct or start ffmpeg before the next segment is +ever cut, which defeats the whole enforcement path. Redirect and preflight routes are +classified for the same reason: they serve almost no bytes but **issue a further +capability**. + +**The transcode node publishes a correlation key and nothing else** — no subject, no +profile, no viewer IP, no client. A node cannot know who is watching: its start request +carries no user, profile or media ownership fields at all. Its capture hook must never +fall back to the generic capture, which would record the *proxy's* address as a viewer +IP. The node's URL `{session_id}` is the transcode **transport** id, not the canonical +session id; the canonical id is resolved from the forwarded `X-Silo-Stream-Token`, and +otherwise falls back to `node-transport:` rather than joining a session it cannot +prove. + +**The rule: viewer bytes and viewer IP are owned exclusively by the outermost +viewer-facing edge.** Proxy→node hops, proxy artifact relays and download-prepare +transports are `internal_relay`, never cap-relevant, never viewer bytes. + +Two enrolment decisions worth not relitigating: + +- **ABS wraps per route, never as another `r.Use`** on the group `Mount` shares with + socket.io. The mounted-router socket test runs with telemetry both off and on. +- **`handleEbookFile` gets no attach** — it is a stub that unconditionally 404s. + +### 4.2b Download-class transfers are not playback sessions + +Several byte paths have a user but no stable playback session: native direct download, +compat `/Items/{id}/Download`, ABS bare file and public RSS feed file, and +ebook/comic/PDF reads. Proxy downloads are worse — they mint a fresh session id +containing `time.Now().UnixNano()` per redirect, so a "session" there resets on every +reconnect by construction. + +The view therefore holds **two kinds of live activity**: + +- `LogicalPlaybackSession` — stable identity, cap-relevant, participates in per-session + consumption rules. +- `Transfer` — download-class pours. Same byte meter, same user-level ledger, but + **never** subject to per-session ratio rules and never cap-relevant. + +**Identity normalization** is what lets their bytes sum together. ABS subjects normalize +onto the shared `UserSubject` — the ABS user-id *string* is the numeric silo account id +— so ABS bytes sum with native and compat per user. Only positive integers qualify: +`"0"` and `"-1"` parse but name no account, so they stay `abs_user`. The RSS feed route +resolves the feed owner. + +### 4.3 Identity at the viewer boundary — as built + +`clientip.Middleware` is now mounted on the standalone proxy and ABS routers as well as +native and jellycompat. This was a live defect on `main`, though narrower than it first +appears: ABS falls back to `RemoteAddr`, so the recorded address was not empty but was +the *proxy peer* rather than the viewer. Mounting the resolver deliberately changes +recorded session IPs and `RemoteAddr`-based logs to the resolved viewer address. + +`buildProxyRedirectURL` now populates `UserID`, `ProfileID` and `MediaFileID` in the +compat proxy stream token, which it previously omitted entirely — the ownerless-record +hole that let the earlier attempt bucket streams under user 0 and silently exempt them +from the cap. Two accepted costs: the token grows ~80–130 URL characters, and claims +are signed but **not encrypted**, so internal ids are readable to anyone already +holding the (already sensitive) stream URL. + +Stream tokens now carry an **immutable session creation time**. The existing JWT `iat` +could not serve: signing overwrites the registered claims wholesale on every mint, and +replans mint replacement tokens from a live session. The read order is explicit — +explicit original timestamp → compat `PlaybackSession.CreatedAt` → JWT `iat` → fall +back to `time.Now()` and mark the row degraded. **A missing timestamp never invalidates +an otherwise valid old token.** The top-level compat `CreatedAt` is preferred over a +nested `RecipeCard` field because the durable compat store unmarshals and rewrites the +whole JSON document, so a mixed-version rolling deploy would let an old replica +silently drop a nested unknown field. + +### 4.4 Writer-chain conformance — as built + +Because `io.Copy` finds `io.ReaderFrom` by direct assertion and never through +`Unwrap()`, every `ResponseWriter` wrapper on a media route must forward `ReadFrom` +while preserving its accounting, implement `Unwrap()` or the interrupt path is lost, +and preserve `Hijacker` wherever metering could sit over an upgradable route. Seven +wrappers were repaired — proxy egress, native request logger, native metrics, native +activity log, jellycompat logger, the jellycompat image-proxy-tag writer (which had +neither `ReadFrom` nor `Unwrap`) and the ABS access log (which lacked `Unwrap`) — over +shared helpers in `internal/httpstream` (`ReaderFromOf`, `CopyChunked`, `WriterOnly`). +Chi compression is the eighth and cannot be repaired from outside; it is instead +bypassed on exact bulk routes by `httpstream.CompressExcept`. Blanket compression +bypass is wrong: subtitle font bundles are JSON and must keep +`Content-Encoding`/`Vary`. + +**sendfile was dead through the whole proxy chain, and this is not a telemetry +finding.** `CopyChunked` drove `rf.ReadFrom(io.LimitReader(src, chunk))`, so every +accounting layer handed the next a **freshly nested** `*io.LimitedReader`. Go's kernel +fast path unwraps **exactly one** limiter before it looks for the `*os.File`, and +`http.ServeContent` always calls `io.CopyN`, which contributes the first one — so a +*single* accounting layer was already enough to lose sendfile. Measured with +`strace -f -e trace=sendfile` over an 8 MiB body: + +| Path | before | after | +|---|---:|---:| +| bare `http.ServeContent`, no wrappers | 5 | 5 | +| mounted proxy direct-play router | **0** | **6** | + +The fix, in `internal/httpstream/readfrom.go`: when `src` is already limited, slice it +by handing down a limiter over the *same* underlying reader and decrementing the +caller's budget, so the innermost `ReaderFrom` still sees one limiter over the file. +`readfrom_test.go` asserts the reader shape, since `strace` is not available in CI. + +> **If you touch `CopyChunked`, re-run the strace comparison.** A byte-exact body and a +> correct `Range` status prove HTTP correctness, not sendfile. + +**Slice size is a correctness constraint, not a tuning knob.** The deadline is an +absolute time, so slice size ÷ stall window is a hard floor on the sustained client +rate. The original 64 MiB slice against the 180s window implied ~3 Mbit/s and was +reaping healthy slow clients; it is now `ReadFromChunkDefault` (4 MiB, ~186 kbit/s). + +**Byte semantics.** `BytesAccepted` is body bytes accepted at the family's wrapper, +below any outer body transform. It equals wire bytes on bulk routes excluded from +compression and is **pre-compression** on the compressible subtitle and font routes. +That is deliberate and documented at each capture site; do not "fix" it by moving the +wrapper. + +**Tests drive mounted routers over real sockets** — GET, HEAD, single and multiple +Range, conditional responses, `Accept-Encoding` present and absent, HTTP/2, the +proxy→node hop, and the ABS socket.io upgrade. This is not optional: the earlier +attempt's ABS revocation test passed for weeks because it called handlers directly and +bypassed the middleware that broke the feature. + +--- + +## 5. Rules — deferred to P1 + +Rule evaluation, rip detection and the usage ledger are not built and are not designed +against real data yet. The earlier draft is retained in appendix section E. + +The reason is the same as §3: a rule needs a distribution to threshold against, and this +branch is what produces the distribution. + + +## 6. Phases and status + +### P0a — identity and writer prerequisites ✅ built + +No telemetry state at all, and independently correct on its own: the compat ownership +hole and the ABS proxy-attribution bug are live defects on `main` regardless of this +project. Four pieces plus one the phase did not originally name — the rolling write +deadline's 64 MiB `ReadFrom` slice (§4.4). Details in §4.3 and §4.4. + +### P0b — local shadow telemetry ✅ built, all five families enrolled + +Process-local, observation-only. `Observation`, `LogicalPlaybackSession`, `Transfer`, +release-fold, bounded retention (§2.2). One router family at a time, benchmarked before +the next. + +**The family gate.** `SILO_STREAM_TELEMETRY_FAMILIES` defaults to +`native,proxy,transcode_node`. The default is deliberately **not** every family: proxy +and transcode node are separate processes where `SILO_STREAM_TELEMETRY_ENABLED` is +already a per-family switch, while jellycompat and ABS share the API process with +native, so defaulting them on would widen instrumentation across two more live byte +paths on upgrade alone. "Set the variable before deploying" is a runbook, not a safe +default. + +**Rollout.** Name a shared-process family explicitly to enable it, one at a time. The +same variable is the kill switch — drop one misbehaving family without losing the rest. +The resolved set is logged at startup. An unrecognised name disables telemetry entirely +and names the variable, because a typo that silently observed nothing would be worse +than no telemetry. Once a family has run in production, move it into +`defaultObservedFamilies` in `internal/streamtelemetry/config.go`. + +### P0c — distributed read-only view ✅ built + +Publisher epoch/sequence, Redis snapshots, freshness, and the `complete`/`degraded` +merge (§2.5, §8). Still read-only, so no election and no sanctions are needed yet. + +### P0d — admin parity ✅ comparison built, repoint deliberately not + +`GET /api/v1/admin/stream-telemetry/parity`, a new additive admin endpoint behind the +same authorization as `/admin/sessions`. Read-only: no `/api/v1` response changed, no +migration, no Postgres or Redis write. + +The phase was specified as "serve a debug projection, **compare it against both legacy +projections**, *then* repoint admin sessions, stats and events". **The comparison +shipped; the repoint did not**, for two reasons that will not change by trying harder: + +1. **Parity evidence now exists, and it argues against a blind repoint.** The 18-hour + soak below produced it: the two projections do not describe the same population, and + where they differ the legacy side is the one that is wrong (#666). A repoint is a + correction to make deliberately, with the discrepancy understood, not a swap. +2. **The admin session payload is a join, not a swap.** `playbackSessionRow` carries + ~50 display fields — title, poster, season/episode, position, decisions, source + codecs — that telemetry is explicitly *not* canonical for. + +The repoint now belongs to the separate retirement change below, with this endpoint as +its input. + +**The two legacy projections it compares against:** + +| Projection | Written by | Read by | +|---|---|---| +| `playback_sessions_sync` (Postgres) | `internal/worker/reconciler.go` from `playback.SessionManager` | `GET /api/v1/admin/sessions`, `active_streams` in `admin_stats.go` | +| `silo:sessions:{nodeHash}:{sessionID}` (Redis, 60s TTL, 30s refresh) | `internal/nodesessions` on each proxy and transcode node | `GET /api/v1/admin/node-sessions` | + +**The cached view.** `BuildGlobalView` measured **347 ms at 50,000 sessions**, so it can +neither run at sweep frequency nor be rebuilt per request. `streamtelemetry.ViewCache` +is a **read-driven TTL cache with single-flight refresh**, not a ticker: a ticker would +pay the full rebuild on every server forever whether or not an admin is looking. A +reader holding a cached value never queues behind an in-flight rebuild; only a reader +with nothing waits. A failed refresh keeps the last good view and reports the error — +going blind is worse than being visibly stale — and before the first successful build +`available` is false, never empty-but-complete, which a consumer would read as "nothing +is streaming". `build_took_ms` is reported on every read and is the number to watch as +session counts grow. P1 can add a background ticker to the same type without changing +the read path. + +**The comparison rules that are decisions, not details.** `CompareLiveSessions` is a +**pure function** over `LiveSession` — the fields *every* projection can express: +session id, subject, profile, media file, play method, node, start time. Comparing a +field only one side can express would manufacture mismatches and bury the real ones. + +- **Only a field both sides carry can disagree.** A legacy row with no profile id is a + gap in that projection, not a contradiction; those are counted in `fields_absent`. +- **`agrees` covers set membership and real contradiction, not absence.** Folding + absences in would make the flag permanently false — legacy rows carry no value for + several of these fields — and therefore useless. Read `fields_absent` too. +- **Start times compare with one second of tolerance.** Two independent writers cannot + be expected to agree to the nanosecond, and nothing downstream needs them to: victim + ordering only has to be a total order. +- **Play method is rendered only when the merged view has exactly one** (§2.5 leaves + the scalar unset when publishers disagree). +- **Node comes from the viewer-edge publisher only**, so a relayed session never claims + a node that served no viewer. +- **Every list is capped at 50 with an explicit dropped count.** Silent truncation + would read as "covered everything". +- **The view's completeness travels with the diff.** A degraded view is missing + sessions by construction, so a report built on one is evidence of blindness, not + disagreement. A source that cannot be read reports itself unavailable *with a reason* + rather than being omitted. + +**A single report is a signal, not proof.** It samples three independently updated +stores; a session that starts or ends between the reads shows as a one-sided +difference. Repeated agreement over time is what a cutover needs. + +`internal/api/handlers/nodes.go` keeps its own Redis scan on purpose: it passes stored +JSON through opaquely so an older node's extra fields survive, and a decode-and-re-encode +reader would drop them. `nodesessions.ListAll` is a second, decoding reader for the +parity path only. + +### P0 soak — validated in production, 2026-08-17/18 + +Run on a live deployment carrying real traffic, rolled family-by-family: phase 0 (off) → +phase 1 (`native`) → phase 2 (`+jellycompat`), each held until quiet before widening. + +**Result over 18 hours / 185 samples:** + +| Measure | Result | +|---|---| +| `view.complete` | 183/183 | +| Build failures / stale views / clock skew | 0 / 0 / 0 | +| `build_took_ms` | median 3, p95 10, max 52 | +| Redis `silo:stelem:*` | 2 keys | +| Contradictions between publishers | 1, transient, after a container recreate | +| Container restarts caused | 0 | + +Live sessions resumed mid-stream across three container recreates — QSV transcode, +`-c:v copy` direct play and HLS all confirmed. + +**What the soak did *not* exercise**, stated plainly so the evidence is not read wider +than it is: `abs` (no audiobook traffic exists on the host), `proxy` and `transcode_node` +(single-node `MODE=integrated`), and the multi-publisher merge — every sample had exactly +one publisher. Those rest on the per-family manifest tests and the two-publisher +real-Redis integration test, not on production evidence. + +**What it found.** The parity projection surfaced a defect in the legacy view it is +compared against: `playback_sessions_sync` treats a progress POST as liveness, so +sessions that stopped fetching bytes persist indefinitely and their transcodes are never +reaped. Telemetry is byte-path driven and correctly excluded them; they appeared +one-sided in 179 of 185 consecutive samples. Filed as #666 — the finding belongs to the +legacy store, not to this branch. + + +### Legacy retirement — its own project, gated on parity evidence + +Nine consumers: the nodesessions admin endpoint, proxy health and status capacity, the +admin session query, admin active-stream counts, stop/delete and session events, +reconciliation, cleanup and shutdown. That touches health payloads, admin statistics, +realtime invalidation, stop cleanup, and two `/api/v1` endpoints under the +additive-only rule. Dual-publish, prove parity, migrate each named consumer, retire +writes last. + +**What is authoritative for what.** Telemetry is canonical *only* for server-observed +live activity, viewer bytes, viewer addresses and enforcement. The control stores stay +authoritative for their own lifecycle concerns and are not absorbed: +`playback.SessionManager` (synchronous admission, transports, cleanup), download rows +and the bandwidth limiter, the transcode-node job map, the jellycompat login and +playback-session stores, and ABS playback-session rows. **The one hard rule is that no +rule may read liveness or byte counts from any of them** — that is what would re-create +a second source of truth. + +### P1 — enforcement (next) + +`EvaluationInput` assembly, the fenced evaluator, the sanction store and its admin lift +path, the three-part cut handle and in-flight fan-out, admin terminate. Then `OverCap` +and `DeliveryRate` — both read the live view and need no new stores. + +**Two things block it** (§2.5, appendix E, former §3.4): the fence token primitive, which does not exist in +`internal/cache`; and sanction cache coherence, which needs a generation counter plus a +periodic Postgres reconcile. + +### P2 — harvest, P3 — heuristics + +P2 is the `HarvestProgressStore` with interval-coverage accounting, then +`LibraryHarvest` alert-only. **Re-evaluate before starting:** if `DeliveryRate` proves +effective in production, this phase buys only the patient ripper, and it is by far the +most expensive thing left. P3 is `RestreamFanout`, alert-only. + +**Deferred with no phase:** the durable usage ledger (appendix E, former §5.4); `TranscodeRate` and +`JobView` (a live job snapshot cannot count jobs *started* over a window); +`SessionOverConsumption` as an enforcing rule. + +### P0 is not zero-risk + +No policy risk is not the same as no playback risk. P0 inserts executable code into +every live byte path and can change which optional interfaces are visible, whether +`io.Copy` selects `ReadFrom`, flush timing, HEAD and Range behaviour, error propagation +and connection reuse, allocation and sharded-map contention on every HLS request, and +ABS socket.io if `Hijacker` is not preserved. Treat it as a production streaming +change: the §4.4 conformance suite, benchmarks, and the family flag as a canary. + +That risk has now been exercised rather than only reasoned about — 18 hours of live +traffic across three container recreates with no restart, no stall and no regression +attributable to the telemetry path (see the soak subsection above). The families that +carried that traffic were `native` and `jellycompat`; the rest still rest on tests. + +--- + +## 7. Hot-path budget and measured cost + +Per `Write`, after the bytes have gone out: **one atomic load (cut flag) before the +write, one atomic add (bytes) after it.** Nothing else. The `lastServedAt` stamp lives +on the collector, which sets it from its own sweep time whenever it observes a byte +delta. This is *minimal post-write accounting*, not zero — live byte telemetry cannot +be entirely off-path and this design does not claim otherwise. + +Measured with paired `disabled`/`enabled` sub-benchmarks in one run, `-count=5`: + +| Path | allocs/op | B/op | +|---|---:|---:| +| proxy direct play | +10 | +1.1 KB | +| proxy transcode segment | +11 | +1.3 KB | +| jellycompat direct stream | +10 | +1.2 KB | +| ABS public track | +10 | +1.2 KB | + +Throughput ranges overlap in every case. `internal/streamtelemetry/benchmark_test.go` +covers direct `ReadFrom`, progressive remux writes, high-RPS HLS (including an +`enabled_with_collector` variant that exposes sweep contention) and HTTP/2. Compare +paired results from the same run; figures from another host are not a baseline. + +--- + +## 8. Snapshot transport — as built + +One Redis **hash per publisher instance**, keyed by process instance id and never by +URL hash. A blob-per-node encoding was rejected: a blob TTL ties telemetry-publisher +health to node liveness, so a stalled publisher or a rejected oversized write makes +every session vanish atomically while the node keeps serving. + +| Key | Type | Contents | +|---|---|---| +| `{prefix}:snap:{publisherID}` | hash | `meta`, `s:{sessionID}` and `t:{transferID}` fields | +| `{prefix}:roster` | sorted set | publisher id scored by heartbeat Unix nanoseconds | + +Every publish is one `MULTI`/`EXEC`: optionally delete the hash for a full resync, +delete removed fields in chunks of 512, set changed fields in chunks of 512, always set +metadata, refresh the snapshot expiry, update the roster heartbeat, prune scores +strictly older than two membership TTLs, and refresh the roster expiry to ten TTLs. The +two-TTL prune margin keeps one publisher's clock from eagerly removing another. + +Delta state is a 128-bit truncated SHA-256 digest per encoded field. Any transaction +error clears it and forces a full publish; a periodic full publish bounds drift even +without a reported error. `Leave` atomically removes the publisher from the roster and +deletes its snapshot; shutdown stops the collector first so a final sweep cannot re-add +it. + +**Encoding** is versioned JSON with explicit field tags; all times are signed Unix +nanoseconds with zero reserved for the zero `time.Time`. Version 1 is additive: fields +may be added, never removed or retyped. Unknown properties and unknown outcome/token +map keys survive a rolling deployment; an unknown codec version or malformed field +degrades only its own publisher. JSON was chosen over compact binary deliberately — the +codec is isolated behind the store, and a hand-written binary schema for a large, +evolving session shape is not worth its maintenance cost at these sizes: + +| Benchmark | Result | +|---|---:| +| Representative session encoded size | 811 bytes | +| Session encode | 9,998 ns/op · 1,930 B/op · 17 allocs/op | +| Session decode | 39,674 ns/op · 2,640 B/op · 64 allocs/op | +| Global merge at 50,000 sessions | 347 ms · 234 MB · 600,333 allocs | + +**Read bounds.** Decode rejects negative counters and byte totals and caps every map +and slice before the data reaches the merge. A read selects live roster entries bounded +by the publisher cap, pipelines `HLEN` per publisher and skips any hash larger than +`MaxSessions + MaxTransfers + 16`, then fetches eligible hashes with pipelined +`HGETALL` — which keeps each per-publisher snapshot atomic, unlike paged `HSCAN`. +Fields are sorted before reader caps are applied. Missing metadata, publisher-id +mismatch, oversized hashes, count mismatch and field decode errors are all attributed +to their publisher; a decodable partial snapshot is degraded and still merged. + +Clocks are assumed roughly synchronized. A heartbeat or capture time farther ahead than +the freshness window sets `ClockSkewSuspected`, which is **diagnostic only** — P1's +fence token, not a timestamp comparison, is what makes mutations safe. + +--- + +## 9. Risks and open questions + +**Settled by P0:** + +- ~~Publisher membership~~ — the heartbeat *is* the roster (§2.5). +- ~~Global-view rebuild cost~~ — `ViewCache`, read-driven TTL, single-flight (§6/P0d). + +**Blocking, by phase:** + +| Open item | Blocks | Note | +|---|---|---| +| Fence token primitive | P1 | Nothing in `internal/cache` provides one (§2.5). | +| Sanction cache coherence | P1 | Generation counter + periodic Postgres reconcile. Pub/sub was tried and rejected: it is lossy, and a replica missing a lift denies indefinitely (appendix E, former §3.4). | +| Harvest interval production | P2 | Nothing carries representation offsets today; multipart ranges yield no usable `Content-Range`. Window semantics and the canonical title/part key are undefined (appendix E, former §5.3). | +| `DeliveryRate` threshold | P1 | Needs real traffic. Read the distribution before picking a multiplier. | + +**Standing risks:** + +- **Enrolment completeness remains the top risk**, not the machinery. §4.1's manifest + test is the mitigation; without it this document rots into another hand-maintained + list that misses seven routes. +- **Identity at internal hops is unsolved by design.** §4.2 states the boundary; the + choice between correlation and an authenticated internal envelope is open and should + be settled before P1. +- **Escalation tuning is guesswork until real data exists.** 10/30/suspend and the + decay window are starting points — the main reason P0 ships observation-only. +- **Bounded registry.** It must be bounded or it is a memory DoS, but + bounded-with-serve-through means saturation blinds monitoring. P1 owns the + resolution: fail closed for download-class pours plus a per-user concurrent-transfer + cap. +- **HTTP/2 and hijacked connections** need real tests; an immediate `ResponseController` + deadline under HTTP/2 flow control is unverified against the Go version in use. +- **Restart window.** Ephemeral verdicts are lost on restart, so a cut session can + resume until the next tick re-derives — bounded by the sticky window, and accepted + per requirement 5. Sanctions have no such window. + +**Known loose ends found during P0, deliberately not fixed here:** + +- **A proxy `download` token is accepted on `/stream/direct/{token}`.** `verifyToken` + checks only the signature; only the download handler checks `PlayMethod`. Replaying a + download token on the direct route serves the file and records a cap-relevant + *playback* session with play method `download`. Not a privilege escalation — the same + principal was already authorized for that file — but a telemetry-classification + wrinkle at an abuse boundary. Changing the proxy's token-scope model is its own + concern. +- **`api.Dependencies.ABSHandler` is dead config**, declared and never called by + `internal/api`. Pre-existing. +- **A pre-existing data race** in `TestHandleReplanPlaybackV3BoundsDeferredLeaseRelease` + reproduces identically on a clean `main` worktree. + +--- + +## Configuration + +All settings are read once at startup. Invalid **core** settings disable telemetry and +log the offending variable as an error; invalid **distributed-only** settings disable +distributed mode while leaving local observation running. + +| Variable | Default | Scope | Meaning | +|---|---:|---|---| +| `SILO_STREAM_TELEMETRY_ENABLED` | `false` | core | Master switch, per process. | +| `SILO_STREAM_TELEMETRY_FAMILIES` | `native,proxy,transcode_node` | core | Which route families are wrapped. Also the kill switch. | +| `SILO_STREAM_TELEMETRY_SWEEP_INTERVAL` | `1s` | core | Collector period. | +| `SILO_STREAM_TELEMETRY_RETENTION` | `5m` | core | How long a session survives its last observation. | +| `SILO_STREAM_TELEMETRY_MAX_SESSIONS` | `10000` | core | Local session cap. | +| `SILO_STREAM_TELEMETRY_MAX_TRANSFERS` | `10000` | core | Local transfer cap. | +| `SILO_STREAM_TELEMETRY_MAX_OBSERVATIONS` | `50000` | core | Local in-flight observation cap. | +| `SILO_STREAM_TELEMETRY_DISTRIBUTED` | `false` | distributed | Publish and read snapshots through Redis. | +| `SILO_STREAM_TELEMETRY_FRESHNESS` | `5s` | distributed | Maximum usable snapshot age; at least three sweep intervals. | +| `SILO_STREAM_TELEMETRY_MEMBERSHIP_TTL` | `60s` | distributed | Heartbeat age after which a publisher has departed; must exceed freshness. | +| `SILO_STREAM_TELEMETRY_KEY_PREFIX` | `silo:stelem` | distributed | Non-empty, whitespace-free Redis namespace. | +| `SILO_STREAM_TELEMETRY_FULL_RESYNC_EVERY` | `60` | distributed | Successful publishes between full hash replacements. | +| `SILO_STREAM_TELEMETRY_MAX_PUBLISHERS` | `256` | distributed | Roster entries considered by a read. | +| `SILO_STREAM_TELEMETRY_MAX_MERGED_SESSIONS` | `50000` | distributed | Reader-side session cap across publishers. | +| `SILO_STREAM_TELEMETRY_MAX_MERGED_TRANSFERS` | `50000` | distributed | Reader-side transfer cap across publishers. | +| `SILO_STREAM_TELEMETRY_VIEW_TTL` | `5s` | distributed | How stale a served merged view may be before a read rebuilds it. | + +Startup performs a two-second Redis ping for diagnostics only; a failure does not stop +the process or fall back to the local store — the publisher retries each sweep and +self-heals when Redis returns. + +--- + +## Operating it + +**Turning it on is the next task, and everything in P1 depends on it.** Every remaining +threshold is a guess until the merged view has been compared against what admins see +today. + +```bash +# 1. default family set: native, proxy, transcode_node +SILO_STREAM_TELEMETRY_ENABLED=true +SILO_STREAM_TELEMETRY_DISTRIBUTED=true + +# 2. read repeatedly, over days — one report is a sample, not proof +curl -fsS localhost:8091/api/v1/admin/stream-telemetry/parity + +# 3. widen one family at a time, only after the previous one is quiet +SILO_STREAM_TELEMETRY_FAMILIES=native,proxy,transcode_node,jellycompat +``` + +What to watch: + +- `build_took_ms` on every parity read, as session counts grow. +- `complete` vs `degraded` before believing any diff — a degraded view is missing + sessions by construction. +- `stale`, `age_ms` and `last_error` on the view, so a stale answer is never mistaken + for a fresh one. +- The startup line naming the resolved family set. + +**Testing notes that cost time to learn:** + +- Any test that starts a `Registry` must `Stop` it — the package-level clock seam races + leaked collector goroutines. A single green `-race` run proves nothing for that + package; run it several times. +- Handler-level tests bypass the middleware under test. Drive **mounted routers over + real sockets**. +- `PlaybackSessionStore` derives `ExpiresAt = CreatedAt + ttl` when `ExpiresAt` is + zero, so a test pinning a fixed `CreatedAt` starts failing once wall-clock passes it. + +--- + +## AI-use disclosure + +Written with AI assistance (Claude Opus 5), consolidating eight design revisions and +three phase documents against the implementation as built. The design it records went +through four adversarial review rounds by Codex `gpt-5.6-sol` at high reasoning effort, +whose findings were verified against the cited code before acceptance; the enforcement +posture in appendix E, former §3.2, the identity-disagreement rule in §2.5, and the phase split in §6 were +decided by the maintainer. Two of the ten commits it describes had no cross-model +review — see the appendix. From 07383f7dc363a7af0fb0d2454e0a5a4c0c87c864 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:33:07 -0400 Subject: [PATCH 11/44] feat(playback): add header-authenticated media transport --- docs/architecture/playback-protocol-v3.md | 44 +++- .../fixtures/valid/capability_response.json | 1 + .../v3/fixtures/valid/decision_response.json | 1 + docs/feature-changelog.md | 3 + internal/api/handlers/playback.go | 29 ++- internal/api/handlers/playback_test.go | 85 +++++++ internal/api/handlers/playback_v3.go | 194 ++++++++++++--- internal/api/handlers/playback_v3_test.go | 222 ++++++++++++++++++ .../api/handlers/playback_v3_union_test.go | 4 +- internal/api/router.go | 6 +- internal/nodepool/planner.go | 57 +++++ internal/nodepool/planner_test.go | 29 +++ internal/playback/protocol_v3.go | 45 ++-- internal/playback/protocol_v3_test.go | 19 +- internal/playback/session.go | 100 ++++---- .../protocol_v3/capability_response.json | 1 + .../protocol_v3/conformance_matrix.json | 1 + .../protocol_v3/decision_response.json | 1 + internal/playback/transcode_manager.go | 3 + internal/playback/transcode_manifest_test.go | 17 ++ 20 files changed, 742 insertions(+), 120 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 71221fae9..b5ab11a5b 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -98,13 +98,13 @@ the document is always the full one: "protocol_versions": [3], "features": ["playback_plan_v3", "neutral_playback_v3_contract_v1", "layout_aware_passthrough", "playback_route_diagnostics", "device_quirks_v1", "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", - "plan_source_duration_v1"], + "header_authenticated_media_v1", "plan_source_duration_v1"], "deliveries": ["original_http", "server_remux_progressive", "server_remux_hls", "server_transcode_hls"], "transformations": [{"name": "audio_to_aac", "executor": "server", "recipe_version": "1", "validated_claims": ["audio_decode"]}] } ``` -The nine feature strings above are the full set this server version advertises: +The ten feature strings above are the full set this server version advertises: | Feature | What it promises | | --- | --- | @@ -116,6 +116,7 @@ The nine feature strings above are the full set this server version advertises: | `seek_reanchor_v1` | The `seek_reanchor` replan operation is available (§6) | | `output_change_v1` | The `output_change` intent replan is available; clients must keep the active route when this feature is absent | | `direct_stream_resume_v1` | A direct route may resume mid-file rather than restarting | +| `header_authenticated_media_v1` | An opted-in client receives only API-local media URLs without signed credentials in their query or path, and authenticates every media request with its normal Authorization header (§4.1) | | `plan_source_duration_v1` | `source.duration_seconds` is populated when known, so its absence means *unknown* rather than *unsupported* (§5) | That last one is the reason feature detection is a list and not a version @@ -446,6 +447,45 @@ omits entirely is unavailable — the server will not guess. expires: `none` means the URL is stable for the session, `session` means re-request headers from `header_refresh_url` rather than restarting playback. +### 4.1 Header-authenticated media URLs + +`header_authenticated_media_v1` is an engine-neutral client opt-in. A client +uses it only after the server advertises the same token, then includes it in the +top-level `client_features` on start and replan requests. For that attempt the +server returns only relative URLs on the authenticated API origin: + +- direct and progressive remux: `/stream/{session_id}` (an ordinary `seek` + parameter may still be present); +- remux/transcode HLS: `/playback/transcode/{session_id}/master.m3u8`, with + relative, credential-free segment URLs in the manifest; +- subtitle artifacts, inventory sidecars and font bundles: + `/stream/{session_id}/subtitles/...`. + +None of those client-visible URLs contains the signed playback token (`st`) or +a token-bearing proxy path, and no proxy or transcode-node origin is returned. +A pooled transcode node may still execute HLS behind the API server; the API +relays its manifest and segments over the same authenticated client route. +Direct-play and progressive-remux proxy routes are bypassed because those +nodes accept a signed URL token rather than the user's API credential, so the +normal local-remux fallback policy still applies. + +The client must attach its current `Authorization: Bearer ...` header to the +manifest/file request and every derived request, including HLS segments, +subtitle artifacts and font bundles. `stream.headers` deliberately does not +echo the bearer token: plans are persisted for idempotent replay, while the +client already owns the current access credential. `header_refresh: none` +means the relative media URL itself is stable; normal API-token refresh remains +out of band, after which a client can retry or reload that URL with the new +header. + +The signed playback token is what carries a reconstruction recipe across an +API restart. Opting it out therefore also opts out of transparent session +reconstruction: a missing in-memory session returns the normal expired/missing +response and the client starts a fresh attempt. Once selected, this mode is +sticky for the lifetime of the attempt; a client that can no longer honor it +must stop and start a new attempt rather than downgrade a replan to a +credential-bearing URL. + --- ## 5. The timeline model diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json index ca2e7674b..da2a89d5d 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json @@ -12,6 +12,7 @@ "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", + "header_authenticated_media_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json index 2cca07dfd..867778b4e 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json @@ -9,6 +9,7 @@ "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", + "header_authenticated_media_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index 7a3a5344b..320bac333 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -2,6 +2,9 @@ ## 2026-08-21 +### Keep signed playback credentials out of client-visible media URLs +Playback protocol v3 now advertises the engine-neutral `header_authenticated_media_v1` opt-in. Capable clients receive API-local direct, remux, HLS, subtitle, and font URLs without a signed stream token in the query or path, and attach their current API Authorization header to every media request instead. Existing clients keep the restart-resilient token URLs unchanged. Remote HLS executors can still run behind the API route, while direct/progressive proxy delivery is bypassed in this mode; transparent reconstruction after an API restart is intentionally replaced by a fresh client playback attempt. + ### Admin accounts are never capped by an access group An account promoted to admin kept its access group, so the Default Group's stream cap and library list still applied to it. Admins are now ungrouped everywhere: promoting clears the group, demoting lands the account on the default group unless the request names one, and `POST /admin/users`, `PUT /admin/users/{id}`, and `POST /admin/invitations` reject `role: "admin"` together with an `access_group_id` with `422`. Policy resolution ignores any group an admin row still carries, and a migration clears the admins that were grouped before this change. diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 5e7ac3cbc..8cb80bbf8 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -428,8 +428,10 @@ func (h *PlaybackHandler) streamCardFromQuery(r *http.Request, sessionID string) // loadTranscodeServeSession resolves the playback Session for the transcode // manifest/segment serve routes while keeping stream-token verification off the -// hot path. The overwhelmingly common case is a live in-memory session, which -// needs no token at all, so the cheap GetSession lookup runs first and the +// hot path. A V3 session that negotiated header-authenticated media requires a +// live authenticated owner on every request; a legacy session retains its UUID +// bearer behavior. The overwhelmingly common case is a live in-memory session, +// so the cheap GetSession lookup runs first and the // (HMAC + JSON) token decode is performed only on a not-found miss where a // reconstruct is actually required. On that miss it delegates to the shared // LoadOrReconstructSession front door so reconstruct/ownership semantics stay @@ -439,9 +441,12 @@ func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID s requestUserID := apimw.GetUserID(r.Context()) session, err := h.sessionMgr.GetSession(sessionID) if err == nil { - // Live session: enforce the same ownership rule as LoadOrReconstructSession - // (a zero caller is allowed; a non-zero mismatch is refused). No token - // verification on this hot path. + if session.RequireMediaAuthorization && requestUserID == 0 { + return nil, playback.SessionUnauthorized, nil + } + // Live session: secure transports require a user above; legacy bearer + // routes allow zero. Either way, a present but mismatched identity is + // forbidden. No token verification on this hot path. if requestUserID != 0 && session.UserID != requestUserID { return nil, playback.SessionForbidden, nil } @@ -1297,8 +1302,9 @@ func alignedSeekSeconds(seekSeconds float64, segmentDuration int, targetVideoCod } // HandleGetTranscodeManifest handles GET /playback/transcode/{session_id}/master.m3u8. -// Auth is optional — the session UUID serves as an access token (same pattern -// as /stream/{session_id}). When auth context is present, ownership is verified. +// Legacy transports allow the session UUID to act as the access capability. +// Header-authenticated V3 transports require the live session owner on every +// request; their UUID is only a route identifier. // // Known-duration encoded sessions expose a synthetic full VOD manifest so the // player can seek immediately. Copy-video sessions expose FFmpeg's real @@ -1317,6 +1323,9 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return + case playback.SessionUnauthorized: + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return } transcodeSession := h.tm.GetTranscodeSession(sessionID) @@ -1358,7 +1367,8 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h } // HandleGetTranscodeSegment handles GET /playback/transcode/{session_id}/segment/{name}. -// Auth is optional — the session UUID serves as an access token. +// Authorization follows the same negotiated legacy-versus-header-authenticated +// rule as the manifest endpoint above. func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") session, status, card := h.loadTranscodeServeSession(r, sessionID) @@ -1372,6 +1382,9 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return + case playback.SessionUnauthorized: + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return } transcodeSession := h.tm.GetTranscodeSession(sessionID) diff --git a/internal/api/handlers/playback_test.go b/internal/api/handlers/playback_test.go index 619f42c31..80f1400a9 100644 --- a/internal/api/handlers/playback_test.go +++ b/internal/api/handlers/playback_test.go @@ -242,6 +242,91 @@ func newAuthorizedPlaybackContext() context.Context { return apimw.SetProfileID(ctx, "profile-1") } +func TestHeaderAuthenticatedMediaEnforcesHLSOwnerOnEveryRequest(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + manager.RegisterReconstructed(&playback.Session{ + ID: "secure-hls-session", + UserID: 1, + PlayMethod: playback.PlayTranscode, + RequireMediaAuthorization: true, + }) + manager.RegisterReconstructed(&playback.Session{ + ID: "legacy-hls-session", + UserID: 1, + PlayMethod: playback.PlayTranscode, + }) + handler := NewPlaybackHandler(manager) + + type endpoint struct { + name string + handle func(http.ResponseWriter, *http.Request) + path func(string) string + params func(string) map[string]string + } + endpoints := []endpoint{ + { + name: "manifest", + handle: handler.HandleGetTranscodeManifest, + path: func(id string) string { + return "/api/v1/playback/transcode/" + id + "/master.m3u8" + }, + params: func(id string) map[string]string { return map[string]string{"session_id": id} }, + }, + { + name: "segment", + handle: handler.HandleGetTranscodeSegment, + path: func(id string) string { + return "/api/v1/playback/transcode/" + id + "/segment/seg_00001.m4s" + }, + params: func(id string) map[string]string { + return map[string]string{"session_id": id, "name": "seg_00001.m4s"} + }, + }, + } + + request := func(endpoint endpoint, sessionID string, userID int) *http.Request { + req := httptest.NewRequest(http.MethodGet, endpoint.path(sessionID), nil) + if userID != 0 { + ctx := apimw.SetClaims(req.Context(), &auth.Claims{ + UserID: userID, Role: "user", TokenType: auth.TokenTypeAccess, + }) + req = req.WithContext(ctx) + } + routeCtx := chi.NewRouteContext() + for key, value := range endpoint.params(sessionID) { + routeCtx.URLParams.Add(key, value) + } + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) + } + + for _, endpoint := range endpoints { + t.Run(endpoint.name, func(t *testing.T) { + for _, test := range []struct { + name string + sessionID string + userID int + want int + }{ + {name: "secure missing auth", sessionID: "secure-hls-session", want: http.StatusUnauthorized}, + {name: "secure wrong owner", sessionID: "secure-hls-session", userID: 2, want: http.StatusForbidden}, + // The media process is intentionally absent in this unit fixture; + // reaching 404 proves the authenticated owner passed the gate. + {name: "secure owner accepted", sessionID: "secure-hls-session", userID: 1, want: http.StatusNotFound}, + // Legacy UUID-bearer behavior remains unchanged. + {name: "legacy missing auth accepted", sessionID: "legacy-hls-session", want: http.StatusNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + rr := httptest.NewRecorder() + endpoint.handle(rr, request(endpoint, test.sessionID, test.userID)) + if rr.Code != test.want { + t.Fatalf("status = %d body=%s, want %d", rr.Code, rr.Body.String(), test.want) + } + }) + } + }) + } +} + func withPlaybackRouteParam(req *http.Request, key, value string) *http.Request { routeCtx := chi.NewRouteContext() routeCtx.URLParams.Add(key, value) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 80ff90f0a..c15ce8c5e 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -77,6 +77,50 @@ type preparedTimelineV3 struct { copySeekAnchorResolved bool } +type headerAuthenticatedMediaContextKeyV3 struct{} + +// withHeaderAuthenticatedMediaV3 records the request's negotiated media-auth +// mode without carrying a credential. Transport preparation happens several +// layers below the v3 request decoder; keeping the bounded boolean on the +// request preserves that negotiation across direct, remux, HLS and replan +// paths while leaving the durable request body as the source of truth. +func withHeaderAuthenticatedMediaV3(r *http.Request, clientFeatures []string) *http.Request { + if r == nil { + return nil + } + enabled := playback.HasFeatureV3(clientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + return r.WithContext(context.WithValue(r.Context(), headerAuthenticatedMediaContextKeyV3{}, enabled)) +} + +func headerAuthenticatedMediaV3(r *http.Request) bool { + if r == nil { + return false + } + return headerAuthenticatedMediaContextV3(r.Context()) +} + +func headerAuthenticatedMediaContextV3(ctx context.Context) bool { + if ctx == nil { + return false + } + enabled, _ := ctx.Value(headerAuthenticatedMediaContextKeyV3{}).(bool) + return enabled +} + +func pinHeaderAuthenticatedMediaFeatureV3(clientFeatures []string, enabled bool) []string { + pinned := make([]string, 0, len(clientFeatures)+1) + for _, feature := range clientFeatures { + if strings.EqualFold(strings.TrimSpace(feature), playback.FeatureHeaderAuthenticatedMediaV3) { + continue + } + pinned = append(pinned, feature) + } + if enabled { + pinned = append(pinned, playback.FeatureHeaderAuthenticatedMediaV3) + } + return pinned +} + type transportErrorV3 struct { reason string message string @@ -280,32 +324,59 @@ type capabilitySessionPlannerV3 interface { PlanSessionWith(sessionID, currentTranscodeURL string, needsTranscode bool, estBitrateKbps int, eligible func(*nodepool.Node) bool) nodepool.Plan } -// planNodeSessionV3 selects transcode/proxy nodes for the session. Plans that -// carry server transformations restrict selection to nodes whose advertised +// localEgressSessionPlannerV3 lets a pooled transcode executor feed the API +// server while the API remains the only client-facing media origin. A planner +// that lacks this optional method still works; its proxy selection is discarded +// before a URL is returned, though the concrete nodepool planner implements the +// method so production reservation accounting stays exact. +type localEgressSessionPlannerV3 interface { + PlanTranscodeSessionWithLocalEgress(sessionID, currentTranscodeURL string, eligible func(*nodepool.Node) bool) nodepool.Plan +} + +// planNodeSessionV3 selects execution nodes for the session. Plans that carry +// server transformations restrict selection to nodes whose advertised // capabilities validate against the plan, so load balancing in a // heterogeneous pool cannot land a recipe on a node that would reject it when -// a capable sibling exists. Capability-blind selection remains for -// transformation-free plans and non-enumerating planners. -func (h *PlaybackHandler) planNodeSessionV3(ctx context.Context, session *playback.Session, result playback.PlannerResultV3) nodepool.Plan { - selector, selectable := h.NodePlanner.(capabilitySessionPlannerV3) - enumerator, enumerable := h.NodePlanner.(transcodeNodeEnumeratorV3) - if !selectable || !enumerable || !planRequiresServerTransformationsV3(result.Plan) { - return h.NodePlanner.PlanSession(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps) - } - capable := make(map[string]struct{}) - for nodeURL, advertised := range h.pooledNodeTransformationsV3(ctx, enumerator.TranscodeNodeURLs()) { - if validateAdvertisedTransformationsV3(result.Plan, advertised) == nil { - capable[nodeURL] = struct{}{} +// a capable sibling exists. In local-egress mode the API relays the selected +// transcode node and no client-facing proxy is selected or returned. +func (h *PlaybackHandler) planNodeSessionV3(ctx context.Context, session *playback.Session, result playback.PlannerResultV3, localEgress bool) nodepool.Plan { + var eligible func(*nodepool.Node) bool + if enumerator, ok := h.NodePlanner.(transcodeNodeEnumeratorV3); ok && planRequiresServerTransformationsV3(result.Plan) { + capable := make(map[string]struct{}) + for nodeURL, advertised := range h.pooledNodeTransformationsV3(ctx, enumerator.TranscodeNodeURLs()) { + if validateAdvertisedTransformationsV3(result.Plan, advertised) == nil { + capable[nodeURL] = struct{}{} + } + } + // The predicate runs under the planner lock: a set lookup only. + eligible = func(node *nodepool.Node) bool { + if node == nil { + return false + } + _, found := capable[node.URL] + return found } } - // The predicate runs under the planner lock: a set lookup only. - return selector.PlanSessionWith(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps, func(node *nodepool.Node) bool { - if node == nil { - return false + + if localEgress { + if selector, ok := h.NodePlanner.(localEgressSessionPlannerV3); ok { + return selector.PlanTranscodeSessionWithLocalEgress(session.ID, session.TranscodeNodeURL, eligible) } - _, ok := capable[node.URL] - return ok - }) + } + + var plan nodepool.Plan + if selector, ok := h.NodePlanner.(capabilitySessionPlannerV3); ok && eligible != nil { + plan = selector.PlanSessionWith(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps, eligible) + } else { + plan = h.NodePlanner.PlanSession(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps) + } + if localEgress { + // Compatibility fallback for a custom planner that has not learned the + // exact local-egress reservation method. The proxy may have been selected + // internally, but it is never exposed as client media authority. + plan.ProxyNode = nil + } + return plan } // validateAdvertisedTransformationsV3 verifies that every server-executed @@ -612,6 +683,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr if result.Plan == nil { return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "The server produced no playback plan."} } + r = withHeaderAuthenticatedMediaV3(r, req.ClientFeatures) if checker, ok := h.sessionMgr.(transcodePermissionChecker); ok && (result.PlayMethod == playback.PlayTranscode || result.TranscodeAudio) { if err := checker.CheckTranscodingAllowed(r.Context(), userID, result.PlayMethod == playback.PlayTranscode); err != nil { reason := "transcoding_disabled" @@ -744,7 +816,7 @@ func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback. return h.prepareIdentityTransportV3(r, session, file, result, timeline) } if h.NodePlanner != nil { - plan := h.planNodeSessionV3(r.Context(), session, result) + plan := h.planNodeSessionV3(r.Context(), session, result, headerAuthenticatedMediaV3(r)) if plan.TranscodeNode != nil { transformations, err := h.remoteTransformationsV3(r.Context(), plan.TranscodeNode.URL) if err == nil { @@ -858,11 +930,26 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p routeSession.TargetAudioBitrateKbps = result.TargetAudioBitrateKbps routeSession.RemuxDVMode = remuxDVModeForPlanV3(result.Plan) - proxyNode, proxyErr := h.planIdentityProxyV3(r, session.ID, result) - if proxyErr != nil { - return preparedTransportV3{}, proxyErr + var proxyNode *nodepool.Node + if headerAuthenticatedMediaV3(r) { + // Proxy identity routes authenticate with a signed token in the URL path. + // Keep this negotiated mode on the authenticated API origin instead, so + // no client-visible URL can carry or disclose that credential. + if localErr := h.refuseLocalIdentityWorkV3(r, result); localErr != nil { + return preparedTransportV3{}, localErr + } + } else { + var proxyErr *transportErrorV3 + proxyNode, proxyErr = h.planIdentityProxyV3(r, session.ID, result) + if proxyErr != nil { + return preparedTransportV3{}, proxyErr + } + } + streamURL := fmt.Sprintf("/stream/%s", routeSession.ID) + servedByProxy := false + if !headerAuthenticatedMediaV3(r) { + streamURL, servedByProxy = h.identityStreamURLV3(&routeSession, file, proxyNode) } - streamURL, servedByProxy := h.identityStreamURLV3(&routeSession, file, proxyNode) releaseProxyReservation := func() { if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { releaser.ReleaseSession(session.ID) @@ -1255,8 +1342,11 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play return preparedTransportV3{}, transportErr } } - card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) - url := appendStreamToken(fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID), h.signSessionToken(card)) + url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) + if !headerAuthenticatedMediaV3(r) { + card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) + url = appendStreamToken(url, h.signSessionToken(card)) + } committed := false previousNodeURL := session.TranscodeNodeURL previousTransportID := remoteTransportID(session) @@ -1324,13 +1414,16 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla h.tm.StopRemoteTranscode(transportID, node.URL) return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "The selected transcode node rejected the playback transport.", retryable: true} } - hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) - card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) - url := h.buildProxyManifestURL(card, nodePlan.ProxyNode) + url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) + if !headerAuthenticatedMediaV3(r) { + hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) + card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) + url = h.buildProxyManifestURL(card, nodePlan.ProxyNode) + } // buildProxyManifestURL only returns an absolute proxy URL when a proxy was // planned and the token could be signed; otherwise the client fetches the // manifest from this server and the local liveness path applies. - servedByProxy := nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") + servedByProxy := !headerAuthenticatedMediaV3(r) && nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") committed := false previousNodeURL := session.TranscodeNodeURL previousTransportID := remoteTransportID(session) @@ -1386,7 +1479,30 @@ func sourceVideoTranscodeFactsV3(file *models.MediaFile, result playback.Planner } func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3) playback.SessionStreamState { - state := playback.SessionStreamState{PlayMethod: result.PlayMethod, BasePlayMethod: result.PlayMethod, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), TranscodeAudio: result.TranscodeAudio, RemuxDVMode: remuxDVModeForPlanV3(result.Plan), TranscodeNodeURL: transport.nodeURL, TranscodeTransportID: transport.transportID, TranscodeRouteSet: true, ClientIP: clientip.FromContext(ctx), ClientName: session.ClientName, ClientVersion: session.ClientVersion, ClientUserAgent: session.ClientUserAgent, StreamBitrateKbps: result.TargetBitrateKbps, TargetVideoCodec: result.TargetVideoCodec, TargetAudioCodec: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, TargetResolution: result.TargetResolution, SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn} + state := playback.SessionStreamState{ + PlayMethod: result.PlayMethod, + BasePlayMethod: result.PlayMethod, + AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), + TranscodeAudio: result.TranscodeAudio, + RemuxDVMode: remuxDVModeForPlanV3(result.Plan), + TranscodeNodeURL: transport.nodeURL, + TranscodeTransportID: transport.transportID, + TranscodeRouteSet: true, + RequireMediaAuthorization: headerAuthenticatedMediaContextV3(ctx), + MediaAuthorizationSet: true, + ClientIP: clientip.FromContext(ctx), + ClientName: session.ClientName, + ClientVersion: session.ClientVersion, + ClientUserAgent: session.ClientUserAgent, + StreamBitrateKbps: result.TargetBitrateKbps, + TargetVideoCodec: result.TargetVideoCodec, + TargetAudioCodec: result.TargetAudioCodec, + TargetAudioChannels: result.TargetAudioChannels, + TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, + TargetResolution: result.TargetResolution, + SubtitleTrackIndex: result.SubtitleTransportTrackIndex, + SubtitleBurnIn: result.SubtitleBurnIn, + } if result.Plan != nil && (result.Plan.Delivery == playback.DeliveryTranscodeHLSV3 || result.Plan.Delivery == playback.DeliveryRemuxHLSV3) { state.SegmentDuration = 2 } @@ -1616,6 +1732,13 @@ func (h *PlaybackHandler) HandleReplanPlaybackV3(w http.ResponseWriter, r *http. if req.ClientFeatures == nil { req.ClientFeatures = append([]string(nil), record.NormalizedRequest.ClientFeatures...) } + // Media authentication is fixed at start. Neither an omitted/empty feature + // list nor a later opt-in can switch modes mid-attempt: a legacy URL from an + // earlier plan may remain usable until its signed recipe expires, so allowing + // legacy-to-header-auth upgrades would leave two different security contracts + // alive for the same session. Stop/start is the explicit mode boundary. + headerAuthenticatedAttempt := playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + req.ClientFeatures = pinHeaderAuthenticatedMediaFeatureV3(req.ClientFeatures, headerAuthenticatedAttempt) if err := req.Validate(); err != nil { writeError(w, http.StatusBadRequest, "bad_request", "Invalid replan request") return @@ -2056,7 +2179,10 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte } artifactRecipe = frozenRecipe } - transportReused := trackChange && h.hasActiveHLSTransportV3(session) && sidecarOnlyHLSReplanV3(record, result.Plan, artifactRecipe, req.ClientPlaybackContext.Output.OutputContextID) + currentHeaderAuthenticatedMedia := playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + nextHeaderAuthenticatedMedia := playback.HasFeatureV3(start.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + transportReused := currentHeaderAuthenticatedMedia == nextHeaderAuthenticatedMedia && trackChange && h.hasActiveHLSTransportV3(session) && sidecarOnlyHLSReplanV3(record, result.Plan, artifactRecipe, req.ClientPlaybackContext.Output.OutputContextID) + r = withHeaderAuthenticatedMediaV3(r, start.ClientFeatures) var transport preparedTransportV3 if transportReused { // A sidecar selection changes the plan and subtitle artifact, but it does diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 525c44164..6edde656f 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -511,6 +511,119 @@ func TestHandleStartPlaybackV3ReturnsExecutableDirectPlan(t *testing.T) { } } +func TestHandleStartPlaybackV3NegotiatesHeaderAuthenticatedDirectAndSubtitleURLs(t *testing.T) { + for _, test := range []struct { + name string + optIn bool + wantStream bool + }{ + {name: "opted-in URLs carry no playback credential", optIn: true}, + {name: "legacy URL keeps restart token", wantStream: true}, + } { + t.Run(test.name, func(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.ExternalSubtitles = []models.ExternalSubtitle{{Path: writePlaybackTestMediaFile(t, "movie.eng.srt"), Language: "eng", Format: "srt"}} + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-stream-signing-secret" + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + start := v3HandlerStartRequest() + if test.optIn { + start.ClientFeatures = append(start.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + } + subtitleIndex := 0 + start.SubtitleTrackID = playback.TrackIDV3(file.ID, "subtitle", subtitleIndex) + start.SubtitleTrackIndex = &subtitleIndex + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, start))).WithContext(newAuthorizedPlaybackContext())) + + var response playback.DecisionResponseV3 + if rr.Code != http.StatusCreated || json.Unmarshal(rr.Body.Bytes(), &response) != nil || response.PlaybackPlan == nil { + t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String()) + } + streamURL, err := url.Parse(response.PlaybackPlan.Stream.URL) + if err != nil { + t.Fatal(err) + } + if streamURL.IsAbs() || streamURL.Path != "/stream/"+response.SessionID { + t.Fatalf("stream URL = %q, want API-local session route", response.PlaybackPlan.Stream.URL) + } + if got := streamURL.Query().Get(streamTokenParam); (got != "") != test.wantStream { + t.Fatalf("stream token present = %v for URL %q, want %v", got != "", response.PlaybackPlan.Stream.URL, test.wantStream) + } + if len(response.PlaybackPlan.Stream.Headers) != 0 { + t.Fatalf("plan persisted bearer material in headers: %#v", response.PlaybackPlan.Stream.Headers) + } + + artifact := response.PlaybackPlan.Subtitle.Artifact + if artifact == nil || len(response.PlaybackPlan.Subtitle.Inventory) != 1 { + t.Fatalf("subtitle contract = %#v", response.PlaybackPlan.Subtitle) + } + for _, raw := range []string{artifact.URL, response.PlaybackPlan.Subtitle.Inventory[0].URL} { + parsed, parseErr := url.Parse(raw) + if parseErr != nil || parsed.IsAbs() || parsed.Query().Get(streamTokenParam) != "" || !strings.HasPrefix(parsed.Path, "/stream/"+response.SessionID+"/subtitles/") { + t.Fatalf("subtitle URL = %q, want tokenless API-local route (parse error %v)", raw, parseErr) + } + } + if test.optIn && !playback.HasFeatureV3(response.ServerFeatures, playback.FeatureHeaderAuthenticatedMediaV3) { + t.Fatalf("server features = %v, want %q", response.ServerFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + } + }) + } +} + +func TestHandleReplanPlaybackV3CannotDowngradeHeaderAuthenticatedAttempt(t *testing.T) { + file := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-stream-signing-secret" + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + start := v3HandlerStartRequest() + start.ClientFeatures = append(start.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, start))).WithContext(newAuthorizedPlaybackContext())) + var started playback.DecisionResponseV3 + if startRR.Code != http.StatusCreated || json.Unmarshal(startRR.Body.Bytes(), &started) != nil || started.PlaybackPlan == nil { + t.Fatalf("start status=%d body=%s", startRR.Code, startRR.Body.String()) + } + + nextContext := start.ClientPlaybackContext + nextContext.Output.OutputContextID = "route-2" + replanned := postPlaybackReplanV3(t, handler, started.SessionID, playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + ClientFeatures: []string{}, // explicit attempted downgrade + Operation: playback.ReplanOperationOutputChangeV3, + PlaybackAttemptID: start.PlaybackAttemptID, + ReplanRequestID: "header-auth-replan-0001", + FailedPlanID: started.PlaybackPlan.PlanID, + PlanAttemptID: "header-auth-attempt-0001", + PlanAttemptKey: started.PlaybackPlan.PlanAttemptKey, + AttemptCount: 1, + PositionSeconds: 12, + SelectedTracks: started.PlaybackPlan.SelectedTracks, + Capabilities: start.Capabilities, + ClientPlaybackContext: nextContext, + }) + if replanned.PlaybackPlan == nil { + t.Fatalf("replan = %#v", replanned) + } + parsed, err := url.Parse(replanned.PlaybackPlan.Stream.URL) + if err != nil || parsed.IsAbs() || parsed.Query().Get(streamTokenParam) != "" { + t.Fatalf("replan URL = %q, want tokenless API-local route (parse error %v)", replanned.PlaybackPlan.Stream.URL, err) + } + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if !playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) { + t.Fatalf("durable client features = %v, secure transport mode was downgraded", record.NormalizedRequest.ClientFeatures) + } +} + // The inventory is the authoritative subtitle menu, so it has to be fetchable // before the user has picked anything. A start that resolves to `off` still // publishes session-scoped URLs on every sidecar entry; gating them on the @@ -2395,6 +2508,72 @@ func TestPrepareTransportV3RequiresRemoteManifestReadiness(t *testing.T) { } } +func TestPrepareTransportV3KeepsHeaderAuthenticatedRemoteHLSBehindAPI(t *testing.T) { + for _, test := range []struct { + name string + features []string + wantProxy bool + }{ + {name: "header authenticated", features: []string{playback.FeatureHeaderAuthenticatedMediaV3}}, + {name: "legacy proxy URL", wantProxy: true}, + } { + t.Run(test.name, func(t *testing.T) { + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/hw-capabilities": + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}, + {Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{Status: "started"}) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-stream-signing-secret" + handler.NodePlanner = staticNodePlannerV3{plan: nodepool.Plan{ + TranscodeNode: &nodepool.Node{URL: remote.URL}, + ProxyNode: &nodepool.Node{URL: "http://proxy.example"}, + }} + plan := &playback.PlanV3{ + PlanID: "plan:remote-header-auth", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{ + {Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}, + {Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}, + }, + } + request := withHeaderAuthenticatedMediaV3(httptest.NewRequest(http.MethodPost, "/", nil), test.features) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-remote-auth", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + if transportErr != nil { + t.Fatalf("prepare remote HLS: %v", transportErr) + } + defer transport.rollback() + + parsed, err := url.Parse(transport.url) + if err != nil { + t.Fatal(err) + } + if test.wantProxy { + if !parsed.IsAbs() || parsed.Host != "proxy.example" || !strings.HasPrefix(parsed.Path, "/stream/transcode/") { + t.Fatalf("legacy HLS URL = %q, want signed proxy route", transport.url) + } + } else if parsed.IsAbs() || parsed.Path != "/playback/transcode/session-remote-auth/master.m3u8" || parsed.RawQuery != "" || strings.Contains(transport.url, "proxy.example") { + t.Fatalf("header-authenticated HLS URL = %q, want tokenless API-local manifest", transport.url) + } + if transport.nodeURL != remote.URL { + t.Fatalf("remote executor = %q, want %q behind API facade", transport.nodeURL, remote.URL) + } + }) + } +} + func TestPrepareTransportV3SendsResolvedCopyAnchorToRemoteExecutor(t *testing.T) { var startRequest transcodenode.TranscodeStartRequest remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -3784,6 +3963,49 @@ func TestPrepareTransportV3RoutesProgressiveRemuxThroughProxyNodeWithSeekAndDV(t } } +func TestPrepareTransportV3NegotiatesHeaderAuthenticatedProgressiveRemuxURL(t *testing.T) { + for _, test := range []struct { + name string + features []string + wantToken bool + }{ + {name: "header authenticated", features: []string{playback.FeatureHeaderAuthenticatedMediaV3}}, + {name: "legacy", wantToken: true}, + } { + t.Run(test.name, func(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-stream-signing-secret" + stubCopySeekAnchorV3(handler) + plan := identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3) + plan.EffectiveMediaFileID = 42 + plan.Timeline = playback.TimelineV3{SourceStartSeconds: 39.5} + request := withHeaderAuthenticatedMediaV3(httptest.NewRequest(http.MethodPost, "/", nil), test.features) + + transport, transportErr := handler.prepareTransportV3( + request, + &playback.Session{ID: "session-remux-auth", UserID: 7, ProfileID: "profile-1", MediaFileID: 42}, + v3HandlerFixtureFile(t), + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, + ) + if transportErr != nil { + t.Fatalf("prepare remux: %v", transportErr) + } + defer transport.rollback() + + parsed, err := url.Parse(transport.url) + if err != nil { + t.Fatal(err) + } + if parsed.IsAbs() || parsed.Path != "/stream/session-remux-auth" || parsed.Query().Get("seek") != "39.5" { + t.Fatalf("remux URL = %q, want API-local seek route", transport.url) + } + if got := parsed.Query().Get(streamTokenParam); (got != "") != test.wantToken { + t.Fatalf("remux stream token present = %v for URL %q, want %v", got != "", transport.url, test.wantToken) + } + }) + } +} + func TestPrepareTransportV3FallsBackLocallyWithoutEligibleProxy(t *testing.T) { handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) handler.JWTSecret = "test-secret" diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go index 8677f4ed1..d8d7d16f4 100644 --- a/internal/api/handlers/playback_v3_union_test.go +++ b/internal/api/handlers/playback_v3_union_test.go @@ -150,13 +150,13 @@ func TestPlanNodeSessionV3PrefersCapabilityMatchingNode(t *testing.T) { {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, }, } - selected := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-hetero"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode}) + selected := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-hetero"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode}, false) if selected.TranscodeNode == nil || selected.TranscodeNode.URL != capable.URL { t.Fatalf("capability-requiring plan selected %+v, want the capable node", selected.TranscodeNode) } free := &playback.PlanV3{PlanID: "plan:copy", Delivery: playback.DeliveryRemuxHLSV3, Transformations: []playback.TransformationV3{}} - loadBased := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-copy"}, playback.PlannerResultV3{Plan: free, PlayMethod: playback.PlayRemux}) + loadBased := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-copy"}, playback.PlannerResultV3{Plan: free, PlayMethod: playback.PlayRemux}, false) if loadBased.TranscodeNode == nil || loadBased.TranscodeNode.URL != incapable.URL { t.Fatalf("transformation-free plan selected %+v, want load-based selection", loadBased.TranscodeNode) } diff --git a/internal/api/router.go b/internal/api/router.go index 12c347a2b..3f8cd5fdc 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2654,9 +2654,9 @@ func NewRouter(deps Dependencies) chi.Router { r.Route("/playback", func(r chi.Router) { r.Get("/capability", playbackHandler.HandlePlaybackCapabilityV3) - // HLS transcode delivery — no profile auth needed; - // session ID (UUID) serves as the access token, same - // pattern as /stream/{session_id}. + // HLS transcode delivery. Legacy sessions treat the UUID + // as a bearer capability; negotiated V3 sessions require + // the authenticated owner inside the handler. r.Get("/transcode/{session_id}/master.m3u8", playbackHandler.HandleGetTranscodeManifest) r.Get("/transcode/{session_id}/segment/{name}", playbackHandler.HandleGetTranscodeSegment) diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 112c7bfcf..fb272720f 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -231,6 +231,36 @@ func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTr return plan } +// PlanTranscodeSessionWithLocalEgress selects and reserves only a transcode +// node. The API server remains the client-facing media endpoint and relays the +// selected node's manifest and segments, so no proxy node is needed or charged +// against its job/bandwidth budget. This is intentionally separate from +// PlanSessionWith: its normal grouped-node policy assumes the client talks to a +// selected proxy directly. +func (p *Planner) PlanTranscodeSessionWithLocalEgress(sessionID, currentTranscodeURL string, eligible func(*Node) bool) Plan { + if p == nil || p.transcodes == nil || sessionID == "" { + return Plan{} + } + p.mu.Lock() + defer p.mu.Unlock() + + now := p.now() + p.pruneReservations(now) + delete(p.reserved, sessionID) + + transcodes := p.transcodes.Nodes() + groupHealthy := groupHealth(nil, transcodes) + if eligible != nil { + transcodes = filterNodes(transcodes, eligible) + } + node := p.pickLocalEgressTranscode(transcodes, groupHealthy, currentTranscodeURL, now) + if node == nil { + return Plan{} + } + p.reserved[sessionID] = &reservation{transcodeURL: node.URL, createdAt: now} + return Plan{TranscodeNode: node} +} + // filterNodes returns the nodes accepted by keep, preserving pool order so // round-robin cursors stay meaningful across selections. func filterNodes(nodes []*Node, keep func(*Node) bool) []*Node { @@ -371,6 +401,33 @@ func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[st return current } +// pickLocalEgressTranscode applies the transcode half of normal session +// admission without requiring a healthy proxy partner. The API server is the +// egress hop for this route, so unrelated proxy health and capacity must not +// suppress an otherwise healthy transcode executor. +func (p *Planner) pickLocalEgressTranscode(transcodes []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { + var best, current *Node + for _, node := range transcodes { + if node == nil || !node.Healthy || !node.Enabled || !p.underCap(node, now) || + node.Group != nil && !groupHealthy[*node.Group] { + continue + } + if node.URL == currentURL { + current = node + } + if best == nil || p.effectiveJobs(node, now) < p.effectiveJobs(best, now) { + best = node + } + } + if current == nil || best == nil || current == best { + return best + } + if p.effectiveJobs(best, now)+2 <= p.effectiveJobs(current, now) { + return best + } + return current +} + // transcodeEligible reports whether a transcode node may take a new session: // it must be healthy and under cap, and a grouped node additionally requires // its whole group healthy and — when the group contains proxies — at least diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index 47a06b8f4..85e213aec 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -84,6 +84,35 @@ func TestPlanSessionWithRestrictsEligibleTranscodeNodes(t *testing.T) { } } +func TestPlanTranscodeSessionWithLocalEgressDoesNotUseProxyCapacity(t *testing.T) { + group := strPtr("rack-a") + proxy := proxyNode(1, "http://proxy-a", group) + proxy.Healthy = false + transcode := transcodeNode(2, "http://tc-a", group, 0) + f := newFixture([]*Node{proxy}, []*Node{transcode}) + + plan := f.planner.PlanTranscodeSessionWithLocalEgress("s-local-egress", "", func(node *Node) bool { + return node != nil && node.URL == transcode.URL + }) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != transcode.URL { + t.Fatalf("local-egress plan = %#v, want healthy transcode despite unrelated proxy health", plan) + } + if plan.ProxyNode != nil { + t.Fatalf("local-egress plan exposed proxy %#v", plan.ProxyNode) + } + reservation := f.planner.reserved["s-local-egress"] + if reservation == nil || reservation.transcodeURL != transcode.URL || reservation.proxyURL != "" || reservation.kbps != 0 { + t.Fatalf("local-egress reservation = %#v, want transcode-only accounting", reservation) + } + + if none := f.planner.PlanTranscodeSessionWithLocalEgress("s-ineligible", "", func(*Node) bool { return false }); none.TranscodeNode != nil { + t.Fatalf("ineligible local-egress plan selected %#v", none.TranscodeNode) + } + if _, reserved := f.planner.reserved["s-ineligible"]; reserved { + t.Fatal("ineligible local-egress plan left a reservation") + } +} + func TestReleaseSessionDropsProvisionalReservation(t *testing.T) { node := transcodeNode(1, "http://tc-1", nil, 0) node.MaxJobs = intPtr(1) diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index 8263d085c..9242b542e 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -11,25 +11,31 @@ import ( ) const ( - ProtocolV3 = 3 - FeaturePlaybackPlanV3 = "playback_plan_v3" - FeatureNeutralContractV3 = "neutral_playback_v3_contract_v1" - FeatureLayoutPassthrough = "layout_aware_passthrough" - FeatureClientVideoTransforms = "client_video_transformations_v1" - FeatureRouteDiagnostics = "playback_route_diagnostics" - FeatureDeviceQuirksV3 = "device_quirks_v1" - FeatureSeekReanchorV3 = "seek_reanchor_v1" - FeatureOutputChangeV3 = "output_change_v1" - FeatureDirectStreamResumeV3 = "direct_stream_resume_v1" - FeaturePlanSourceDurationV3 = "plan_source_duration_v1" - PlanRecipeVersionV3 = "v3.4" - ClientDV7ToDV81V3 = "client_dv7_to_dv81" - ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" - ClientDVTransformVersionV3 = "1" - ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" - ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" - ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" - DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" + ProtocolV3 = 3 + FeaturePlaybackPlanV3 = "playback_plan_v3" + FeatureNeutralContractV3 = "neutral_playback_v3_contract_v1" + FeatureLayoutPassthrough = "layout_aware_passthrough" + FeatureClientVideoTransforms = "client_video_transformations_v1" + FeatureRouteDiagnostics = "playback_route_diagnostics" + FeatureDeviceQuirksV3 = "device_quirks_v1" + FeatureSeekReanchorV3 = "seek_reanchor_v1" + FeatureOutputChangeV3 = "output_change_v1" + FeatureDirectStreamResumeV3 = "direct_stream_resume_v1" + FeaturePlanSourceDurationV3 = "plan_source_duration_v1" + // FeatureHeaderAuthenticatedMediaV3 advertises an opt-in transport mode + // whose client-visible stream and subtitle URLs carry no signed playback + // credential. A client that sends this token promises to attach its normal + // access-token Authorization header to every media request, including HLS + // manifests/segments and sidecar subtitle/font requests. + FeatureHeaderAuthenticatedMediaV3 = "header_authenticated_media_v1" + PlanRecipeVersionV3 = "v3.4" + ClientDV7ToDV81V3 = "client_dv7_to_dv81" + ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" + ClientDVTransformVersionV3 = "1" + ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" + ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" + ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" + DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" ) // ServerFeaturesV3 returns the complete feature set advertised by protocol-v3 @@ -45,6 +51,7 @@ func ServerFeaturesV3() []string { FeatureSeekReanchorV3, FeatureOutputChangeV3, FeatureDirectStreamResumeV3, + FeatureHeaderAuthenticatedMediaV3, // Advertised so a client can tell "this server does not populate // source.duration_seconds" apart from "this server knows the runtime // is genuinely unknown". Without the distinction both look like an diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index e57b6ae77..5092b428d 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -22,15 +22,16 @@ func TestServerFeaturesV3ReturnsCompleteIndependentSlices(t *testing.T) { first := ServerFeaturesV3() second := ServerFeaturesV3() expected := map[string]struct{}{ - FeaturePlaybackPlanV3: {}, - FeatureNeutralContractV3: {}, - FeatureLayoutPassthrough: {}, - FeatureRouteDiagnostics: {}, - FeatureDeviceQuirksV3: {}, - FeatureSeekReanchorV3: {}, - FeatureOutputChangeV3: {}, - FeatureDirectStreamResumeV3: {}, - FeaturePlanSourceDurationV3: {}, + FeaturePlaybackPlanV3: {}, + FeatureNeutralContractV3: {}, + FeatureLayoutPassthrough: {}, + FeatureRouteDiagnostics: {}, + FeatureDeviceQuirksV3: {}, + FeatureSeekReanchorV3: {}, + FeatureOutputChangeV3: {}, + FeatureDirectStreamResumeV3: {}, + FeatureHeaderAuthenticatedMediaV3: {}, + FeaturePlanSourceDurationV3: {}, } if len(first) != len(expected) { t.Fatalf("server features = %v, want %d entries", first, len(expected)) diff --git a/internal/playback/session.go b/internal/playback/session.go index de74f0c84..86256f676 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -32,6 +32,12 @@ type Session struct { ClientChannel string // opaque reported client distribution channel, when available ClientUserAgent string // trimmed request user agent for the playback session IsJellyfinCompat bool // immutable origin identity for Jellyfin compatibility sessions + // RequireMediaAuthorization distinguishes v3 transports whose session ID is + // only a route identifier from legacy HLS transports where that UUID also + // acts as the bearer capability. It is live-session state by design: secure + // transports carry no reconstruction token and start a fresh attempt after + // an API restart. + RequireMediaAuthorization bool TranscodeNodeURL string // URL of assigned transcode node (empty = local/integrated) TranscodeTransportID string // remote node process identity; empty means session ID @@ -76,26 +82,28 @@ type Session struct { // change after a session is created (audio track, client IP, transcode target, // and reported bitrate). type SessionStreamState struct { - PlayMethod PlayMethod - BasePlayMethod PlayMethod - AudioTrackIndex int - TranscodeAudio bool - RemuxDVMode RemuxDVMode - ClientIP string - ClientName string - ClientVersion string - ClientUserAgent string - StreamBitrateKbps int - TargetResolution string - TargetVideoCodec string - TargetAudioCodec string - TargetAudioChannels int - TargetAudioBitrateKbps int - TargetBitrateKbps int - TranscodeHWAccel string - TranscodeNodeURL string - TranscodeTransportID string - TranscodeRouteSet bool + PlayMethod PlayMethod + BasePlayMethod PlayMethod + AudioTrackIndex int + TranscodeAudio bool + RemuxDVMode RemuxDVMode + ClientIP string + ClientName string + ClientVersion string + ClientUserAgent string + StreamBitrateKbps int + TargetResolution string + TargetVideoCodec string + TargetAudioCodec string + TargetAudioChannels int + TargetAudioBitrateKbps int + TargetBitrateKbps int + TranscodeHWAccel string + TranscodeNodeURL string + TranscodeTransportID string + TranscodeRouteSet bool + RequireMediaAuthorization bool + MediaAuthorizationSet bool // Byte-affecting transcode recipe fields preserved so an offloaded restart // (e.g. audio switch) can rebuild the exact same stream. SubtitleTrackIndex @@ -896,6 +904,9 @@ func applySessionStreamStateLocked(s *Session, state SessionStreamState) { s.TranscodeNodeURL = state.TranscodeNodeURL s.TranscodeTransportID = state.TranscodeTransportID } + if state.MediaAuthorizationSet { + s.RequireMediaAuthorization = state.RequireMediaAuthorization + } s.SubtitleTrackIndex = state.SubtitleTrackIndex s.SubtitleBurnIn = state.SubtitleBurnIn s.SegmentDuration = state.SegmentDuration @@ -909,29 +920,31 @@ func applySessionStreamStateLocked(s *Session, state SessionStreamState) { func snapshotSessionStreamStateLocked(s *Session) SessionStreamState { return SessionStreamState{ - PlayMethod: s.PlayMethod, - BasePlayMethod: s.BasePlayMethod, - AudioTrackIndex: s.AudioTrackIndex, - TranscodeAudio: s.TranscodeAudio, - RemuxDVMode: s.RemuxDVMode, - ClientIP: s.ClientIP, - ClientName: s.ClientName, - ClientVersion: s.ClientVersion, - ClientUserAgent: s.ClientUserAgent, - StreamBitrateKbps: s.StreamBitrateKbps, - TargetResolution: s.TargetResolution, - TargetVideoCodec: s.TargetVideoCodec, - TargetAudioCodec: s.TargetAudioCodec, - TargetAudioChannels: s.TargetAudioChannels, - TargetAudioBitrateKbps: s.TargetAudioBitrateKbps, - TargetBitrateKbps: s.TargetBitrateKbps, - TranscodeHWAccel: s.TranscodeHWAccel, - TranscodeNodeURL: s.TranscodeNodeURL, - TranscodeTransportID: s.TranscodeTransportID, - TranscodeRouteSet: true, - SubtitleTrackIndex: s.SubtitleTrackIndex, - SubtitleBurnIn: s.SubtitleBurnIn, - SegmentDuration: s.SegmentDuration, + PlayMethod: s.PlayMethod, + BasePlayMethod: s.BasePlayMethod, + AudioTrackIndex: s.AudioTrackIndex, + TranscodeAudio: s.TranscodeAudio, + RemuxDVMode: s.RemuxDVMode, + ClientIP: s.ClientIP, + ClientName: s.ClientName, + ClientVersion: s.ClientVersion, + ClientUserAgent: s.ClientUserAgent, + StreamBitrateKbps: s.StreamBitrateKbps, + TargetResolution: s.TargetResolution, + TargetVideoCodec: s.TargetVideoCodec, + TargetAudioCodec: s.TargetAudioCodec, + TargetAudioChannels: s.TargetAudioChannels, + TargetAudioBitrateKbps: s.TargetAudioBitrateKbps, + TargetBitrateKbps: s.TargetBitrateKbps, + TranscodeHWAccel: s.TranscodeHWAccel, + TranscodeNodeURL: s.TranscodeNodeURL, + TranscodeTransportID: s.TranscodeTransportID, + TranscodeRouteSet: true, + RequireMediaAuthorization: s.RequireMediaAuthorization, + MediaAuthorizationSet: true, + SubtitleTrackIndex: s.SubtitleTrackIndex, + SubtitleBurnIn: s.SubtitleBurnIn, + SegmentDuration: s.SegmentDuration, } } @@ -955,6 +968,7 @@ func restoreSessionStreamStateLocked(s *Session, state SessionStreamState) { s.TranscodeHWAccel = state.TranscodeHWAccel s.TranscodeNodeURL = state.TranscodeNodeURL s.TranscodeTransportID = state.TranscodeTransportID + s.RequireMediaAuthorization = state.RequireMediaAuthorization s.SubtitleTrackIndex = state.SubtitleTrackIndex s.SubtitleBurnIn = state.SubtitleBurnIn s.SegmentDuration = state.SegmentDuration diff --git a/internal/playback/testdata/protocol_v3/capability_response.json b/internal/playback/testdata/protocol_v3/capability_response.json index ca2e7674b..da2a89d5d 100644 --- a/internal/playback/testdata/protocol_v3/capability_response.json +++ b/internal/playback/testdata/protocol_v3/capability_response.json @@ -12,6 +12,7 @@ "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", + "header_authenticated_media_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index 5a850578f..8e251e89e 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -5408,6 +5408,7 @@ "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", + "header_authenticated_media_v1", "plan_source_duration_v1" ], "outcome": "adaptation_unavailable", diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json index 2cca07dfd..867778b4e 100644 --- a/internal/playback/testdata/protocol_v3/decision_response.json +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -9,6 +9,7 @@ "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", + "header_authenticated_media_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go index 19ace5069..ac7e80a36 100644 --- a/internal/playback/transcode_manager.go +++ b/internal/playback/transcode_manager.go @@ -326,6 +326,9 @@ const ( SessionLoadFailed // SessionForbidden: a live session exists but belongs to another user. SessionForbidden + // SessionUnauthorized: the live session negotiated authenticated media + // requests, but this request has no authenticated user identity. + SessionUnauthorized ) // LoadOrReconstructSession is the single front door every serve handler uses to diff --git a/internal/playback/transcode_manifest_test.go b/internal/playback/transcode_manifest_test.go index 675aea9f1..06d57ff6a 100644 --- a/internal/playback/transcode_manifest_test.go +++ b/internal/playback/transcode_manifest_test.go @@ -70,6 +70,23 @@ func TestBuildPlaybackManifest_CopyVideoUsesRealManifest(t *testing.T) { if strings.Contains(text, "#EXT-X-PLAYLIST-TYPE:VOD") { t.Fatalf("copy-mode manifest should not be synthetic VOD:\n%s", text) } + + tokenless, err := session.BuildPlaybackManifest("segment/", "") + if err != nil { + t.Fatalf("BuildPlaybackManifest tokenless: %v", err) + } + for _, want := range []string{ + "#EXT-X-MAP:URI=\"segment/init.mp4\"", + "segment/seg_00009.m4s", + "segment/seg_00010.m4s", + } { + if !strings.Contains(string(tokenless), want) { + t.Fatalf("tokenless manifest missing %q:\n%s", want, tokenless) + } + } + if strings.Contains(string(tokenless), "?st=") || strings.Contains(string(tokenless), "?token=") { + t.Fatalf("tokenless manifest propagated a credential query:\n%s", tokenless) + } } func TestBuildPlaybackManifest_AdvancedCopyGenerationKeepsHistoricalRemountPosition(t *testing.T) { From d76226449a0aa0667ef916b8238d6afe6a7d082f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:26:07 -0400 Subject: [PATCH 12/44] feat(playback): negotiate bounded software decode --- docs/architecture/playback-protocol-v3.md | 27 ++- .../fixtures/valid/capability_response.json | 1 + .../v3/fixtures/valid/decision_response.json | 1 + .../playback-v3/v3/replan-request.schema.json | 5 +- .../playback-v3/v3/start-request.schema.json | 5 +- docs/feature-changelog.md | 5 + internal/api/handlers/downloads.go | 22 ++- internal/api/handlers/downloads_test.go | 82 +++++++++ internal/downloads/policy.go | 3 +- internal/playback/capabilities_v3.go | 6 +- internal/playback/protocol_v3.go | 74 +++++--- internal/playback/protocol_v3_test.go | 158 +++++++++++++++++- internal/playback/resolver.go | 53 +++++- internal/playback/resolver_test.go | 93 +++++++++++ .../protocol_v3/capability_response.json | 1 + .../protocol_v3/decision_response.json | 1 + 16 files changed, 490 insertions(+), 47 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index b5ab11a5b..af3677414 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -98,13 +98,13 @@ the document is always the full one: "protocol_versions": [3], "features": ["playback_plan_v3", "neutral_playback_v3_contract_v1", "layout_aware_passthrough", "playback_route_diagnostics", "device_quirks_v1", "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", - "header_authenticated_media_v1", "plan_source_duration_v1"], + "header_authenticated_media_v1", "software_video_decode_v1", "plan_source_duration_v1"], "deliveries": ["original_http", "server_remux_progressive", "server_remux_hls", "server_transcode_hls"], "transformations": [{"name": "audio_to_aac", "executor": "server", "recipe_version": "1", "validated_claims": ["audio_decode"]}] } ``` -The ten feature strings above are the full set this server version advertises: +The eleven feature strings above are the full set this server version advertises: | Feature | What it promises | | --- | --- | @@ -117,6 +117,7 @@ The ten feature strings above are the full set this server version advertises: | `output_change_v1` | The `output_change` intent replan is available; clients must keep the active route when this feature is absent | | `direct_stream_resume_v1` | A direct route may resume mid-file rather than restarting | | `header_authenticated_media_v1` | An opted-in client receives only API-local media URLs without signed credentials in their query or path, and authenticates every media request with its normal Authorization header (§4.1) | +| `software_video_decode_v1` | Exact/platform-attested clients may qualify bounded `video_decode[]` entries with `hardware: false` for direct/original delivery; without the opt-in those evidence tiers remain hardware-only (§3) | | `plan_source_duration_v1` | `source.duration_seconds` is populated when known, so its absence means *unknown* rather than *unsupported* (§5) | That last one is the reason feature detection is a list and not a version @@ -308,14 +309,15 @@ required and are one of: | Tier | Who reports it | What the server does with it | | --- | --- | --- | | `exact` | Android (`MediaCodecList`) | Full strict validation. The server walks `video_decode[]` and requires a hardware entry matching codec, profile, level, bit depth, and every `max_*` bound. Only this tier can earn a validated audio **passthrough** claim. | -| `platform_attested` | Apple (VideoToolbox) | Same walk, but profile and level are **skipped** — the platform attests the codec family rather than enumerating modes. All other bounds still apply. | +| `platform_attested` | Apple (platform-backed Aether decoder stack) | Same walk. Profile and level are **skipped for hardware entries** because the platform cannot enumerate them; explicitly opted-in software entries enforce any profiles/levels the pinned stack supplies. All other bounds still apply. | | `declared` | Web (`isTypeSupported`) | Flat list match only: `codecs_video` / `codecs_video_hardware` membership. No `video_decode[]` walk. | Four rules follow from the table and are easy to get wrong: **A flat claim without backing detail is a refusal, not a pass.** On `exact` and `platform_attested`, if a codec appears in `codecs_video` but no `video_decode[]` -entry names that codec with `hardware: true`, the source is *not* eligible for a +entry names that codec with `hardware: true` (or an explicitly opted-in bounded +software entry), the source is *not* eligible for a direct route. The plan is downgraded and carries the decision reason `evidence_insufficient_for_direct` plus the matching degradation warning, which distinguishes "your evidence didn't support this" from "your device said no." A @@ -329,6 +331,23 @@ If an entry matched the codec but the source exceeded one of its bounds — a the plan is downgraded with no evidence warning. The two cases mean different things and a client should not conflate them in its telemetry. +Software decode remains explicit. At `exact` and `platform_attested`, a +`hardware: false` entry participates only when the request advertises +`software_video_decode_v1`; the same codec, bit-depth, dimension, frame-rate, +bitrate, and supplied profile/level bounds are then enforced. Existing clients therefore retain the +historical hardware-only behavior at these tiers. + +Download creation reuses this bounded vocabulary additively inside `caps`: +`client_features`, `video_evidence`, and `video_decode` have the same meanings +and limits. Opting in requires a non-empty exact/platform-attested detailed +list; malformed partial opt-ins are rejected rather than falling back to flat +claims. This matters when hardware and software decoders have different +ceilings: the flat `max_resolution` remains a coarse device ceiling, while the +detailed entry decides whether a particular original file is safe. Apple keeps +the legacy coarse ceiling at 1080p so older servers fail safely; a detailed +hardware entry may independently preserve a 4K original on a new server. +Legacy flat-only download clients keep the previous resolver behavior. + **An omitted bound means "unconstrained", not "unknown".** Within a `video_decode[]` entry, an empty `profiles`, `levels`, or `bit_depths` list and a zero `max_width` / `max_height` / `max_frame_rate` / `max_bitrate_kbps` are each diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json index da2a89d5d..ba8d3fdca 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json @@ -13,6 +13,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "software_video_decode_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json index 867778b4e..c70258f4c 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json @@ -10,6 +10,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "software_video_decode_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/replan-request.schema.json b/docs/design/schemas/playback-v3/v3/replan-request.schema.json index 3b22567ed..605ac5e55 100644 --- a/docs/design/schemas/playback-v3/v3/replan-request.schema.json +++ b/docs/design/schemas/playback-v3/v3/replan-request.schema.json @@ -206,7 +206,7 @@ }, "capability_evidence": { "type": "string", - "description": "How the capability facts were produced. `exact` is a real platform probe with per-codec profiles/levels/bounds, `platform_attested` a decoder attestation without profile enumeration, `declared` a boolean support statement. Planner strictness follows the tier.", + "description": "How the capability facts were produced. `exact` is a real platform probe with per-codec profiles/levels/bounds, `platform_attested` a platform-backed decoder-stack attestation without profile enumeration, `declared` a boolean support statement. Planner strictness follows the tier; bounded software entries require the explicit software-video feature.", "enum": [ "exact", "platform_attested", @@ -399,7 +399,8 @@ "minimum": 0 }, "hardware": { - "type": "boolean" + "type": "boolean", + "description": "Whether this entry uses a hardware decoder. At strict evidence tiers, false entries participate only when client_features includes software_video_decode_v1." } }, "additionalProperties": true diff --git a/docs/design/schemas/playback-v3/v3/start-request.schema.json b/docs/design/schemas/playback-v3/v3/start-request.schema.json index 002edae54..cbf2ea71e 100644 --- a/docs/design/schemas/playback-v3/v3/start-request.schema.json +++ b/docs/design/schemas/playback-v3/v3/start-request.schema.json @@ -157,7 +157,7 @@ }, "capability_evidence": { "type": "string", - "description": "How the capability facts were produced. `exact` is a real platform probe with per-codec profiles/levels/bounds, `platform_attested` a decoder attestation without profile enumeration, `declared` a boolean support statement. Planner strictness follows the tier.", + "description": "How the capability facts were produced. `exact` is a real platform probe with per-codec profiles/levels/bounds, `platform_attested` a platform-backed decoder-stack attestation without profile enumeration, `declared` a boolean support statement. Planner strictness follows the tier; bounded software entries require the explicit software-video feature.", "enum": [ "exact", "platform_attested", @@ -350,7 +350,8 @@ "minimum": 0 }, "hardware": { - "type": "boolean" + "type": "boolean", + "description": "Whether this entry uses a hardware decoder. At strict evidence tiers, false entries participate only when client_features includes software_video_decode_v1." } }, "additionalProperties": true diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index 320bac333..71accbace 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -1,5 +1,10 @@ # Feature Changelog +## 2026-08-22 + +### Qualify bounded software video decoders without weakening evidence tiers +Playback protocol v3 now advertises the engine-neutral `software_video_decode_v1` opt-in. Exact and platform-attested clients may use bounded `hardware: false` entries from `video_decode[]` for original/direct eligibility only when they send the feature. Download creation accepts the same additive feature, evidence tier, and detailed decoder entries so persistent originals do not flatten a 1080p software claim into a device-wide 4K claim. Existing clients remain hardware-only at strict playback tiers and existing flat download payloads remain unchanged. + ## 2026-08-21 ### Keep signed playback credentials out of client-visible media URLs diff --git a/internal/api/handlers/downloads.go b/internal/api/handlers/downloads.go index ce7fa5302..f76b1b288 100644 --- a/internal/api/handlers/downloads.go +++ b/internal/api/handlers/downloads.go @@ -110,12 +110,15 @@ type downloadRequest struct { // downloadCaps mirrors playback.ClientCapabilities for the request body. type downloadCaps struct { - CodecsVideo []string `json:"codecs_video,omitempty"` - CodecsAudio []string `json:"codecs_audio,omitempty"` - AudioPassthroughCodecs []string `json:"audio_passthrough_codecs,omitempty"` - Containers []string `json:"containers,omitempty"` - MaxResolution string `json:"max_resolution,omitempty"` - HDR bool `json:"hdr,omitempty"` + ClientFeatures []string `json:"client_features,omitempty"` + VideoEvidence playback.CapabilityEvidenceV3 `json:"video_evidence,omitempty"` + CodecsVideo []string `json:"codecs_video,omitempty"` + CodecsAudio []string `json:"codecs_audio,omitempty"` + AudioPassthroughCodecs []string `json:"audio_passthrough_codecs,omitempty"` + Containers []string `json:"containers,omitempty"` + MaxResolution string `json:"max_resolution,omitempty"` + HDR bool `json:"hdr,omitempty"` + VideoDecode []playback.VideoDecodeCapabilityV3 `json:"video_decode,omitempty"` } // patchDownloadRequest is the JSON body for PATCH /downloads/{id}. @@ -276,12 +279,19 @@ func (h *DownloadHandler) HandleCreateDownload(w http.ResponseWriter, r *http.Re } if req.Caps != nil { createReq.Caps = playback.ClientCapabilities{ + ClientFeatures: req.Caps.ClientFeatures, + VideoEvidence: req.Caps.VideoEvidence, CodecsVideo: req.Caps.CodecsVideo, CodecsAudio: req.Caps.CodecsAudio, AudioPassthroughCodecs: req.Caps.AudioPassthroughCodecs, Containers: req.Caps.Containers, MaxResolution: req.Caps.MaxResolution, HDR: req.Caps.HDR, + VideoDecode: req.Caps.VideoDecode, + } + if err := createReq.Caps.NormalizeAndValidateVideoDecode(); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return } } diff --git a/internal/api/handlers/downloads_test.go b/internal/api/handlers/downloads_test.go index 3582b2597..0ce195744 100644 --- a/internal/api/handlers/downloads_test.go +++ b/internal/api/handlers/downloads_test.go @@ -17,6 +17,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/downloads" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtoken" ) @@ -372,6 +373,87 @@ func TestHandleCreateDownloadThreadsQuality(t *testing.T) { } } +func TestHandleCreateDownloadPreservesBoundedSoftwareDecodeEvidence(t *testing.T) { + svc := &fakeDownloadService{created: &downloads.Download{ + ID: "dl1", ContentID: "c1", Status: downloads.StatusQueued, + Format: downloads.FormatOriginal, Quality: downloads.QualityOriginal, + EffectiveQuality: downloads.QualityOriginal, + }} + h := NewDownloadHandler(svc) + body := []byte(`{ + "content_id":"c1", + "quality":"original", + "caps":{ + "client_features":["software_video_decode_v1"], + "video_evidence":"platform_attested", + "codecs_video":["av1"], + "codecs_audio":["aac"], + "containers":["mp4"], + "max_resolution":"2160p", + "video_decode":[{ + "codec":"av1","bit_depths":[8,10],"max_width":1920, + "max_height":1080,"max_frame_rate":60, + "max_bitrate_kbps":40000,"hardware":false + }] + } + }`) + rec := httptest.NewRecorder() + h.HandleCreateDownload(rec, downloadTestRequest(http.MethodPost, "/downloads", body, 7, "", "")) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202 (body: %s)", rec.Code, rec.Body.String()) + } + caps := svc.gotCreateReq.Caps + if caps.VideoEvidence != playback.EvidencePlatformAttestedV3 || + len(caps.ClientFeatures) != 1 || caps.ClientFeatures[0] != playback.FeatureSoftwareVideoDecodeV3 || + len(caps.VideoDecode) != 1 || caps.VideoDecode[0].MaxWidth != 1920 || + caps.VideoDecode[0].Hardware { + t.Fatalf("service received altered software evidence: %+v", caps) + } +} + +func TestHandleCreateDownloadRejectsUnboundedDetailedDecoderInput(t *testing.T) { + svc := &fakeDownloadService{} + h := NewDownloadHandler(svc) + body := []byte(`{ + "content_id":"c1", + "caps":{ + "client_features":["software_video_decode_v1"], + "video_evidence":"platform_attested", + "codecs_video":["av1"], + "video_decode":[{"codec":"av1","max_width":-1,"hardware":false}] + } + }`) + rec := httptest.NewRecorder() + h.HandleCreateDownload(rec, downloadTestRequest(http.MethodPost, "/downloads", body, 7, "", "")) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body: %s)", rec.Code, rec.Body.String()) + } + if svc.gotCreateReq.ContentID != "" { + t.Fatal("invalid detailed decoder evidence reached the download service") + } +} + +func TestHandleCreateDownloadRejectsSoftwareOptInWithoutDetailedEvidence(t *testing.T) { + svc := &fakeDownloadService{} + h := NewDownloadHandler(svc) + body := []byte(`{ + "content_id":"c1", + "caps":{ + "client_features":["software_video_decode_v1"], + "video_evidence":"platform_attested", + "codecs_video":["av1"] + } + }`) + rec := httptest.NewRecorder() + h.HandleCreateDownload(rec, downloadTestRequest(http.MethodPost, "/downloads", body, 7, "", "")) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body: %s)", rec.Code, rec.Body.String()) + } +} + func TestHandleCreateDownloadSeriesThreadsQuality(t *testing.T) { svc := &fakeDownloadService{ series: []*downloads.Download{{ID: "dl1", ContentID: "s1", Format: downloads.FormatOriginal, Quality: downloads.QualityOriginal, EffectiveQuality: downloads.QualityOriginal, Revision: 1}}, diff --git a/internal/downloads/policy.go b/internal/downloads/policy.go index 81752bcf1..83ddd364d 100644 --- a/internal/downloads/policy.go +++ b/internal/downloads/policy.go @@ -469,7 +469,8 @@ func downloadActionDenyError(reasonCode string) error { } func hasCapabilities(caps playback.ClientCapabilities) bool { - return len(caps.CodecsVideo) > 0 || len(caps.CodecsAudio) > 0 || + return caps.VideoEvidence != "" || len(caps.VideoDecode) > 0 || + len(caps.CodecsVideo) > 0 || len(caps.CodecsAudio) > 0 || len(caps.AudioPassthroughCodecs) > 0 || len(caps.Containers) > 0 || caps.MaxResolution != "" || caps.HDR } diff --git a/internal/playback/capabilities_v3.go b/internal/playback/capabilities_v3.go index a673c6015..c81151de8 100644 --- a/internal/playback/capabilities_v3.go +++ b/internal/playback/capabilities_v3.go @@ -128,13 +128,15 @@ func videoEligibleV3(source SourceDescriptorV3, request StartRequestV3) (bool, b // planner); there is no stricter validation to run. return flatClaims, false case EvidenceExactV3, EvidencePlatformAttestedV3: - skipProfileLevel := request.Capabilities.VideoEvidence == EvidencePlatformAttestedV3 + softwareDecodeOptIn := HasFeatureV3(request.ClientFeatures, FeatureSoftwareVideoDecodeV3) matchedCodec := false for _, capability := range request.Capabilities.VideoDecode { - if !strings.EqualFold(capability.Codec, source.VideoCodec) || !capability.Hardware { + if !strings.EqualFold(capability.Codec, source.VideoCodec) || + (!capability.Hardware && !softwareDecodeOptIn) { continue } matchedCodec = true + skipProfileLevel := request.Capabilities.VideoEvidence == EvidencePlatformAttestedV3 && capability.Hardware if !skipProfileLevel { if len(capability.Profiles) > 0 && (source.VideoProfile == "" || !containsFoldV3(capability.Profiles, source.VideoProfile)) { continue diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index 9242b542e..d644deebe 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -22,6 +22,12 @@ const ( FeatureOutputChangeV3 = "output_change_v1" FeatureDirectStreamResumeV3 = "direct_stream_resume_v1" FeaturePlanSourceDurationV3 = "plan_source_duration_v1" + // FeatureSoftwareVideoDecodeV3 lets a strict evidence-tier client opt into + // bounded hardware:false video_decode entries. Without the feature, exact + // and platform_attested retain their historical hardware-only direct-play + // policy, so older clients cannot become software-decoding candidates by + // accident. + FeatureSoftwareVideoDecodeV3 = "software_video_decode_v1" // FeatureHeaderAuthenticatedMediaV3 advertises an opt-in transport mode // whose client-visible stream and subtitle URLs carry no signed playback // credential. A client that sends this token promises to attach its normal @@ -52,6 +58,7 @@ func ServerFeaturesV3() []string { FeatureOutputChangeV3, FeatureDirectStreamResumeV3, FeatureHeaderAuthenticatedMediaV3, + FeatureSoftwareVideoDecodeV3, // Advertised so a client can tell "this server does not populate // source.duration_seconds" apart from "this server knows the runtime // is genuinely unknown". Without the distinction both look like an @@ -248,10 +255,11 @@ type VideoDecodeCapabilityV3 struct { // // - exact: per-codec profiles/levels/bit-depths/bounds from a real platform // probe (Android MediaCodecList). Full strict validation. -// - platform_attested: platform-level decoder attestation without -// profile/level enumeration (Apple VideoToolbox). Codec, resolution, bit -// depth, frame rate, and dynamic range are validated; profile/level -// matching is skipped instead of failing conservative. +// - platform_attested: platform-level decoder-stack attestation without +// profile/level enumeration (for example Apple's VideoToolbox plus a +// pinned software stack). Codec, resolution, bit depth, frame rate, and +// dynamic range are validated; profile/level matching is skipped instead +// of failing conservative. Software entries require an explicit feature. // - declared: boolean support statements (web MediaSource.isTypeSupported). // Copy routes are granted on codec+container+range match from the flat // codec lists; no strict direct claims are made. @@ -267,6 +275,44 @@ func validCapabilityEvidenceV3(v CapabilityEvidenceV3) bool { return v == EvidenceExactV3 || v == EvidencePlatformAttestedV3 || v == EvidenceDeclaredV3 } +func normalizeAndValidateVideoCapabilitiesV3(c *ClientCodecCapabilitiesV3, features []string) error { + if !validCapabilityEvidenceV3(c.VideoEvidence) { + return errors.New("video_evidence is required and must be exact, platform_attested, or declared") + } + if len(features) > 64 || len(c.CodecsVideo) > 64 || len(c.CodecsVideoHardware) > 64 || len(c.VideoDecode) > 64 { + return errors.New("video capability list exceeds supported size") + } + for _, feature := range features { + if len(feature) > 128 { + return errors.New("client feature exceeds supported size") + } + } + for _, values := range [][]string{c.CodecsVideo, c.CodecsVideoHardware} { + for i := range values { + values[i] = strings.ToLower(strings.TrimSpace(values[i])) + if len(values[i]) > 128 { + return errors.New("capability value exceeds supported size") + } + } + } + for i := range c.VideoDecode { + entry := &c.VideoDecode[i] + entry.Codec = strings.ToLower(strings.TrimSpace(entry.Codec)) + if entry.Codec == "" || len(entry.DecoderName) > 128 || entry.MaxWidth < 0 || entry.MaxHeight < 0 || entry.MaxFrameRate < 0 || entry.MaxBitrateKbps < 0 { + return errors.New("invalid detailed video capability") + } + if len(entry.Profiles) > 64 || len(entry.Levels) > 64 || len(entry.BitDepths) > 64 { + return errors.New("detailed video capability exceeds supported size") + } + for _, profile := range entry.Profiles { + if len(profile) > 64 { + return errors.New("detailed video capability value exceeds supported size") + } + } + } + return nil +} + type ClientCodecCapabilitiesV3 struct { // VideoEvidence and AudioEvidence are required closed enums declaring the // provenance of the respective capability facts. @@ -893,8 +939,8 @@ func validateSelectedTrackIdentityV3(kind string, track *TrackIdentityV3) error // payload. features carries the request's top-level client_features — the // only feature-advertisement location in the contract. func validateCapabilitiesV3(c *ClientCodecCapabilitiesV3, ctx *ClientPlaybackContextV3, features []string) error { - if !validCapabilityEvidenceV3(c.VideoEvidence) { - return errors.New("video_evidence is required and must be exact, platform_attested, or declared") + if err := normalizeAndValidateVideoCapabilitiesV3(c, features); err != nil { + return err } if !validCapabilityEvidenceV3(c.AudioEvidence) { return errors.New("audio_evidence is required and must be exact, platform_attested, or declared") @@ -926,7 +972,7 @@ func validateCapabilitiesV3(c *ClientCodecCapabilitiesV3, ctx *ClientPlaybackCon return errors.New("platform_details entry exceeds supported size") } } - for _, values := range [][]string{c.CodecsVideo, c.CodecsVideoHardware, c.CodecsAudio, c.Containers} { + for _, values := range [][]string{c.CodecsAudio, c.Containers} { for i := range values { values[i] = strings.ToLower(strings.TrimSpace(values[i])) if len(values[i]) > 128 { @@ -934,20 +980,6 @@ func validateCapabilitiesV3(c *ClientCodecCapabilitiesV3, ctx *ClientPlaybackCon } } } - for i := range c.VideoDecode { - c.VideoDecode[i].Codec = strings.ToLower(strings.TrimSpace(c.VideoDecode[i].Codec)) - if c.VideoDecode[i].Codec == "" || len(c.VideoDecode[i].DecoderName) > 128 || c.VideoDecode[i].MaxWidth < 0 || c.VideoDecode[i].MaxHeight < 0 || c.VideoDecode[i].MaxFrameRate < 0 || c.VideoDecode[i].MaxBitrateKbps < 0 { - return errors.New("invalid detailed video capability") - } - if len(c.VideoDecode[i].Profiles) > 64 || len(c.VideoDecode[i].Levels) > 64 || len(c.VideoDecode[i].BitDepths) > 64 { - return errors.New("detailed video capability exceeds supported size") - } - for _, profile := range c.VideoDecode[i].Profiles { - if len(profile) > 64 { - return errors.New("detailed video capability value exceeds supported size") - } - } - } for _, hdr := range []*HDRCapabilitiesV3{c.HDRDetails, ctx.Output.HDRDetails} { if err := validateHDRCapabilitiesV3(hdr); err != nil { return err diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index 5092b428d..2f6d7070c 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -31,6 +31,7 @@ func TestServerFeaturesV3ReturnsCompleteIndependentSlices(t *testing.T) { FeatureOutputChangeV3: {}, FeatureDirectStreamResumeV3: {}, FeatureHeaderAuthenticatedMediaV3: {}, + FeatureSoftwareVideoDecodeV3: {}, FeaturePlanSourceDurationV3: {}, } if len(first) != len(expected) { @@ -553,8 +554,8 @@ func TestStartRequestV3RequiresCapabilityEvidenceTiers(t *testing.T) { // The same SDR source must reach a tier-appropriate route for each evidence // tier: exact and platform_attested validate against decode entries (the -// latter without profile/level matching), declared grants the copy route on -// the flat codec+container match alone. +// latter skips profile/level only for hardware attestations), while declared +// grants the copy route on the flat codec+container match alone. func TestPlanPlaybackV3EvidenceTiersReachTierAppropriateRoutes(t *testing.T) { file := detailedFixtureFileV3() file.VideoTracks[0].VideoRange = "SDR" @@ -804,6 +805,159 @@ func TestPlanPlaybackV3TranscodesVP9WithUnknownCodecLevel(t *testing.T) { } } +func TestVideoEligibleV3BoundedSoftwareDecodeRequiresExplicitFeature(t *testing.T) { + source := SourceDescriptorV3{ + VideoCodec: "h264", VideoProfile: "high 10", BitDepth: 10, + Width: 1920, Height: 1080, FrameRate: 24, BitrateKbps: 9_000, + } + req := validStartRequestV3() + req.Capabilities.VideoEvidence = EvidencePlatformAttestedV3 + req.Capabilities.CodecsVideo = []string{"h264"} + req.Capabilities.CodecsVideoHardware = nil + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{ + Codec: "h264", Profiles: []string{"high 10"}, BitDepths: []int{10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: false, + }} + + if eligible, insufficient := videoEligibleV3(source, req); eligible || !insufficient { + t.Fatalf("software entry without opt-in = eligible %v insufficient %v", eligible, insufficient) + } + + req.ClientFeatures = append(req.ClientFeatures, FeatureSoftwareVideoDecodeV3) + if eligible, insufficient := videoEligibleV3(source, req); !eligible || insufficient { + t.Fatalf("opted-in bounded software entry = eligible %v insufficient %v", eligible, insufficient) + } + + source.VideoProfile = "main" + if eligible, insufficient := videoEligibleV3(source, req); eligible || insufficient { + t.Fatalf("software entry outside its exercised profile = eligible %v insufficient %v", eligible, insufficient) + } + + source.VideoProfile = "high 10" + source.Width = 3_840 + if eligible, insufficient := videoEligibleV3(source, req); eligible || insufficient { + t.Fatalf("software entry beyond its width bound = eligible %v insufficient %v", eligible, insufficient) + } +} + +func TestVideoEligibleV3SoftwareEntryCanFollowARejectingHardwareEntryForTheSameCodec(t *testing.T) { + source := SourceDescriptorV3{ + VideoCodec: "h264", VideoProfile: "high 10", BitDepth: 10, + Width: 1920, Height: 1080, FrameRate: 24, BitrateKbps: 9_000, + } + req := validStartRequestV3() + req.Capabilities.VideoEvidence = EvidencePlatformAttestedV3 + req.Capabilities.CodecsVideo = []string{"h264"} + req.Capabilities.CodecsVideoHardware = []string{"h264"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{ + { + Codec: "h264", BitDepths: []int{8}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 25_000, + Hardware: true, + }, + { + Codec: "h264", Profiles: []string{"high 10"}, BitDepths: []int{10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: false, + }, + } + + if eligible, insufficient := videoEligibleV3(source, req); eligible || insufficient { + t.Fatalf("duplicate codec without software opt-in = eligible %v insufficient %v", eligible, insufficient) + } + req.ClientFeatures = append(req.ClientFeatures, FeatureSoftwareVideoDecodeV3) + if eligible, insufficient := videoEligibleV3(source, req); !eligible || insufficient { + t.Fatalf("duplicate codec with software opt-in = eligible %v insufficient %v", eligible, insufficient) + } +} + +func TestPlanPlaybackV3AppleSoftwareEnvelopeSelectsOriginalHTTP(t *testing.T) { + tests := []struct { + name, codec, sourceProfile, claimedProfile, resolution, frameRate string + bitDepth, width, height, maxFrameRate, bitrate, maxBitrate int + }{ + {name: "h264 high 10", codec: "h264", sourceProfile: "High 10", claimedProfile: "high 10", resolution: "1080p", frameRate: "24000/1001", bitDepth: 10, width: 1920, height: 1080, maxFrameRate: 30, bitrate: 9_000, maxBitrate: 10_000}, + {name: "av1 main 10", codec: "av1", sourceProfile: "Main", claimedProfile: "main", resolution: "1080p", frameRate: "24", bitDepth: 10, width: 1920, height: 1080, maxFrameRate: 30, bitrate: 2_500, maxBitrate: 3_000}, + {name: "vp9 profile 0", codec: "vp9", sourceProfile: "Profile 0", claimedProfile: "profile 0", resolution: "1080p", frameRate: "24", bitDepth: 8, width: 1920, height: 1080, maxFrameRate: 30, bitrate: 2_600, maxBitrate: 3_000}, + {name: "mpeg2 main interlaced", codec: "mpeg2video", sourceProfile: "Main", claimedProfile: "main", resolution: "480p", frameRate: "30.303", bitDepth: 8, width: 720, height: 480, maxFrameRate: 31, bitrate: 6_200, maxBitrate: 7_000}, + {name: "vc1 advanced", codec: "vc1", sourceProfile: "Advanced", claimedProfile: "advanced", resolution: "1080p", frameRate: "24", bitDepth: 8, width: 1920, height: 1080, maxFrameRate: 30, bitrate: 31_200, maxBitrate: 32_000}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file := detailedFixtureFileV3() + file.CodecVideo = test.codec + file.Resolution = test.resolution + file.Bitrate = test.bitrate + file.VideoTracks[0] = models.VideoTrack{ + Codec: test.codec, Profile: test.sourceProfile, Width: test.width, Height: test.height, + FrameRate: test.frameRate, Bitrate: test.bitrate, BitDepth: test.bitDepth, + VideoRange: "SDR", VideoRangeType: "SDR", + } + + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureSoftwareVideoDecodeV3) + req.Capabilities.VideoEvidence = EvidencePlatformAttestedV3 + req.Capabilities.CodecsVideo = []string{test.codec} + req.Capabilities.CodecsVideoHardware = nil + req.Capabilities.Containers = []string{"mkv"} + req.Capabilities.MaxResolution = "1080p" + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{ + Codec: test.codec, Profiles: []string{test.claimedProfile}, BitDepths: []int{test.bitDepth}, + MaxWidth: test.width, MaxHeight: test.height, MaxFrameRate: float64(test.maxFrameRate), + MaxBitrateKbps: test.maxBitrate, Hardware: false, + }} + original := req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + original.Containers = []string{"mkv"} + original.VideoCodecs = []string{test.codec} + original.AudioDecodeCodecs = []string{"aac"} + req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = original + + result := PlanPlaybackV3(PlannerInputV3{ + Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 { + t.Fatalf("final Apple software envelope = %s", ExplainPlannerResultV3(result)) + } + }) + } +} + +func TestPlanPlaybackV3ApplePackagedCodecListsExcludeSoftwareOnlyCopy(t *testing.T) { + file := detailedFixtureFileV3() + file.CodecVideo = "vp9" + file.Resolution = "1080p" + file.Bitrate = 2_600 + file.VideoTracks[0] = models.VideoTrack{Codec: "vp9", Profile: "Profile 0", Width: 1920, Height: 1080, FrameRate: "24", Bitrate: 2_600, BitDepth: 8, VideoRange: "SDR", VideoRangeType: "SDR"} + + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureSoftwareVideoDecodeV3) + req.Capabilities.VideoEvidence = EvidencePlatformAttestedV3 + req.Capabilities.CodecsVideo = []string{"vp9"} + req.Capabilities.CodecsVideoHardware = nil + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "vp9", Profiles: []string{"profile 0"}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 30, MaxBitrateKbps: 3_000, Hardware: false}} + original := req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + original.Enabled = false + req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = original + for _, delivery := range []string{DeliveryClassProgressiveV3, DeliveryClassHLSV3} { + packaged := req.ClientPlaybackContext.Deliveries[delivery] + packaged.VideoCodecs = []string{"h264"} + packaged.AudioDecodeCodecs = []string{"aac"} + req.ClientPlaybackContext.Deliveries[delivery] = packaged + } + + result := PlanPlaybackV3(PlannerInputV3{ + Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3(), + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryTranscodeHLSV3 || result.TargetVideoCodec != "h264" { + t.Fatalf("software-only source leaked into a packaged copy route: %s", ExplainPlannerResultV3(result)) + } +} + func TestPlanPlaybackV3BlocksUltrawide4KTranscode(t *testing.T) { file := detailedFixtureFileV3() file.Resolution = "2160p" diff --git a/internal/playback/resolver.go b/internal/playback/resolver.go index 0c652416a..7ff48a81a 100644 --- a/internal/playback/resolver.go +++ b/internal/playback/resolver.go @@ -3,6 +3,7 @@ package playback import ( + "errors" "slices" "github.com/Silo-Server/silo-server/internal/access" @@ -26,12 +27,39 @@ const ( // instead of downmixing+re-encoding to AAC. Distinct from CodecsAudio, which // describes what the client itself can decode. type ClientCapabilities struct { - CodecsVideo []string `json:"codecs_video"` // e.g., h264, hevc, av1 - CodecsAudio []string `json:"codecs_audio"` // e.g., aac, opus, flac - AudioPassthroughCodecs []string `json:"audio_passthrough_codecs,omitempty"` - Containers []string `json:"containers"` // e.g., mp4, webm, mkv - MaxResolution string `json:"max_resolution"` // e.g., 1080p, 2160p - HDR bool `json:"hdr"` + ClientFeatures []string `json:"client_features,omitempty"` + VideoEvidence CapabilityEvidenceV3 `json:"video_evidence,omitempty"` + CodecsVideo []string `json:"codecs_video"` // e.g., h264, hevc, av1 + CodecsAudio []string `json:"codecs_audio"` // e.g., aac, opus, flac + AudioPassthroughCodecs []string `json:"audio_passthrough_codecs,omitempty"` + Containers []string `json:"containers"` // e.g., mp4, webm, mkv + MaxResolution string `json:"max_resolution"` // e.g., 1080p, 2160p + HDR bool `json:"hdr"` + VideoDecode []VideoDecodeCapabilityV3 `json:"video_decode,omitempty"` +} + +// NormalizeAndValidateVideoDecode applies the protocol-v3 detailed decoder +// limits to additive capability payloads such as download creation. Legacy +// flat-only payloads remain valid and unchanged. +func (c *ClientCapabilities) NormalizeAndValidateVideoDecode() error { + softwareOptIn := HasFeatureV3(c.ClientFeatures, FeatureSoftwareVideoDecodeV3) + if c.VideoEvidence == "" && len(c.VideoDecode) == 0 && !softwareOptIn { + return nil + } + if (c.VideoEvidence != EvidenceExactV3 && c.VideoEvidence != EvidencePlatformAttestedV3) || len(c.VideoDecode) == 0 { + return errors.New("detailed download video evidence requires exact or platform_attested entries") + } + detailed := ClientCodecCapabilitiesV3{ + VideoEvidence: c.VideoEvidence, + CodecsVideo: c.CodecsVideo, + VideoDecode: c.VideoDecode, + } + if err := normalizeAndValidateVideoCapabilitiesV3(&detailed, c.ClientFeatures); err != nil { + return err + } + c.CodecsVideo = detailed.CodecsVideo + c.VideoDecode = detailed.VideoDecode + return nil } // AdminSettings controls server-side playback constraints. @@ -54,6 +82,17 @@ type PlayDecision struct { func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSettings) *PlayDecision { // Check if client supports the video codec. videoOK := containsStr(caps.CodecsVideo, file.CodecVideo) + detailedVideoEvidence := (caps.VideoEvidence == EvidenceExactV3 || caps.VideoEvidence == EvidencePlatformAttestedV3) && len(caps.VideoDecode) > 0 + if detailedVideoEvidence { + videoOK, _ = videoEligibleV3(SourceDescriptorFromFileV3(file, 0), StartRequestV3{ + ClientFeatures: caps.ClientFeatures, + Capabilities: ClientCodecCapabilitiesV3{ + VideoEvidence: caps.VideoEvidence, + CodecsVideo: caps.CodecsVideo, + VideoDecode: caps.VideoDecode, + }, + }) + } // Audio is considered OK if the client can decode the codec itself OR its // sink can passthrough it. Passthrough lets us stream-copy surround audio // (EAC3/AC3/DTS/TrueHD) to HDMI AVRs instead of re-encoding to stereo AAC. @@ -63,7 +102,7 @@ func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSett containerOK := containsStr(caps.Containers, file.Container) // Check resolution constraint. - if !resolutionFits(file.Resolution, caps.MaxResolution) { + if !detailedVideoEvidence && !resolutionFits(file.Resolution, caps.MaxResolution) { if !settings.TranscodeEnabled { return &PlayDecision{ Method: PlayDirect, diff --git a/internal/playback/resolver_test.go b/internal/playback/resolver_test.go index 4319c05d1..dbe8cfcca 100644 --- a/internal/playback/resolver_test.go +++ b/internal/playback/resolver_test.go @@ -168,6 +168,99 @@ func TestResolver_Transcode_UnsupportedVideoCodec(t *testing.T) { } } +func TestResolver_DownloadSoftwareDecodeIsFeatureGatedAndBounded(t *testing.T) { + file := &models.MediaFile{ + CodecVideo: "av1", CodecAudio: "aac", Container: "mp4", + Resolution: "1080p", Bitrate: 9_000, + VideoTracks: []models.VideoTrack{{ + Codec: "av1", Profile: "Main", Width: 1920, Height: 1080, FrameRate: "24/1", + Bitrate: 9_000, BitDepth: 10, + }}, + } + caps := playback.ClientCapabilities{ + ClientFeatures: []string{playback.FeatureSoftwareVideoDecodeV3}, + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"av1"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + MaxResolution: "2160p", + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", Profiles: []string{"main"}, BitDepths: []int{10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: false, + }}, + } + + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayDirect { + t.Fatalf("opted-in bounded software source = %q, want direct", decision.Method) + } + + withoutFeature := caps + withoutFeature.ClientFeatures = nil + if decision := playback.Resolve(file, withoutFeature, defaultSettings()); decision.Method != playback.PlayTranscode { + t.Fatalf("software source without opt-in = %q, want transcode", decision.Method) + } + + file.Resolution = "2160p" + file.VideoTracks[0].Width = 3840 + file.VideoTracks[0].Height = 2160 + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayTranscode { + t.Fatalf("software source beyond decoder bounds = %q, want transcode", decision.Method) + } +} + +func TestResolver_DetailedHardwareEvidenceOverridesLegacyDownloadCeiling(t *testing.T) { + file := &models.MediaFile{ + CodecVideo: "hevc", CodecAudio: "aac", Container: "mp4", + Resolution: "2160p", Bitrate: 60_000, + VideoTracks: []models.VideoTrack{{ + Codec: "hevc", Profile: "Main 10", Width: 3840, Height: 2160, + FrameRate: "60/1", Bitrate: 60_000, BitDepth: 10, + }}, + } + caps := playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"hevc"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + // Older servers conservatively stop here. The detailed-aware server + // validates the per-decoder 4K hardware bound instead. + MaxResolution: "1080p", + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "hevc", BitDepths: []int{8, 10}, MaxWidth: 3840, + MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 120_000, + Hardware: true, + }}, + } + + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayDirect { + t.Fatalf("detailed 4K hardware source = %q, want direct", decision.Method) + } +} + +func TestResolver_DetailedDownloadEvidenceFailsClosedWhenProbeFactsAreIncomplete(t *testing.T) { + file := &models.MediaFile{ + CodecVideo: "av1", CodecAudio: "aac", Container: "mp4", + Resolution: "1080p", + } + caps := playback.ClientCapabilities{ + ClientFeatures: []string{playback.FeatureSoftwareVideoDecodeV3}, + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"av1"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + MaxResolution: "2160p", + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", BitDepths: []int{8, 10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + }}, + } + + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayTranscode { + t.Fatalf("incomplete strict evidence = %q, want transcode", decision.Method) + } +} + func TestResolver_Transcode_ResolutionExceeds(t *testing.T) { file := &models.MediaFile{ CodecVideo: "h264", CodecAudio: "aac", Container: "mp4", diff --git a/internal/playback/testdata/protocol_v3/capability_response.json b/internal/playback/testdata/protocol_v3/capability_response.json index da2a89d5d..ba8d3fdca 100644 --- a/internal/playback/testdata/protocol_v3/capability_response.json +++ b/internal/playback/testdata/protocol_v3/capability_response.json @@ -13,6 +13,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "software_video_decode_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json index 867778b4e..c70258f4c 100644 --- a/internal/playback/testdata/protocol_v3/decision_response.json +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -10,6 +10,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "software_video_decode_v1", "plan_source_duration_v1" ], "outcome": "playable", From 97b9833737935b1164fe5adb6d9ef39f9331e747 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:36:07 -0400 Subject: [PATCH 13/44] fix(abs): key the login rate limiter on the transport peer clientip.Middleware overwrites r.RemoteAddr with the header-derived viewer address whenever the TCP peer is a trusted proxy, which includes Docker's bridge. Mounting it on the ABS listener therefore defeated the login limiter's deliberate RemoteAddr-only keying: an attacker behind any reverse proxy could rotate X-Forwarded-For and buy a fresh burst bucket per request. The middleware now preserves the pre-overwrite peer address in the request context, and the limiter reads that instead. Anything else that must key on an address a client cannot forge should do the same. Co-Authored-By: Claude Opus 5 (1M context) --- internal/audiobooks/abs/login_ratelimit.go | 19 +++++-- .../audiobooks/abs/login_ratelimit_test.go | 54 +++++++++++++++++++ internal/clientip/middleware.go | 28 ++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 internal/audiobooks/abs/login_ratelimit_test.go diff --git a/internal/audiobooks/abs/login_ratelimit.go b/internal/audiobooks/abs/login_ratelimit.go index f2d0ab04b..ddbda4fae 100644 --- a/internal/audiobooks/abs/login_ratelimit.go +++ b/internal/audiobooks/abs/login_ratelimit.go @@ -7,6 +7,8 @@ import ( "time" "golang.org/x/time/rate" + + "github.com/Silo-Server/silo-server/internal/clientip" ) // loginLimitBurst caps the number of body-creds /login attempts a single @@ -93,11 +95,22 @@ func (l *LoginLimiter) janitor() { } // clientIP returns the rate-limit key for a request. The standalone listener -// is public, so spoofable forwarding headers are deliberately ignored. +// is public, so spoofable forwarding headers are deliberately ignored: the key +// is always the transport peer. +// +// r.RemoteAddr is not that peer once clientip.Middleware has run — it has been +// replaced with the header-derived viewer address, which an attacker fronted by +// any trusted proxy (including Docker's bridge) can rotate per request to get a +// fresh bucket. PeerFromContext carries the pre-overwrite address for exactly +// this case; RemoteAddr is only correct when the middleware is absent. func clientIP(r *http.Request) string { - host, _, err := net.SplitHostPort(r.RemoteAddr) + addr := r.RemoteAddr + if peer := clientip.PeerFromContext(r.Context()); peer != "" { + addr = peer + } + host, _, err := net.SplitHostPort(addr) if err != nil { - return r.RemoteAddr + return addr } return host } diff --git a/internal/audiobooks/abs/login_ratelimit_test.go b/internal/audiobooks/abs/login_ratelimit_test.go new file mode 100644 index 000000000..83e637457 --- /dev/null +++ b/internal/audiobooks/abs/login_ratelimit_test.go @@ -0,0 +1,54 @@ +package abs + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/clientip" +) + +// The ABS listener is public and clientip.Middleware overwrites RemoteAddr with +// a header-derived address whenever the TCP peer is a trusted proxy. If the +// limiter keyed on RemoteAddr it would hand every spoofed X-Forwarded-For its +// own bucket, which is the whole point of the deliberate RemoteAddr-only rule. +func TestClientIPUsesTransportPeerNotResolvedAddress(t *testing.T) { + r := httptest.NewRequest(http.MethodPost, "/login", nil) + r.RemoteAddr = "203.0.113.7" // as clientip.Middleware rewrites it + r = r.WithContext(clientip.SetPeerContext(r.Context(), "172.17.0.1:51234")) + + if got := clientIP(r); got != "172.17.0.1" { + t.Fatalf("clientIP = %q, want the transport peer 172.17.0.1", got) + } +} + +// Without the middleware there is nothing in the context and RemoteAddr is +// still the untouched peer. +func TestClientIPFallsBackToRemoteAddr(t *testing.T) { + r := httptest.NewRequest(http.MethodPost, "/login", nil) + r.RemoteAddr = "198.51.100.9:44321" + + if got := clientIP(r); got != "198.51.100.9" { + t.Fatalf("clientIP = %q, want 198.51.100.9", got) + } +} + +// A rotating forged X-Forwarded-For must not buy fresh burst allowance. +func TestLoginLimiterNotBypassedByForwardedForRotation(t *testing.T) { + limiter := NewLoginLimiter() + defer limiter.Stop() + + allowed := 0 + for i := 0; i < loginLimitBurst+5; i++ { + r := httptest.NewRequest(http.MethodPost, "/login", nil) + // Same attacker, same TCP peer, a different spoofed viewer IP each time. + r.RemoteAddr = "203.0.113." + string(rune('0'+i%10)) + r = r.WithContext(clientip.SetPeerContext(r.Context(), "172.17.0.1:51234")) + if limiter.allow(clientIP(r)) { + allowed++ + } + } + if allowed > loginLimitBurst { + t.Fatalf("limiter allowed %d attempts, want at most the %d burst", allowed, loginLimitBurst) + } +} diff --git a/internal/clientip/middleware.go b/internal/clientip/middleware.go index f89b38aa5..d1a291466 100644 --- a/internal/clientip/middleware.go +++ b/internal/clientip/middleware.go @@ -9,16 +9,28 @@ type contextKey string const clientIPKey contextKey = "client_ip" +// peerIPKey holds the transport-level peer address exactly as the listener saw +// it, before Middleware overwrote RemoteAddr with the resolved client IP. +const peerIPKey contextKey = "peer_addr" + // Middleware returns chi-compatible middleware that resolves the client IP // and stores it in the request context. It also overwrites r.RemoteAddr so // that downstream middleware (e.g. chi's Logger) and any code reading // RemoteAddr directly sees the real client IP instead of the proxy address. +// +// Because the overwrite is destructive, the original transport peer address is +// preserved in the context as well. Anything that must key on an address a +// client cannot forge — rate limiters, abuse controls — has to read +// PeerFromContext rather than RemoteAddr, since a resolved IP is only as +// trustworthy as the forwarding headers it came from. func Middleware(resolver *Resolver) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer := r.RemoteAddr ip := resolver.ClientIP(r) r.RemoteAddr = ip ctx := context.WithValue(r.Context(), clientIPKey, ip) + ctx = context.WithValue(ctx, peerIPKey, peer) next.ServeHTTP(w, r.WithContext(ctx)) }) } @@ -31,6 +43,22 @@ func FromContext(ctx context.Context) string { return ip } +// PeerFromContext returns the transport peer address (host:port, as the +// listener reported it) captured before Middleware overwrote RemoteAddr. +// Returns empty string when the middleware did not run, in which case +// r.RemoteAddr is still the unmodified peer address. +func PeerFromContext(ctx context.Context) string { + addr, _ := ctx.Value(peerIPKey).(string) + return addr +} + +// SetPeerContext stores a transport peer address in the context. Useful for +// testing handlers that depend on the clientip middleware without going +// through the full chain. +func SetPeerContext(ctx context.Context, addr string) context.Context { + return context.WithValue(ctx, peerIPKey, addr) +} + // SetContext stores a client IP in the context. Useful for testing handlers // that depend on the clientip middleware without going through the full chain. func SetContext(ctx context.Context, ip string) context.Context { From ef565d8e517a21eaabcba1a40d62aed389ee9479 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:36:08 -0400 Subject: [PATCH 14/44] fix(jellycompat): key stream telemetry on the upstream playback session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compat attached observations under PlaybackSession.ID while the proxy, nodesessions and playback_sessions_sync all key on playback.Session.ID, and BuildGlobalView merges by exact SessionID string. One Jellyfin viewing therefore showed as two merged sessions — a byte-less compat twin and the proxy record carrying the traffic — and every compat session looked telemetry_only in parity. Compat now attaches only under UpstreamSessionID. A play session does not learn that id until ensureUpstreamPlayback/ensureTranscodeManifest has run, so the pre-side-effect attach is a no-op on a session's first request and the handler attaches again the moment the id exists, still before any byte is written. A provisional key was rejected deliberately: it recreates exactly the ghost session this fixes, and a session whose id did not exist a moment ago cannot have a pending cut against it. Co-Authored-By: Claude Opus 5 (1M context) --- internal/jellycompat/media_routes.go | 2 +- internal/jellycompat/streams.go | 33 ++++++++++++++++++ internal/jellycompat/streamtelemetry.go | 36 ++++++++++++++------ internal/jellycompat/streamtelemetry_test.go | 32 +++++++++++++++-- 4 files changed, 90 insertions(+), 13 deletions(-) diff --git a/internal/jellycompat/media_routes.go b/internal/jellycompat/media_routes.go index 79fef735e..ea3a22ee8 100644 --- a/internal/jellycompat/media_routes.go +++ b/internal/jellycompat/media_routes.go @@ -26,7 +26,7 @@ var jellycompatMediaRoutes = []streamtelemetry.MediaRoute{ func compatRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyJellycompat, Method: method, Pattern: pattern, - Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "compat_play_session", + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "upstream_playback_session", CapRelevant: capRelevant, Enrolled: true, Capture: compatCapture(pattern)} } diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index 780f1be2d..f63bdd7ac 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -93,6 +93,9 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque writeCompatUpstreamError(w, err) return } + // The attach above is a no-op on the first request of a session, which has + // no upstream id yet. Now it does, and no byte has been written. + attachCompatStream(r.Context(), session, playSession, source.FileID) if h.fileResolver == nil { writeError(w, http.StatusInternalServerError, "ServerError", "File resolver not available") @@ -235,6 +238,9 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re writeCompatUpstreamError(w, err) return } + // See HandleVideoStream: the pre-side-effect attach cannot know the + // upstream id on a session's first request, and this is where it exists. + attachCompatStream(r.Context(), session, playSession, source.FileID) failRemoteStart := func() { h.teardownPlaySession(context.WithoutCancel(r.Context()), playSession, nil, nil) } @@ -293,6 +299,13 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re // Ensure the transcode process is running. manifest, err := h.ensureTranscodeManifest(r.Context(), session, playSession.ID, *source) + if err == nil { + // Local-fallback path: the upstream session was minted in here, so this + // is the first point at which the observation can carry the merged view's + // canonical key. No-op when the earlier attach already succeeded. + playSession = h.refreshPlaySession(playSession) + attachCompatStream(r.Context(), session, playSession, source.FileID) + } if err != nil { if errors.Is(err, errTranscode4KDisallowed) { writeError(w, http.StatusForbidden, "Forbidden", "4K video transcoding is disabled on this server") @@ -347,6 +360,13 @@ func (h *PlaybackHandler) HandleHLSManifest(w http.ResponseWriter, r *http.Reque // Ensure the transcode process is running. manifest, err := h.ensureTranscodeManifest(r.Context(), session, playSession.ID, *source) + if err == nil { + // Local-fallback path: the upstream session was minted in here, so this + // is the first point at which the observation can carry the merged view's + // canonical key. No-op when the earlier attach already succeeded. + playSession = h.refreshPlaySession(playSession) + attachCompatStream(r.Context(), session, playSession, source.FileID) + } if err != nil { if errors.Is(err, errTranscode4KDisallowed) { writeError(w, http.StatusForbidden, "Forbidden", "4K video transcoding is disabled on this server") @@ -1427,6 +1447,19 @@ func (h *PlaybackHandler) reviveUpstreamForReport(ctx context.Context, session * return revived } +// refreshPlaySession re-reads a play session from the store so a caller that +// just triggered upstream-session creation sees the minted UpstreamSessionID. +// Returns the original on a miss so callers never have to nil-check. +func (h *PlaybackHandler) refreshPlaySession(current *PlaybackSession) *PlaybackSession { + if current == nil { + return nil + } + if refreshed, ok := h.playbackStore.Get(current.ID); ok && refreshed != nil { + return refreshed + } + return current +} + func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSession *Session, playSessionID string, source PlaybackMediaSource, method string) (*PlaybackSession, error) { playSession, ok := h.playbackStore.Get(playSessionID) if !ok { diff --git a/internal/jellycompat/streamtelemetry.go b/internal/jellycompat/streamtelemetry.go index e44894392..5d7f02aa9 100644 --- a/internal/jellycompat/streamtelemetry.go +++ b/internal/jellycompat/streamtelemetry.go @@ -28,8 +28,26 @@ import ( // request: Session.StreamAppUserID is the numeric silo account id, so compat // sessions land in the same subject space as native and proxy and a per-user // total sums across families. +// +// The canonical key is the UPSTREAM playback session id, never the compat +// PlaybackSession.ID. Every other publisher — the proxy (from the stream token's +// SessionID claim), nodesessions, and playback_sessions_sync — keys on +// playback.Session.ID, and BuildGlobalView merges by exact SessionID string. +// Keying compat on its own play-session id would split one viewing into two +// merged sessions: a byte-less compat twin and the proxy record carrying all the +// traffic, and would make every compat session look telemetry_only in parity. +// +// A compat play session does not learn its upstream id until +// ensureUpstreamPlayback (or ensureTranscodeManifest) has run, so on the very +// first request of a session this is a no-op and the caller attaches again once +// the id exists. Skipping is deliberate: a provisional key would produce exactly +// the ghost session described above, and a brand-new session cannot have a +// pending cut against it because its id did not exist a moment ago. The +// consequence is that a request failing before the upstream session is minted +// lands in the unattributed counters rather than on a session — which is +// correct, since nothing else in the fleet knows that id either. func attachCompatStream(ctx context.Context, session *Session, play *PlaybackSession, mediaFileID int) { - if session == nil { + if session == nil || play == nil || play.UpstreamSessionID == "" { return } attachment := streamtelemetry.Attachment{ @@ -41,15 +59,13 @@ func attachCompatStream(ctx context.Context, session *Session, play *PlaybackSes TokenIssuedAtSource: streamtelemetry.TokenIssuedAtSourceNone, StartedAtSource: streamtelemetry.StartedAtSourceFirstSeen, } - if play != nil { - attachment.SessionID = play.ID - attachment.PlayMethod = play.UpstreamPlayMethod - if !play.CreatedAt.IsZero() { - // P0a established the top-level compat CreatedAt as the source of - // truth for a compat session's start time. - attachment.StartedAt = play.CreatedAt - attachment.StartedAtSource = streamtelemetry.StartedAtSourceSession - } + attachment.SessionID = play.UpstreamSessionID + attachment.PlayMethod = play.UpstreamPlayMethod + if !play.CreatedAt.IsZero() { + // P0a established the top-level compat CreatedAt as the source of + // truth for a compat session's start time. + attachment.StartedAt = play.CreatedAt + attachment.StartedAtSource = streamtelemetry.StartedAtSourceSession } streamtelemetry.Attach(ctx, attachment) } diff --git a/internal/jellycompat/streamtelemetry_test.go b/internal/jellycompat/streamtelemetry_test.go index baf2e66f7..e0a65291b 100644 --- a/internal/jellycompat/streamtelemetry_test.go +++ b/internal/jellycompat/streamtelemetry_test.go @@ -127,6 +127,22 @@ func (f compatTelemetryFixture) get(t *testing.T, method, url string, headers ma return compatResponse{status: resp.StatusCode, body: string(buf)} } +// playSessions exposes the fixture's compat play sessions so a test can assert +// telemetry is NOT keyed on their ids. +func (f compatTelemetryFixture) playSessions() []PlaybackSession { + store, ok := f.store.(*PlaybackSessionStore) + if !ok { + return nil + } + store.mu.RLock() + defer store.mu.RUnlock() + out := make([]PlaybackSession, 0, len(store.sessions)) + for _, play := range store.sessions { + out = append(out, play) + } + return out +} + func TestMountedCompatRouterAttributesDirectStream(t *testing.T) { registry := compatTelemetryRegistry(t) fixture := newCompatTelemetryServer(t, registry) @@ -150,8 +166,20 @@ func TestMountedCompatRouterAttributesDirectStream(t *testing.T) { if session.Subject != streamtelemetry.UserSubject(91) || session.ProfileID != "profile-7" { t.Fatalf("identity = %+v", session) } - if session.SessionID == "" { - t.Fatal("compat session has no canonical play-session id") + // The canonical key is the UPSTREAM playback session id (what the proxy, + // nodesessions and playback_sessions_sync all publish), not the compat + // PlaybackSession.ID. Keying on the latter splits one viewing into two + // merged sessions and makes every compat session look telemetry_only. + if session.SessionID != "upstream-started" { + t.Fatalf("session id = %q, want the upstream playback session id", session.SessionID) + } + for _, play := range fixture.playSessions() { + if session.SessionID == play.ID { + t.Fatalf("telemetry keyed on the compat play session id %q", play.ID) + } + if play.UpstreamSessionID != session.SessionID { + t.Fatalf("play session upstream id = %q, telemetry session id = %q", play.UpstreamSessionID, session.SessionID) + } } if session.MediaFileID != 42 { t.Fatalf("media file id = %d", session.MediaFileID) From 9bd8c31a872ca47307d0a7fa0983693c475bc0b9 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:39:35 -0400 Subject: [PATCH 15/44] fix(streamtelemetry): make Truncated recoverable and hold early realtime state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in the process-local registry, all found by review: Truncated was sticky for the process lifetime. drop() set it and nothing ever cleared it, so one transient capacity burst pinned the merged view's Complete to false until a restart and made a later real truncation indistinguishable. It now decays over Freshness — the same horizon BuildGlobalView uses to decide a publisher is current — while the monotonic Dropped* counters keep the permanent record. SetRealtimeConnection was a no-op when the session did not exist yet. That is the normal client ordering: the control socket opens as soon as a sessionId exists, before the first media route is hit, so RealtimeConnectionAlive stayed false for the whole of every live session. State for an unknown session is now held per shard, applied when an attach creates the session, capacity-bounded against the session budget, and pruned by the sweep. The distributed cross-checks compared an env-supplied value against the DEFAULT of the other knob, so setting one variable disabled distributed mode and blamed a variable the operator never set. Knobs left at their defaults now move to satisfy the invariant; only a pair pinned to genuinely inconsistent values is an error, and only the variables actually set are named. Co-Authored-By: Claude Opus 5 (1M context) --- internal/streamtelemetry/config.go | 64 +++++++++++-- internal/streamtelemetry/config_test.go | 54 +++++++++++ internal/streamtelemetry/registry.go | 104 ++++++++++++++++++---- internal/streamtelemetry/registry_test.go | 92 +++++++++++++++++++ 4 files changed, 292 insertions(+), 22 deletions(-) diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index 730529758..e841121a2 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -83,10 +83,15 @@ type Config struct { MaxRoutesPerSession int } +// defaultFreshness is how long a published snapshot stays current. It doubles as +// the decay window for a publisher's Truncated flag, since both answer the same +// question: is this publisher's picture of the world usable right now? +const defaultFreshness = 5 * time.Second + func DefaultConfig(nodeID string) Config { return Config{ NodeID: nodeID, SweepInterval: time.Second, Retention: 5 * time.Minute, - Freshness: 5 * time.Second, MembershipTTL: time.Minute, KeyPrefix: "silo:stelem", + Freshness: defaultFreshness, MembershipTTL: time.Minute, KeyPrefix: "silo:stelem", ViewTTL: DefaultViewTTL, FullResyncEvery: 60, MaxPublishers: 256, MaxMergedSessions: 50_000, MaxMergedTransfers: 50_000, MaxSessions: 10_000, MaxTransfers: 10_000, MaxObservations: 50_000, @@ -105,12 +110,17 @@ func ConfigFromEnv(nodeID string) Config { cfg.Enabled = envEnabled(os.Getenv(enabledEnv)) coreInvalid := make([]string, 0) distributedInvalid := make([]string, 0) + // The operator only owns the variables they actually set. The cross-checks + // below relate two knobs, and a violation involving an unset knob is not the + // operator's mistake — it is a default that has to move. + explicit := make(map[string]bool) cfg.Distributed = envEnabled(os.Getenv(distributedEnv)) parseDuration := func(name string, dst *time.Duration) { value := strings.TrimSpace(os.Getenv(name)) if value == "" { return } + explicit[name] = true parsed, err := time.ParseDuration(value) if err != nil || parsed <= 0 { coreInvalid = append(coreInvalid, name) @@ -123,6 +133,7 @@ func ConfigFromEnv(nodeID string) Config { if value == "" { return } + explicit[name] = true parsed, err := time.ParseDuration(value) if err != nil || parsed <= 0 { distributedInvalid = append(distributedInvalid, name) @@ -180,14 +191,57 @@ func ConfigFromEnv(nodeID string) Config { cfg.KeyPrefix = value } } - if cfg.SweepInterval > time.Duration(1<<63-1)/3 || cfg.Freshness < 3*cfg.SweepInterval { - distributedInvalid = append(distributedInvalid, freshnessEnv) + // Cross-checks. Each relates a knob to another knob, so comparing an + // env-supplied value against the other's DEFAULT and then rejecting the + // config is wrong twice over: it disables distributed mode for a single + // variable, and it blames a variable the operator never touched. When only + // one side was set, the unset side moves to satisfy the invariant; only a + // pair the operator pinned to genuinely inconsistent values is an error, and + // then only the variables they set are named. + crossCheckFailed := func(involved ...string) { + named := make([]string, 0, len(involved)) + for _, name := range involved { + if explicit[name] { + named = append(named, name) + } + } + if len(named) == 0 { + // Defaults that violate their own invariant: a code bug, not an + // operator one. Name both so it is findable. + named = involved + } + distributedInvalid = append(distributedInvalid, named...) + } + // Repair first, then validate the RESOLVED values. Repairs only ever move a + // knob the operator left at its default, and are ordered so a later one + // cannot undo an earlier one. + if !explicit[freshnessEnv] && cfg.SweepInterval <= time.Duration(1<<63-1)/3 && cfg.Freshness < 3*cfg.SweepInterval { + cfg.Freshness = 3 * cfg.SweepInterval + } + if !explicit[sweepIntervalEnv] && cfg.Freshness < 3*cfg.SweepInterval { + cfg.SweepInterval = cfg.Freshness / 3 + } + if !explicit[membershipTTLEnv] && cfg.MembershipTTL <= cfg.Freshness { + cfg.MembershipTTL = 2 * cfg.Freshness + } + if !explicit[freshnessEnv] && cfg.MembershipTTL <= cfg.Freshness { + cfg.Freshness = cfg.MembershipTTL / 2 + if !explicit[sweepIntervalEnv] && cfg.Freshness < 3*cfg.SweepInterval { + cfg.SweepInterval = cfg.Freshness / 3 + } + } + // A snapshot older than three sweeps is stale; overflow-guard the + // multiplication the comparison depends on. + if cfg.SweepInterval <= 0 || cfg.SweepInterval > time.Duration(1<<63-1)/3 || cfg.Freshness < 3*cfg.SweepInterval { + crossCheckFailed(sweepIntervalEnv, freshnessEnv) } + // Membership has to outlive freshness, or a publisher leaves the roster + // before it is even considered stale. if cfg.MembershipTTL <= cfg.Freshness { - distributedInvalid = append(distributedInvalid, membershipTTLEnv) + crossCheckFailed(freshnessEnv, membershipTTLEnv) } if cfg.MembershipTTL > time.Duration(1<<63-1)/10 { - distributedInvalid = append(distributedInvalid, membershipTTLEnv) + crossCheckFailed(membershipTTLEnv) } if len(coreInvalid) > 0 { if cfg.Enabled { diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index 6a3b30af5..103dffc0a 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -92,6 +92,60 @@ func TestConfigFromEnvValidation(t *testing.T) { t.Fatalf("config = %+v", cfg) } }) + // Setting ONE variable must not disable distributed mode by colliding with + // the other knob's default: the unset knob moves instead. + t.Run("sweep interval alone raises the unset freshness", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(sweepIntervalEnv, "2s") // default freshness 5s < 3*2s + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.Distributed { + t.Fatalf("one variable disabled distributed mode: %+v", cfg) + } + if cfg.SweepInterval != 2*time.Second || cfg.Freshness != 6*time.Second { + t.Fatalf("sweep/freshness = %v/%v, want 2s/6s", cfg.SweepInterval, cfg.Freshness) + } + }) + t.Run("freshness alone lowers the unset sweep interval", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(freshnessEnv, "2s") // default sweep 1s is fine; 2s < 3s is not + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.Distributed { + t.Fatalf("one variable disabled distributed mode: %+v", cfg) + } + if cfg.Freshness != 2*time.Second || cfg.SweepInterval > cfg.Freshness/3 { + t.Fatalf("sweep/freshness = %v/%v", cfg.SweepInterval, cfg.Freshness) + } + }) + t.Run("freshness alone raises the unset membership ttl", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(freshnessEnv, "60s") // default membership TTL is also 60s + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.Distributed { + t.Fatalf("one variable disabled distributed mode: %+v", cfg) + } + if cfg.Freshness != 60*time.Second || cfg.MembershipTTL <= cfg.Freshness { + t.Fatalf("freshness/membership = %v/%v", cfg.Freshness, cfg.MembershipTTL) + } + }) + t.Run("membership ttl alone lowers the unset freshness", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(distributedEnv, "true") + t.Setenv(membershipTTLEnv, "4s") // default freshness 5s outlives it + cfg := ConfigFromEnv("node") + if !cfg.Enabled || !cfg.Distributed { + t.Fatalf("one variable disabled distributed mode: %+v", cfg) + } + if cfg.MembershipTTL != 4*time.Second || cfg.MembershipTTL <= cfg.Freshness { + t.Fatalf("freshness/membership = %v/%v", cfg.Freshness, cfg.MembershipTTL) + } + }) t.Run("membership not above freshness", func(t *testing.T) { clearConfigEnv(t) t.Setenv(enabledEnv, "true") diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go index ca746b42d..9d42fc274 100644 --- a/internal/streamtelemetry/registry.go +++ b/internal/streamtelemetry/registry.go @@ -21,6 +21,18 @@ var now = time.Now type sessionShard struct { sync.RWMutex sessions map[string]*logicalSession + // pendingRealtime holds realtime-connection state that arrived before the + // session existed. Clients open the control socket as soon as they have a + // sessionId — before the first byte route is hit — so in the normal + // ordering the state would otherwise be dropped and every live session + // would report RealtimeConnectionAlive=false. Applied on session creation + // and pruned by the sweep, so it cannot grow without bound. + pendingRealtime map[string]pendingRealtime +} + +type pendingRealtime struct { + connected bool + at time.Time } type Registry struct { @@ -40,17 +52,23 @@ type Registry struct { droppedBytes atomic.Int64 unattributedObservations atomic.Int64 unattributedBytes atomic.Int64 - truncated atomic.Bool - lastWarnUnixNano atomic.Int64 - lastPublishWarnUnixNano atomic.Int64 - sequence atomic.Uint64 - startOnce sync.Once - stopOnce sync.Once - stop chan struct{} - done chan struct{} - started atomic.Bool - leaveMu sync.Mutex - left bool + // lastDropUnixNano records when an observation was last dropped. Truncated + // is a statement about CURRENT blindness — BuildGlobalView pins + // Complete=false for as long as a publisher reports it — so it has to + // decay, otherwise one transient burst marks a process degraded until it + // restarts and a later real truncation is indistinguishable. The monotonic + // Dropped* counters remain the permanent record. + lastDropUnixNano atomic.Int64 + lastWarnUnixNano atomic.Int64 + lastPublishWarnUnixNano atomic.Int64 + sequence atomic.Uint64 + startOnce sync.Once + stopOnce sync.Once + stop chan struct{} + done chan struct{} + started atomic.Bool + leaveMu sync.Mutex + left bool } func NewRegistry(cfg Config, store SnapshotStore, logger *slog.Logger) *Registry { @@ -69,6 +87,7 @@ func NewRegistry(cfg Config, store SnapshotStore, logger *slog.Logger) *Registry r := &Registry{cfg: cfg, store: store, logger: logger, seed: maphash.MakeSeed(), transfers: make(map[string]*transfer), stop: make(chan struct{}), done: make(chan struct{})} for i := range r.shards { r.shards[i].sessions = make(map[string]*logicalSession) + r.shards[i].pendingRealtime = make(map[string]pendingRealtime) } return r } @@ -164,6 +183,10 @@ func (r *Registry) attach(obs *Observation, attachment Attachment) { return } s = newLogicalSession(attachment, r.cfg, observedAt) + if pending, ok := shard.pendingRealtime[attachment.SessionID]; ok { + s.realtimeAlive = pending.connected + delete(shard.pendingRealtime, attachment.SessionID) + } shard.sessions[attachment.SessionID] = s } s.mu.Lock() @@ -262,7 +285,7 @@ func (r *Registry) release(obs *Observation, outcome httpstream.StreamOutcome) { } func (r *Registry) drop(reason string) { - r.truncated.Store(true) + r.lastDropUnixNano.Store(now().UnixNano()) r.droppedObservations.Add(1) r.warnRateLimited(reason, &r.lastWarnUnixNano) } @@ -283,6 +306,33 @@ func (r *Registry) warnRateLimited(message string, stamp *atomic.Int64, attrs .. } } +// truncatedAt reports whether the registry was blind recently enough for the +// snapshot at `at` to be incomplete. The window matches Freshness, which is the +// same horizon BuildGlobalView uses to decide a publisher is still current. +func (r *Registry) truncatedAt(at time.Time) bool { + last := r.lastDropUnixNano.Load() + if last == 0 { + return false + } + window := r.cfg.Freshness + if window <= 0 { + window = defaultFreshness + } + if at.IsZero() { + at = now() + } + return at.Sub(time.Unix(0, last)) < window +} + +// maxPendingRealtimePerShard spreads the session budget over the shards so held +// realtime state can never outgrow the sessions it is waiting for. +func maxPendingRealtimePerShard(maxSessions int64) int64 { + if maxSessions <= 0 { + return 0 + } + return maxSessions/shardCount + 1 +} + func (r *Registry) shard(id string) *sessionShard { var h maphash.Hash h.SetSeed(r.seed) @@ -290,19 +340,31 @@ func (r *Registry) shard(id string) *sessionShard { return &r.shards[h.Sum64()%shardCount] } +// SetRealtimeConnection records whether a realtime control socket is alive for +// a session. It is routinely called BEFORE the session exists — a client opens +// the socket as soon as it has a sessionId, which is before it requests the +// first media route — so state for an unknown session is held until an attach +// creates it rather than discarded. func (r *Registry) SetRealtimeConnection(sessionID string, connected bool) { if r == nil || !r.cfg.Enabled || sessionID == "" { return } shard := r.shard(sessionID) - shard.RLock() - s := shard.sessions[sessionID] - if s != nil { + shard.Lock() + if s := shard.sessions[sessionID]; s != nil { s.mu.Lock() s.realtimeAlive = connected s.mu.Unlock() + delete(shard.pendingRealtime, sessionID) + shard.Unlock() + return + } + if _, held := shard.pendingRealtime[sessionID]; held || int64(len(shard.pendingRealtime)) < maxPendingRealtimePerShard(r.cfg.MaxSessions) { + shard.pendingRealtime[sessionID] = pendingRealtime{connected: connected, at: now()} + } else { + r.drop("pending realtime capacity exhausted") } - shard.RUnlock() + shard.Unlock() } func (r *Registry) Start(ctx context.Context) { @@ -398,6 +460,14 @@ func (r *Registry) sweep(sweepStart time.Time) Snapshot { r.sessionReservations.Add(-1) } } + // Realtime state whose session never arrived — a socket that opened and + // closed without the client ever requesting media — expires on the same + // horizon as an idle session. + for id, pending := range shard.pendingRealtime { + if sweepStart.Sub(pending.at) >= r.cfg.Retention { + delete(shard.pendingRealtime, id) + } + } shard.Unlock() } r.transfersMu.Lock() @@ -432,7 +502,7 @@ func (r *Registry) Snapshot() Snapshot { return r.SnapshotAt(now()) } // most recent sweep; callers that need current totals must call Sweep. func (r *Registry) SnapshotAt(capturedAt time.Time) Snapshot { view := Snapshot{PublisherID: r.cfg.PublisherID, NodeID: r.cfg.NodeID, PublisherEpoch: r.cfg.PublisherEpoch, Sequence: r.sequence.Load(), CapturedAt: capturedAt, - Truncated: r.truncated.Load(), DroppedObservations: r.droppedObservations.Load(), + Truncated: r.truncatedAt(capturedAt), DroppedObservations: r.droppedObservations.Load(), DroppedBytes: r.droppedBytes.Load(), UnattributedObservations: r.unattributedObservations.Load(), UnattributedBytes: r.unattributedBytes.Load()} for i := range r.shards { diff --git a/internal/streamtelemetry/registry_test.go b/internal/streamtelemetry/registry_test.go index fddbcc6a0..32d658966 100644 --- a/internal/streamtelemetry/registry_test.go +++ b/internal/streamtelemetry/registry_test.go @@ -425,3 +425,95 @@ func TestLocalStoreDeepCopies(t *testing.T) { t.Fatalf("store returned aliased snapshot: %+v", loadedAgain) } } + +// Truncated states current blindness — BuildGlobalView pins Complete=false while +// a publisher reports it — so one transient drop must not mark a process +// degraded for the rest of its life. +func TestTruncatedDecaysAfterFreshness(t *testing.T) { + cfg := testConfig() + cfg.MaxObservations = 0 // force the very first observation to be dropped + registry := NewRegistry(cfg, NewLocalStore(), slog.New(slog.DiscardHandler)) + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("session-1")) + _, _ = w.Write([]byte("payload")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + + dropAt := time.Now() + if !registry.SnapshotAt(dropAt).Truncated { + t.Fatal("snapshot taken at the drop is not truncated") + } + if snapshot := registry.SnapshotAt(dropAt.Add(cfg.Freshness / 2)); !snapshot.Truncated { + t.Fatal("snapshot inside the freshness window is not truncated") + } + later := registry.SnapshotAt(dropAt.Add(cfg.Freshness + time.Second)) + if later.Truncated { + t.Fatal("Truncated is still set an entire freshness window after the drop") + } + // The permanent record stays monotonic. + if later.DroppedObservations == 0 { + t.Fatalf("dropped observations = %d, want the drop to still be counted", later.DroppedObservations) + } +} + +// Clients open the realtime control socket as soon as they have a sessionId, +// which is before they request any media route. State that arrives then has to +// survive until the session exists, or every live session reports a dead socket. +func TestRealtimeConnectionSetBeforeAttachIsApplied(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), slog.New(slog.DiscardHandler)) + registry.SetRealtimeConnection("session-1", true) + + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("session-1")) + _, _ = w.Write([]byte("payload")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + + snapshot := registry.SnapshotAt(time.Now()) + if len(snapshot.Sessions) != 1 { + t.Fatalf("sessions = %+v", snapshot.Sessions) + } + if !snapshot.Sessions[0].RealtimeConnectionAlive { + t.Fatal("realtime state set before the first media route was dropped") + } +} + +// Held state must not outlive the sessions it waits for. +func TestPendingRealtimeStateIsBounded(t *testing.T) { + cfg := testConfig() + cfg.MaxSessions = 0 + registry := NewRegistry(cfg, NewLocalStore(), slog.New(slog.DiscardHandler)) + registry.SetRealtimeConnection("session-1", true) + + shard := registry.shard("session-1") + shard.RLock() + held := len(shard.pendingRealtime) + shard.RUnlock() + if held != 0 { + t.Fatalf("pending realtime entries = %d, want the capacity bound to refuse it", held) + } +} + +// A socket that opens and closes without the client ever requesting media leaves +// state behind; the sweep has to reclaim it. +func TestPendingRealtimeStateIsPrunedBySweep(t *testing.T) { + registry := NewRegistry(testConfig(), NewLocalStore(), slog.New(slog.DiscardHandler)) + registry.SetRealtimeConnection("orphan-session", true) + + shard := registry.shard("orphan-session") + shard.RLock() + held := len(shard.pendingRealtime) + shard.RUnlock() + if held != 1 { + t.Fatalf("pending realtime entries = %d, want 1", held) + } + + // testConfig sets Retention to 1ms, so one sweep past it collects the entry. + registry.sweep(time.Now().Add(time.Second)) + shard.RLock() + held = len(shard.pendingRealtime) + shard.RUnlock() + if held != 0 { + t.Fatalf("pending realtime entries after sweep = %d, want 0", held) + } +} From 6ec97c198ba13e684926ec87a249ad0c735657aa Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:43:22 -0400 Subject: [PATCH 16/44] fix(httpstream): give every ReadFrom slice a full stall window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bumpStep throttle was written for the 32 KB Write path, where one SetWriteDeadline per chunk would be wasteful. Applying it to ReadFrom slices buys nothing — a slice is already bounded at 4 MiB — and costs correctness: a slice completing less than a step after the last bump got no refresh, so the next one started with as little as window-step remaining. The real guaranteed floor was ~203 kbit/s, not the 186 kbit/s the constant and both design documents promise, and a client sustaining the documented rate was reaped as stalled. Slices now bump unconditionally, before the first as well as between each, which is what the pre-CopyChunked loop did. Costs at most one syscall per 4 MiB. The existing deadline tests construct the writer with step=0 and so never exercised the throttle; the two added here fail on the unfixed code. Co-Authored-By: Claude Opus 5 (1M context) --- internal/httpstream/readfrom_deadline_test.go | 78 +++++++++++++++++++ internal/httpstream/rolling_deadline.go | 34 ++++++-- 2 files changed, 107 insertions(+), 5 deletions(-) diff --git a/internal/httpstream/readfrom_deadline_test.go b/internal/httpstream/readfrom_deadline_test.go index cb632b42c..b5cc0beed 100644 --- a/internal/httpstream/readfrom_deadline_test.go +++ b/internal/httpstream/readfrom_deadline_test.go @@ -150,3 +150,81 @@ func TestReadFromChunkAllowsSlowClients(t *testing.T) { readFromChunk, DefaultStallWindow, floor, maxAcceptableFloorBitsPerSec) } } + +// TestReadFromRollsDeadlineUnderProductionStep is the same guarantee with the +// real bumpStep in play. The other deadline tests construct the writer with +// step=0, so they never exercise the throttle — and the throttle was the bug: +// a slice completing less than a step after the last bump got no refresh, so the +// next slice started with as little as window-step remaining and a client +// sustaining the documented floor rate was reaped despite never stalling. +func TestReadFromRollsDeadlineUnderProductionStep(t *testing.T) { + slice := sliceDuration() + window := slice * 3 + total := readFromChunk * 3 + + done := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // A step far longer than the whole transfer: with the throttle applied to + // slices, every bump after the first would be suppressed. + sw := newRollingDeadlineWriter(w, window, time.Hour) + sw.WriteHeader(http.StatusOK) + _, err := sw.ReadFrom(&pacedReader{remaining: total, piece: testPiece, pause: testPiecePause}) + done <- err + })) + srv.Config.WriteTimeout = 0 + srv.Start() + defer srv.Close() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + n, err := io.Copy(io.Discard, resp.Body) + if err != nil { + t.Fatalf("stream died after %d/%d bytes under the production bump step: %v", n, total, err) + } + if n != total { + t.Fatalf("short body: got %d bytes, want %d", n, total) + } + if handlerErr := <-done; handlerErr != nil { + t.Fatalf("handler ReadFrom returned %v; the step throttle must not shorten a slice's window", handlerErr) + } +} + +// A handler that commits headers and then waits before its first write must not +// spend that wait against the window set at construction. +func TestReadFromBumpsBeforeTheFirstSlice(t *testing.T) { + slice := sliceDuration() + window := slice * 3 + total := readFromChunk + + done := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + sw := newRollingDeadlineWriter(w, window, time.Hour) + sw.WriteHeader(http.StatusOK) + // Stand in for waiting on artifact readiness: longer than the window, so + // only a bump before the first slice can save the transfer. + time.Sleep(window + 50*time.Millisecond) + _, err := sw.ReadFrom(&pacedReader{remaining: total, piece: testPiece, pause: testPiecePause}) + done <- err + })) + srv.Config.WriteTimeout = 0 + srv.Start() + defer srv.Close() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + n, err := io.Copy(io.Discard, resp.Body) + if err != nil || n != total { + t.Fatalf("first slice ran against a stale deadline: %d/%d bytes, err %v", n, total, err) + } + if handlerErr := <-done; handlerErr != nil { + t.Fatalf("handler ReadFrom returned %v after a long pre-write wait", handlerErr) + } +} diff --git a/internal/httpstream/rolling_deadline.go b/internal/httpstream/rolling_deadline.go index 79c3e2fb3..cd7460159 100644 --- a/internal/httpstream/rolling_deadline.go +++ b/internal/httpstream/rolling_deadline.go @@ -29,12 +29,16 @@ const ( stallWindowEnv = "SILO_STREAM_WRITE_STALL_TIMEOUT" // bumpStep rate-limits deadline updates so a busy stream issues one - // SetWriteDeadline per step rather than one per 32 KB chunk. + // SetWriteDeadline per step rather than one per 32 KB chunk. It applies to + // Write only: a ReadFrom slice is already bounded at readFromChunk, so + // bumping around one costs at most a syscall per 4 MiB and the throttle + // would only shorten the window a slice runs against. bumpStep = 15 * time.Second // readFromChunk bounds each ReadFrom slice so the deadline keeps rolling. // At the default 180s window, 4 MiB permits steady clients down to roughly - // 186 kbit/s without expiring mid-slice. + // 186 kbit/s without expiring mid-slice. That figure is only true because + // every slice starts against a freshly set deadline — see forceBump. readFromChunk int64 = ReadFromChunkDefault ) @@ -97,10 +101,26 @@ func (s *RollingDeadlineWriter) bump() { if s.disabled { return } - now := time.Now() - if !s.lastBump.IsZero() && now.Sub(s.lastBump) < s.step { + if !s.lastBump.IsZero() && time.Since(s.lastBump) < s.step { return } + s.forceBump() +} + +// forceBump sets the deadline unconditionally, ignoring the step throttle. +// +// The throttle exists so a fast stream does not issue one SetWriteDeadline per +// 32 KB Write. A ReadFrom slice is already bounded at readFromChunk, so the +// throttle buys nothing there and costs correctness: a throttled slice starts +// with as little as window-step remaining, which raises the sustained rate a +// client must hold to survive from the documented 186 kbit/s to ~203 kbit/s and +// reaps healthy slow clients. Every slice therefore gets a full window, which is +// what the pre-CopyChunked loop did. +func (s *RollingDeadlineWriter) forceBump() { + if s.disabled { + return + } + now := time.Now() if err := s.rc.SetWriteDeadline(now.Add(s.window)); err != nil { s.disabled = true return @@ -141,8 +161,12 @@ func (s *RollingDeadlineWriter) ReadFrom(r io.Reader) (int64, error) { if s.statusCode == 0 { s.statusCode = http.StatusOK } + // Before the FIRST slice as well as between slices: a handler that sets + // headers and then waits on readiness before its first write would otherwise + // run that slice against the window set at construction. + s.forceBump() return CopyChunked(rf, r, readFromChunk, func(n int64, err error) { - s.bump() + s.forceBump() s.recordWrite(n, err) }) } From 5a9e6e72b3bfddfc9a5033ddc22c8aedd4a595b8 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:43:22 -0400 Subject: [PATCH 17/44] fix(proxy): credit the egress meter often enough to measure slow viewers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit meteredResponseWriter previously hid io.ReaderFrom on purpose, so every byte reached egressMeter.Add through a ~32 KB Write. Forwarding ReadFrom restored sendfile but moved crediting to once per completed 4 MiB slice, which a 200-500 kbit/s direct-play viewer takes 60-170 s to fill. RateKbps averages over 60 s, so those streams read as zero for most samples: /api/v1/status under-reports committed egress and nodepool's effectiveEgressKbps can admit sessions onto a saturated proxy. Metered slices are now 256 KiB — a credit every 4-10 s at those rates, well inside the window, and still 8x more per sendfile call than the Write path it replaced. Slice size here is a rate-fidelity constraint, not a tuning knob. Co-Authored-By: Claude Opus 5 (1M context) --- internal/proxy/egress.go | 16 +++++++++++++- internal/proxy/egress_readfrom_test.go | 30 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/internal/proxy/egress.go b/internal/proxy/egress.go index b38072e24..fe8191875 100644 --- a/internal/proxy/egress.go +++ b/internal/proxy/egress.go @@ -60,6 +60,20 @@ func (m *egressMeter) RateKbps() int { return int(total * 8 / 1000 / meterWindowSeconds) } +// meterChunk bounds one zero-copy slice on a metered response. +// +// This is a rate-fidelity constraint, not a tuning knob. The meter is a +// per-second ring averaged over 60 s, and a slice credits it only when the slice +// completes, so the slice has to be short relative to that window at the SLOWEST +// rate worth measuring. At the shared 4 MiB default a 200-500 kbit/s viewer +// takes 60-170 s per slice: RateKbps reads that stream as zero for most samples, +// /api/v1/status under-reports committed egress, and the planner's +// effectiveEgressKbps can admit new sessions onto a saturated proxy. 256 KiB +// credits the same viewer roughly every 4-10 s, well inside the window, while +// still handing the kernel 8x more per sendfile call than the 32 KiB Write path +// this replaced. +const meterChunk int64 = 256 << 10 + // meteredResponseWriter counts every byte written to the client. Chunked // ReaderFrom delegation preserves both sendfile and the rolling rate window. type meteredResponseWriter struct { @@ -78,7 +92,7 @@ func (w *meteredResponseWriter) ReadFrom(src io.Reader) (int64, error) { if !ok { return io.Copy(httpstream.WriterOnly(w), src) } - return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + return httpstream.CopyChunked(rf, src, meterChunk, func(n int64, _ error) { w.meter.Add(n) }) } diff --git a/internal/proxy/egress_readfrom_test.go b/internal/proxy/egress_readfrom_test.go index 59d0292fe..7af28f1cf 100644 --- a/internal/proxy/egress_readfrom_test.go +++ b/internal/proxy/egress_readfrom_test.go @@ -5,6 +5,9 @@ import ( "io" "net/http" "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/httpstream" ) type egressReaderFromSpy struct{ bytes.Buffer } @@ -27,3 +30,30 @@ func TestMeteredResponseWriterReadFromCountsBytes(t *testing.T) { t.Fatalf("meter rate = %d, want > 0", got) } } + +// A slice credits the meter only when it completes, so the slice has to be small +// enough that a slow viewer still registers inside the 60 s rate window. This +// pins the granularity: a transfer the size of one shared 4 MiB default slice +// must produce several credits, not one. +func TestMeteredResponseWriterReadFromCreditsIncrementally(t *testing.T) { + spy := &egressReaderFromSpy{} + meter := newEgressMeter() + credits := 0 + meter.now = func() time.Time { + // Add consults the clock exactly once per credit, and returns before + // doing so for a zero-byte slice, so this counts credits. + credits++ + return time.Unix(1_000_000, 0) + } + w := &meteredResponseWriter{ResponseWriter: spy, meter: meter} + + const body = httpstream.ReadFromChunkDefault + if _, err := w.ReadFrom(bytes.NewReader(make([]byte, body))); err != nil { + t.Fatalf("ReadFrom: %v", err) + } + wantCredits := int(body / meterChunk) + if credits != wantCredits { + t.Fatalf("meter credited %d times over %d bytes, want %d (one per %d-byte slice)", + credits, body, wantCredits, meterChunk) + } +} From ff19ab64122c5710c069264de9a878f58e55e45d Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:43:22 -0400 Subject: [PATCH 18/44] fix(downloads): roll the direct-download deadline and carry the profile handleDirectDownload passed the raw ResponseWriter to ServeDirect, so unlike the sibling /downloads/{id}/file it had no rolling deadline and the API server's absolute 120 s WriteTimeout truncated any original large enough to take longer. Excluding the route from compression made it one unbounded sendfile, so the whole body now rides on that single deadline. redirectDirectDownload hardcoded an empty profile id in both the proxy redirect and the telemetry attach, while the local branch two lines away reads the real one. Proxy-served traffic was therefore missing from per-profile attribution in telemetry, in the stream token claim and in the node session. Co-Authored-By: Claude Opus 5 (1M context) --- internal/api/handlers/downloads.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/api/handlers/downloads.go b/internal/api/handlers/downloads.go index 75f0977e1..23962080f 100644 --- a/internal/api/handlers/downloads.go +++ b/internal/api/handlers/downloads.go @@ -540,7 +540,11 @@ func (h *DownloadHandler) handleDirectDownload(w http.ResponseWriter, r *http.Re serveCtx := downloads.WithServeAuthorized(r.Context(), func(target downloads.FileTarget) { attachTransfer(r.Context(), userID, apimw.GetProfileID(r.Context()), target.MediaFileID) }) - if err := h.svc.ServeDirect(serveCtx, w, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil { + // A multi-gigabyte original outlives the API server's absolute WriteTimeout, + // exactly as on /downloads/{id}/file above; roll the deadline with progress + // instead of truncating the body at 120 s. + sw := httpstream.NewRollingDeadlineWriter(w) + if err := h.svc.ServeDirect(serveCtx, sw, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil { h.writeDownloadError(w, err) return } @@ -555,9 +559,15 @@ func (h *DownloadHandler) redirectDirectDownload(ctx context.Context, w http.Res if err != nil { return false, err } - handled, err := h.redirectToProxy(w, r, secret, target, userID, "") + // The profile has to travel with the redirect. Hardcoding "" here recorded + // the telemetry transfer, the proxy's own attach (from the token claim) and + // the node session against no profile at all, so proxy-served traffic went + // missing from per-profile attribution while the same file served locally + // was attributed correctly. + profileID := apimw.GetProfileID(ctx) + handled, err := h.redirectToProxy(w, r, secret, target, userID, profileID) if handled { - attachTransfer(ctx, userID, "", target.MediaFileID) + attachTransfer(ctx, userID, profileID, target.MediaFileID) } return handled, err } From fc46ad893c2befc36ac38de0cafda7e9cc2314d5 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:46:39 -0400 Subject: [PATCH 19/44] fix(streamtelemetry): fold ranged transfers, guard delta publishes, split conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transfers were one record per HTTP request keyed by observation id, so ranged byte routes — audiobook file reads, download resumes, ebook fetches — could exhaust MaxTransfers within one retention window while RequestCount, the field that exists to count exactly this, stayed pinned at 1. A transfer is now one subject pouring one file over one route, and overlapping requests fold into it. A delta publish rewrites only changed fields and assumed the Redis hash still held the rest. An eviction, an out-of-band DEL, a replica failover or a lapsed PExpire drops it with no error, leaving under-reported sessions for up to FullResyncEvery publishes. An HLEN inside the same transaction now catches the mismatch and forces the next publish full, self-healing in one sweep. recordConflicts appended started_at_replaced without setting hasIdentityConflict, so the exported flag could disagree with the exported list. A pure authority upgrade that confirms the recorded instant now records nothing at all — it is not a conflict and should never have consumed the budget — and a replacement that moves the value sets both. Also documents two limitations rather than half-fixing them: clock skew is only detectable for a publisher running ahead, since the roster score is the publisher's own clock; and observedWriter.ReadFrom samples the cut flag once, which the enforcement change that first calls cut.Store has to make uniform across h1 and h2. Co-Authored-By: Claude Opus 5 (1M context) --- internal/streamtelemetry/global.go | 11 +++ internal/streamtelemetry/registry.go | 56 ++++++++++--- internal/streamtelemetry/registry_test.go | 95 +++++++++++++++++++++++ internal/streamtelemetry/session.go | 36 ++++++--- internal/streamtelemetry/store_redis.go | 17 ++++ internal/streamtelemetry/writer.go | 12 +++ 6 files changed, 207 insertions(+), 20 deletions(-) diff --git a/internal/streamtelemetry/global.go b/internal/streamtelemetry/global.go index a416aa10a..ab4c39fa3 100644 --- a/internal/streamtelemetry/global.go +++ b/internal/streamtelemetry/global.go @@ -213,6 +213,17 @@ func BuildGlobalView(set PublisherSet, at time.Time, params ViewParams) GlobalMo ref := PublisherRef{PublisherID: member.PublisherID, NodeID: snapshot.NodeID} status := PublisherStatus{PublisherRef: ref, LastHeartbeat: member.LastHeartbeat, CapturedAt: snapshot.CapturedAt, Epoch: snapshot.PublisherEpoch, Sequence: snapshot.Sequence, Truncated: snapshot.Truncated} + // Skew is only detectable in one direction from a single sample. A + // publisher whose clock runs AHEAD stamps a future time and is caught + // here. One running BEHIND is indistinguishable from one that stopped + // publishing: the roster score is the publisher's own CapturedAt, so + // heartbeat and snapshot drift together and there is no independent + // clock to compare against. Such a publisher is classified stale and its + // sessions leave the merged view — the safe direction, since the + // alternative is serving data that may be minutes old as current. + // PublisherStatus carries Epoch and Sequence precisely so two successive + // reads of the parity endpoint distinguish "behind but advancing" from + // "stalled"; BuildGlobalView is a pure function of one sample and cannot. heartbeatAge := at.Sub(member.LastHeartbeat) if heartbeatAge < -params.Freshness { view.ClockSkewSuspected = true diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go index 9d42fc274..a831f856c 100644 --- a/internal/streamtelemetry/registry.go +++ b/internal/streamtelemetry/registry.go @@ -5,6 +5,7 @@ import ( "hash/maphash" "log/slog" "sort" + "strconv" "sync" "sync/atomic" "time" @@ -151,17 +152,42 @@ func (r *Registry) attach(obs *Observation, attachment Attachment) { attachment.TokenIssuedAtSource = TokenIssuedAtSourceNone } if obs.route.Class == ClassTransfer { - if !reserve(&r.transferReservations, r.cfg.MaxTransfers) { + // One record per subject/file/route, not one per HTTP request. Ranged + // byte routes issue many small overlapping GETs — an audiobook client + // alone can sustain tens per second — and a record per request would + // exhaust MaxTransfers within one retention window while requestCount, + // which exists to count exactly this, stayed pinned at 1. + key := transferKey(attachment, obs.route) + r.transfersMu.Lock() + t := r.transfers[key] + if t == nil { + if !reserve(&r.transferReservations, r.cfg.MaxTransfers) { + r.transfersMu.Unlock() + obs.countingOnly = true + r.drop("transfer capacity exhausted") + return + } + t = &transfer{id: key, subject: attachment.Subject, profileID: attachment.ProfileID, + mediaFileID: attachment.MediaFileID, route: obs.route, capture: obs.Capture, + observations: make(map[string]*Observation), + outcomes: make(map[httpstream.StreamOutcome]int64)} + r.transfers[key] = t + } + t.mu.Lock() + if len(t.observations) >= r.cfg.MaxObservationsPerSession { + t.mu.Unlock() + r.transfersMu.Unlock() obs.countingOnly = true - r.drop("transfer capacity exhausted") + r.drop("per-transfer observation capacity exhausted") return } - t := &transfer{id: obs.id, subject: attachment.Subject, profileID: attachment.ProfileID, - mediaFileID: attachment.MediaFileID, openObservations: 1, requestCount: 1, - route: obs.route, capture: obs.Capture, observation: obs, - outcomes: make(map[httpstream.StreamOutcome]int64)} - r.transfersMu.Lock() - r.transfers[t.id] = t + t.observations[obs.id] = obs + t.openObservations++ + t.requestCount++ + // The newest request's capture wins: viewer IP, device and client can + // legitimately change across a resumed download. + t.capture = obs.Capture + t.mu.Unlock() r.transfersMu.Unlock() obs.attachment = &attachment obs.target.transfer = t @@ -262,7 +288,7 @@ func (r *Registry) release(obs *Observation, outcome httpstream.StreamOutcome) { t.openObservations-- t.lastObservationEnd = now() t.outcomes[outcome]++ - t.observation = nil + delete(t.observations, obs.id) t.mu.Unlock() } else if target.session != nil { s := target.session @@ -333,6 +359,14 @@ func maxPendingRealtimePerShard(maxSessions int64) int64 { return maxSessions/shardCount + 1 } +// transferKey identifies one pour: a subject moving one media file over one +// route. Deliberately excludes anything per-request so overlapping Range GETs +// for the same file fold into a single record. +func transferKey(a Attachment, route MediaRoute) string { + return string(a.Subject.Kind) + "\x00" + a.Subject.ID + "\x00" + a.ProfileID + "\x00" + + strconv.Itoa(a.MediaFileID) + "\x00" + routeID(route.Method, route.Pattern) +} + func (r *Registry) shard(id string) *sessionShard { var h maphash.Hash h.SetSeed(r.seed) @@ -474,8 +508,8 @@ func (r *Registry) sweep(sweepStart time.Time) Snapshot { for id, t := range r.transfers { t.mu.Lock() total := t.bytesFolded - if t.observation != nil { - total += t.observation.BytesAccepted() + for _, obs := range t.observations { + total += obs.BytesAccepted() } if total > t.lastSweptBytes { t.lastByteAccepted = sweepStart diff --git a/internal/streamtelemetry/registry_test.go b/internal/streamtelemetry/registry_test.go index 32d658966..fe224803a 100644 --- a/internal/streamtelemetry/registry_test.go +++ b/internal/streamtelemetry/registry_test.go @@ -517,3 +517,98 @@ func TestPendingRealtimeStateIsPrunedBySweep(t *testing.T) { t.Fatalf("pending realtime entries after sweep = %d, want 0", held) } } + +// Ranged byte routes issue many small GETs for one file. A record per request +// would exhaust MaxTransfers inside a retention window and leave RequestCount — +// which exists to count exactly this — pinned at 1. +func TestRangedTransferRequestsFoldIntoOneRecord(t *testing.T) { + cfg := testConfig() + cfg.Retention = time.Hour // keep the record alive across the sweep below + registry := NewRegistry(cfg, NewLocalStore(), slog.New(slog.DiscardHandler)) + handler := registry.Observe(testRoute(ClassTransfer))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("")) + _, _ = w.Write([]byte("chunk")) + })) + const requests = 25 + for range requests { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + } + + snapshot := registry.Sweep() + if len(snapshot.Transfers) != 1 { + t.Fatalf("transfers = %d, want 1 folded record", len(snapshot.Transfers)) + } + transfer := snapshot.Transfers[0] + if transfer.RequestCount != requests { + t.Fatalf("request count = %d, want %d", transfer.RequestCount, requests) + } + if transfer.BytesAccepted != requests*int64(len("chunk")) { + t.Fatalf("bytes = %d, want %d", transfer.BytesAccepted, requests*int64(len("chunk"))) + } + if registry.transferReservations.Load() != 1 { + t.Fatalf("reservations = %d, want 1", registry.transferReservations.Load()) + } +} + +// A different file, subject or route is a different pour. +func TestTransfersSeparateByFileAndSubject(t *testing.T) { + cfg := testConfig() + cfg.Retention = time.Hour + registry := NewRegistry(cfg, NewLocalStore(), slog.New(slog.DiscardHandler)) + serve := func(attachment Attachment) { + handler := registry.Observe(testRoute(ClassTransfer))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), attachment) + _, _ = w.Write([]byte("chunk")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + } + base := testAttachment("") + serve(base) + otherFile := base + otherFile.MediaFileID = 43 + serve(otherFile) + otherUser := base + otherUser.Subject = UserSubject(8) + serve(otherUser) + + if snapshot := registry.Sweep(); len(snapshot.Transfers) != 3 { + t.Fatalf("transfers = %d, want 3 distinct pours", len(snapshot.Transfers)) + } +} + +// HasIdentityConflict and IdentityConflicts must agree. A started-at authority +// upgrade that confirms the recorded instant is not a conflict at all and must +// not consume the per-session budget; one that moves it is, and sets the flag. +func TestStartedAtAuthorityUpgradeAndConflictAgree(t *testing.T) { + at := time.Unix(100, 0) + + t.Run("pure upgrade records nothing", func(t *testing.T) { + session := newLogicalSession(Attachment{Subject: UserSubject(7), SessionID: "s", + StartedAt: at, StartedAtSource: StartedAtSourceFirstSeen}, testConfig(), at) + session.recordConflicts(Attachment{Subject: UserSubject(7), SessionID: "s", + StartedAt: at, StartedAtSource: StartedAtSourceClaim}, at, 16) + if session.hasIdentityConflict || len(session.identityConflicts) != 0 { + t.Fatalf("benign upgrade recorded a conflict: %+v", session.identityConflicts) + } + if session.startedAtSource != StartedAtSourceClaim { + t.Fatalf("authority did not upgrade: %v", session.startedAtSource) + } + }) + + t.Run("moved value sets the flag and the list", func(t *testing.T) { + session := newLogicalSession(Attachment{Subject: UserSubject(7), SessionID: "s", + StartedAt: at, StartedAtSource: StartedAtSourceFirstSeen}, testConfig(), at) + moved := at.Add(-90 * time.Second) + session.recordConflicts(Attachment{Subject: UserSubject(7), SessionID: "s", + StartedAt: moved, StartedAtSource: StartedAtSourceClaim}, at, 16) + if !session.hasIdentityConflict { + t.Fatal("started_at was replaced but HasIdentityConflict is false") + } + if len(session.identityConflicts) != 1 || session.identityConflicts[0].Field != "started_at_replaced" { + t.Fatalf("conflicts = %+v", session.identityConflicts) + } + if !session.startedAt.Equal(moved) { + t.Fatalf("started at = %v, want %v", session.startedAt, moved) + } + }) +} diff --git a/internal/streamtelemetry/session.go b/internal/streamtelemetry/session.go index 6327e8ae2..88d4d8e5e 100644 --- a/internal/streamtelemetry/session.go +++ b/internal/streamtelemetry/session.go @@ -93,8 +93,13 @@ type transfer struct { requestCount int64 route MediaRoute capture CaptureSet - observation *Observation - outcomes map[httpstream.StreamOutcome]int64 + // observations holds every in-flight request folded into this transfer. + // Ranged byte routes (audiobook file reads, download resumes, ebook page + // fetches) issue many overlapping small GETs for the same file, so a + // transfer is a subject pouring one file over one route, not one request — + // which is what requestCount has always claimed to count. + observations map[string]*Observation + outcomes map[httpstream.StreamOutcome]int64 } func newLogicalSession(a Attachment, cfg Config, observedAt time.Time) *logicalSession { @@ -175,16 +180,29 @@ func (s *logicalSession) recordConflicts(a Attachment, observedAt time.Time, max s.playMethods.add(a.PlayMethod) } if rank := startedAtRank(a.StartedAtSource); !a.StartedAt.IsZero() && rank > startedAtRank(s.startedAtSource) { - old := s.startedAt.Format(time.RFC3339Nano) + previous := s.startedAt s.startedAt = a.StartedAt s.startedAtSource = a.StartedAtSource s.startedDegraded = a.StartedAtSource == StartedAtSourceIssuedAt || a.StartedAtSource == StartedAtSourceFirstSeen - if len(s.identityConflicts) < max { - s.identityConflicts = append(s.identityConflicts, IdentityConflict{ - Field: "started_at_replaced", Existing: old, Offered: a.StartedAt.Format(time.RFC3339Nano), ObservedAt: observedAt, - }) - } else { - s.identityOverflowed = true + // Only a change of VALUE is a conflict. A pure authority upgrade that + // confirms the instant already recorded — the common proxy-then-claim + // case — is not one, and recording it consumed the per-session conflict + // budget and could set IdentityConflictsOverflowed for nothing. When the + // value does move, two sources genuinely disagree about when playback + // began, so hasIdentityConflict is set alongside the entry: consumers + // filtering on the flag and consumers reading the list must not + // disagree about whether a session is conflicted. + if !previous.Equal(a.StartedAt) { + s.hasIdentityConflict = true + if len(s.identityConflicts) < max { + s.identityConflicts = append(s.identityConflicts, IdentityConflict{ + Field: "started_at_replaced", + Existing: previous.Format(time.RFC3339Nano), + Offered: a.StartedAt.Format(time.RFC3339Nano), ObservedAt: observedAt, + }) + } else { + s.identityOverflowed = true + } } } } diff --git a/internal/streamtelemetry/store_redis.go b/internal/streamtelemetry/store_redis.go index 8d73892b0..3c33f5b6a 100644 --- a/internal/streamtelemetry/store_redis.go +++ b/internal/streamtelemetry/store_redis.go @@ -123,6 +123,15 @@ func (s *RedisStore) Publish(ctx context.Context, snapshot Snapshot) error { return err } key := s.snapshotKey(snapshot.PublisherID) + // A delta publish rewrites only the fields whose local digest changed, so it + // silently assumes the hash still holds everything else. It may not: a + // maxmemory eviction, an out-of-band DEL, a failover to a replica missing + // the key, or a publish gap long enough for PExpire to lapse all drop it + // without an error. HLEN runs inside the same transaction, so it reports the + // post-write field count; a mismatch means the key was reconstructed from + // the delta alone and the next publish must be full. Costs one pipelined + // command and self-heals in one sweep instead of up to FullResyncEvery. + var fieldCount *redis.IntCmd _, err = s.client.TxPipelined(ctx, func(pipe redis.Pipeliner) error { if full { pipe.Del(ctx, key) @@ -152,6 +161,9 @@ func (s *RedisStore) Publish(ctx context.Context, snapshot Snapshot) error { cutoff := snapshot.CapturedAt.Add(-2 * s.cfg.MembershipTTL).UnixNano() pipe.ZRemRangeByScore(ctx, s.rosterKey(), "-inf", "("+strconv.FormatInt(cutoff, 10)) pipe.PExpire(ctx, s.rosterKey(), 10*s.cfg.MembershipTTL) + if !full { + fieldCount = pipe.HLen(ctx, key) + } return nil }) if err != nil { @@ -164,6 +176,11 @@ func (s *RedisStore) Publish(ctx context.Context, snapshot Snapshot) error { s.published[field] = digest128(value) } s.needFullResync = false + if fieldCount != nil { + if held, hlenErr := fieldCount.Result(); hlenErr != nil || held != int64(len(fields)) { + s.needFullResync = true + } + } s.publishCount++ return nil } diff --git a/internal/streamtelemetry/writer.go b/internal/streamtelemetry/writer.go index 7b74213c7..7b4c96983 100644 --- a/internal/streamtelemetry/writer.go +++ b/internal/streamtelemetry/writer.go @@ -80,6 +80,18 @@ func (w *observedWriter) Write(p []byte) (int, error) { return n, err } +// ReadFrom samples the cut flag once, at entry, whereas Write samples it every +// ~32 KB. That difference is protocol-visible: on HTTP/1.1 the h1 writer is an +// io.ReaderFrom, so a cut cannot interrupt an in-flight transfer and a 20 GB +// direct play drains to the end; on HTTP/2 there is no ReaderFrom, the fallback +// io.Copy goes through Write, and the same cut lands within 32 KB. +// +// Latent today — nothing calls cut.Store and this package is observational +// (doc.go) — and deliberately left that way rather than half-fixed: a per-slice +// cut check here would still only act at readFromChunk granularity, so the two +// protocols would still disagree, just less visibly. The enforcement change that +// introduces a caller for cut.Store owns making the granularity uniform, and +// must land with a test that a cut behaves identically over h1 and h2. func (w *observedWriter) ReadFrom(reader io.Reader) (int64, error) { if w.observation.cut.Load() { return 0, context.Canceled From 813509d6f3304ef96fe3242d76882d390acd94f9 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:54:01 -0400 Subject: [PATCH 20/44] refactor(httpstream): one ForwardReadFrom helper for all nine wrappers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine ResponseWriter wrappers across five packages hand-rolled the same tail: assert the inner writer's io.ReaderFrom, CopyChunked through it, fall back to io.Copy over WriterOnly. Because io.Copy finds ReaderFrom by direct assertion and never through Unwrap, this forwarding is mandatory on every media-route wrapper — so a fix to it had to be re-applied nine times and a missed site silently dropped to the fallback, losing zero-copy sendfile along with that wrapper's byte accounting. Behavior is unchanged; each call site keeps its own chunk size and record callback. Co-Authored-By: Claude Opus 5 (1M context) --- internal/activitylog/middleware.go | 6 +----- internal/api/middleware/metrics.go | 6 +----- internal/api/middleware/request_logger.go | 6 +----- internal/audiobooks/abs/access_log.go | 6 +----- internal/httpstream/readfrom.go | 22 ++++++++++++++++++++++ internal/httpstream/rolling_deadline.go | 8 +------- internal/jellycompat/image_proxy_tags.go | 6 +----- internal/jellycompat/logging.go | 12 ++---------- internal/proxy/egress.go | 6 +----- internal/streamtelemetry/writer.go | 6 +----- 10 files changed, 32 insertions(+), 52 deletions(-) diff --git a/internal/activitylog/middleware.go b/internal/activitylog/middleware.go index e6ccb8757..9c40c2a10 100644 --- a/internal/activitylog/middleware.go +++ b/internal/activitylog/middleware.go @@ -207,11 +207,7 @@ func (w *statusWriter) ReadFrom(src io.Reader) (int64, error) { if !w.wroteHeader { w.status, w.wroteHeader = http.StatusOK, true } - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, 0, nil) + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, 0, nil) } // Hijack implements http.Hijacker, required for WebSocket upgrades. diff --git a/internal/api/middleware/metrics.go b/internal/api/middleware/metrics.go index 9c401b97b..007493eb4 100644 --- a/internal/api/middleware/metrics.go +++ b/internal/api/middleware/metrics.go @@ -78,11 +78,7 @@ func (w *statusWriter) ReadFrom(src io.Reader) (int64, error) { if !w.written { w.status, w.written = http.StatusOK, true } - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, 0, nil) + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, 0, nil) } // Hijack implements http.Hijacker, required for WebSocket upgrades. diff --git a/internal/api/middleware/request_logger.go b/internal/api/middleware/request_logger.go index 271dd3655..55db3ffd1 100644 --- a/internal/api/middleware/request_logger.go +++ b/internal/api/middleware/request_logger.go @@ -101,11 +101,7 @@ func (w *requestStatusWriter) ReadFrom(src io.Reader) (int64, error) { if !w.wroteHeader { w.status, w.wroteHeader = http.StatusOK, true } - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, 0, nil) + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, 0, nil) } func (w *requestStatusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { diff --git a/internal/audiobooks/abs/access_log.go b/internal/audiobooks/abs/access_log.go index 0fa5c6291..490ded580 100644 --- a/internal/audiobooks/abs/access_log.go +++ b/internal/audiobooks/abs/access_log.go @@ -108,11 +108,7 @@ func (s *statusRecorder) ReadFrom(src io.Reader) (int64, error) { if s.status == 0 { s.status = http.StatusOK } - rf, ok := httpstream.ReaderFromOf(s.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(s), src) - } - return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + return httpstream.ForwardReadFrom(s.ResponseWriter, s, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { s.bytes += int(n) }) } diff --git a/internal/httpstream/readfrom.go b/internal/httpstream/readfrom.go index 8a3e49bab..57edb545d 100644 --- a/internal/httpstream/readfrom.go +++ b/internal/httpstream/readfrom.go @@ -17,6 +17,28 @@ func ReaderFromOf(w http.ResponseWriter) (io.ReaderFrom, bool) { return rf, ok } +// ForwardReadFrom forwards a ResponseWriter wrapper's ReadFrom to the writer it +// wraps, preserving both the kernel sendfile path and the wrapper's accounting. +// +// Every wrapper on a media route has to implement this: io.Copy discovers +// io.ReaderFrom by direct type assertion and never through Unwrap, so a single +// wrapper that omits ReadFrom kills zero-copy for the entire chain below it. +// Nine wrappers across five packages hand-rolled the identical tail, which meant +// any fix to the forwarding logic had to be re-applied nine times and a missed +// site silently degraded to the io.Copy fallback. +// +// inner is the wrapped writer; self is the wrapper, used only for the fallback +// so that a writer without a ReaderFrom still routes bytes through the wrapper's +// own Write rather than around it. chunk and record are passed to CopyChunked. +func ForwardReadFrom(inner http.ResponseWriter, self io.Writer, src io.Reader, chunk int64, record func(n int64, err error)) (int64, error) { + rf, ok := ReaderFromOf(inner) + if !ok { + // WriterOnly hides ReadFrom so io.Copy cannot recurse into the caller. + return io.Copy(WriterOnly(self), src) + } + return CopyChunked(rf, src, chunk, record) +} + // CopyChunked drives rf.ReadFrom in slices of chunk bytes, calling record after // each slice. A non-positive chunk performs a single unbounded transfer. func CopyChunked(rf io.ReaderFrom, src io.Reader, chunk int64, record func(n int64, err error)) (int64, error) { diff --git a/internal/httpstream/rolling_deadline.go b/internal/httpstream/rolling_deadline.go index cd7460159..d5fc38107 100644 --- a/internal/httpstream/rolling_deadline.go +++ b/internal/httpstream/rolling_deadline.go @@ -152,12 +152,6 @@ func (s *RollingDeadlineWriter) Write(p []byte) (int, error) { // (sendfile for *os.File bodies, as used by http.ServeContent) while still // rolling the deadline between bounded slices. func (s *RollingDeadlineWriter) ReadFrom(r io.Reader) (int64, error) { - rf, ok := ReaderFromOf(s.w) - if !ok { - // WriterOnly hides this method so io.Copy doesn't recurse into it. - s.bump() - return io.Copy(WriterOnly(s), r) - } if s.statusCode == 0 { s.statusCode = http.StatusOK } @@ -165,7 +159,7 @@ func (s *RollingDeadlineWriter) ReadFrom(r io.Reader) (int64, error) { // headers and then waits on readiness before its first write would otherwise // run that slice against the window set at construction. s.forceBump() - return CopyChunked(rf, r, readFromChunk, func(n int64, err error) { + return ForwardReadFrom(s.w, s, r, readFromChunk, func(n int64, err error) { s.forceBump() s.recordWrite(n, err) }) diff --git a/internal/jellycompat/image_proxy_tags.go b/internal/jellycompat/image_proxy_tags.go index bdf411126..39baaa376 100644 --- a/internal/jellycompat/image_proxy_tags.go +++ b/internal/jellycompat/image_proxy_tags.go @@ -75,11 +75,7 @@ func (w *compatImageProxyTagResponseWriter) ReadFrom(src io.Reader) (int64, erro } func (w *compatImageProxyTagResponseWriter) readFromPassthrough(src io.Reader) (int64, error) { - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, 0, nil) + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, 0, nil) } func (w *compatImageProxyTagResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } diff --git a/internal/jellycompat/logging.go b/internal/jellycompat/logging.go index 20355dd48..272b2051b 100644 --- a/internal/jellycompat/logging.go +++ b/internal/jellycompat/logging.go @@ -41,11 +41,7 @@ func (w *loggingResponseWriter) ReadFrom(src io.Reader) (int64, error) { if w.status == 0 { w.status = http.StatusOK } - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, 0, nil) + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, 0, nil) } func (w *loggingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { @@ -170,11 +166,7 @@ func (w *debugResponseWriter) ReadFrom(src io.Reader) (int64, error) { if w.status == 0 { w.status = http.StatusOK } - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, httpstream.ReadFromChunkDefault, func(n int64, _ error) { w.totalBytes += int(n) }) } diff --git a/internal/proxy/egress.go b/internal/proxy/egress.go index fe8191875..c3edb2c69 100644 --- a/internal/proxy/egress.go +++ b/internal/proxy/egress.go @@ -88,11 +88,7 @@ func (w *meteredResponseWriter) Write(b []byte) (int, error) { } func (w *meteredResponseWriter) ReadFrom(src io.Reader) (int64, error) { - rf, ok := httpstream.ReaderFromOf(w.ResponseWriter) - if !ok { - return io.Copy(httpstream.WriterOnly(w), src) - } - return httpstream.CopyChunked(rf, src, meterChunk, func(n int64, _ error) { + return httpstream.ForwardReadFrom(w.ResponseWriter, w, src, meterChunk, func(n int64, _ error) { w.meter.Add(n) }) } diff --git a/internal/streamtelemetry/writer.go b/internal/streamtelemetry/writer.go index 7b4c96983..be4ab052c 100644 --- a/internal/streamtelemetry/writer.go +++ b/internal/streamtelemetry/writer.go @@ -96,14 +96,10 @@ func (w *observedWriter) ReadFrom(reader io.Reader) (int64, error) { if w.observation.cut.Load() { return 0, context.Canceled } - readerFrom, ok := httpstream.ReaderFromOf(w.w) - if !ok { - return io.Copy(httpstream.WriterOnly(w), reader) - } if w.statusCode == 0 { w.statusCode = http.StatusOK } - return httpstream.CopyChunked(readerFrom, reader, httpstream.ReadFromChunkDefault, func(n int64, err error) { + return httpstream.ForwardReadFrom(w.w, w, reader, httpstream.ReadFromChunkDefault, func(n int64, err error) { if w.bodyEligible { w.observation.AddBytes(n) } From f25ef9b5c06e89e7a35c386c8ed6239318b19757 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:54:12 -0400 Subject: [PATCH 21/44] refactor(streamtelemetry): share viewer-IP, env and client-info helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four families built the same clientip-then-RemoteAddr fallback chain inline while streamtelemetry already had it unexported; a fix to it (IPv6 handling, say) would have had to land in four places or the families would report different viewer addresses into the same merged view. Exported as ViewerIP and adopted everywhere. envEnabled was the ninth independent "is this env var truthy" parser in the tree, each accepting slightly different spellings. Adds internal/envutil and adopts it in both telemetry packages; the remaining copies should migrate as the code around them is touched. checkVersion re-parsed every record into a throwaway header struct before unmarshalling it again into a wire type that already carries the version, so a merged-view rebuild — measured at ~347 ms for 50 000 sessions, nearly all decode — did the JSON work twice. ConfigFromEnv ran twice at startup because the view cache re-read the environment just to get ViewTTL, logging any invalid variable twice; it now takes the TTL off the registry that already parsed it. playbackClientInfoFromRequest wrapped PlaybackClientInfoFromRequest wrapped playback.ClientInfoFromRequest — three names, one body. Callers now use the playback package directly. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/silo/main.go | 9 +++-- internal/api/handlers/playback.go | 10 ------ .../api/handlers/playback_sessions_test.go | 4 +-- internal/api/handlers/playback_v3.go | 4 +-- internal/api/media_routes.go | 14 ++------ internal/envutil/bool.go | 29 +++++++++++++++ internal/jellycompat/media_routes.go | 10 +----- internal/proxy/media_routes.go | 10 +----- internal/streamtelemetry/codec.go | 35 +++++++++---------- internal/streamtelemetry/config.go | 11 +++--- internal/streamtelemetry/registry.go | 10 ++++++ internal/streamtelemetry/route.go | 7 ++++ internal/telemetry/config.go | 14 ++------ 13 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 internal/envutil/bool.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 3c0125383..3d6244661 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -200,11 +200,14 @@ func newStreamTelemetryRegistry(ctx context.Context, nodeID string, redisClient // newStreamTelemetryViewCache builds the bounded-staleness cache the admin // parity endpoint reads. It shares one cached view across every reader so the // merged rebuild is paid at most once per TTL, not once per request. -func newStreamTelemetryViewCache(registry *streamtelemetry.Registry, nodeID string) *streamtelemetry.ViewCache { +func newStreamTelemetryViewCache(registry *streamtelemetry.Registry) *streamtelemetry.ViewCache { if registry == nil { return nil } - return streamtelemetry.NewViewCache(registry, streamtelemetry.ConfigFromEnv(nodeID).ViewTTL, slog.Default()) + // Reads the TTL off the registry rather than calling ConfigFromEnv again: + // a second parse logs every invalid variable twice and the two calls could + // disagree if the environment changed between them. + return streamtelemetry.NewViewCache(registry, registry.ViewTTL(), slog.Default()) } func resolvePluginCacheDir() string { @@ -898,7 +901,7 @@ func main() { if mode == "" || mode == "integrated" || mode == "api" { streamTelemetryRegistry = newStreamTelemetryRegistry(appCtx, nodeID, apiRedisClient) streamTelemetryRegistry.Start(appCtx) - streamTelemetryViewCache = newStreamTelemetryViewCache(streamTelemetryRegistry, nodeID) + streamTelemetryViewCache = newStreamTelemetryViewCache(streamTelemetryRegistry) } // Assigned below once the trusted-proxy config is seeded; captured by the diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index f2a200eff..eb147fa52 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -1108,16 +1108,6 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req h.handleStartPlaybackV3(w, r, body) } -// PlaybackClientInfoFromRequest captures and normalizes playback client headers -// at the HTTP request boundary. -func PlaybackClientInfoFromRequest(r *http.Request) playback.ClientInfo { - return playback.ClientInfoFromRequest(r) -} - -func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo { - return PlaybackClientInfoFromRequest(r) -} - // HandleUpdateProgress handles POST /playback/{session_id}/progress. func (h *PlaybackHandler) HandleUpdateProgress(w http.ResponseWriter, r *http.Request) { userID := apimw.GetUserID(r.Context()) diff --git a/internal/api/handlers/playback_sessions_test.go b/internal/api/handlers/playback_sessions_test.go index 07cc459c9..97a8e3667 100644 --- a/internal/api/handlers/playback_sessions_test.go +++ b/internal/api/handlers/playback_sessions_test.go @@ -412,7 +412,7 @@ func TestPlaybackClientInfoFromRequestClampsHeaders(t *testing.T) { req.Header.Set("X-Silo-Client-Build", strings.Repeat("b", 100)) req.Header.Set("X-Silo-Client-Channel", strings.Repeat("c", 100)) - got := playbackClientInfoFromRequest(req) + got := playback.ClientInfoFromRequest(req) for _, tc := range []struct { field string @@ -441,7 +441,7 @@ func TestNormalizeClientMetadataCountsRunes(t *testing.T) { // counts it, well past it as bytes. req.Header.Set("X-Silo-Client-Channel", strings.Repeat("δ", 40)) - got := playbackClientInfoFromRequest(req) + got := playback.ClientInfoFromRequest(req) if runes := utf8.RuneCountInString(got.Channel); runes != 32 { t.Errorf("Channel runes = %d, want the 32-character bound", runes) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index c66d5a300..430c40a39 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -538,7 +538,7 @@ type playbackStartRequestDigestsV3 struct { // unconditionally would write "web" into the one field that is contractually // semver, on every browser session. func playbackClientInfoForStartV3(r *http.Request, clientContext playback.ClientPlaybackContextV3) playback.ClientInfo { - info := playbackClientInfoFromRequest(r) + info := playback.ClientInfoFromRequest(r) if info.Name == "" { return info } @@ -2772,7 +2772,7 @@ func (h *PlaybackHandler) HandlePlaybackRouteEventV3(w http.ResponseWriter, r *h return } event.Diagnostics = sanitizeDiagnosticsV3(event.Diagnostics) - client := h.playbackClientInfoWithSessionFallbackV3(firstNonEmptyValue(event.SessionID, identity.SessionID), playbackClientInfoFromRequest(r)) + client := h.playbackClientInfoWithSessionFallbackV3(firstNonEmptyValue(event.SessionID, identity.SessionID), playback.ClientInfoFromRequest(r)) h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: event, UserID: userID, ProfileID: profileID, ClientName: client.Name, ClientVersion: client.Version, ClientBuild: client.Build, ClientChannel: client.Channel, ClientModel: event.Diagnostics["device_model"]}) w.WriteHeader(http.StatusAccepted) } diff --git a/internal/api/media_routes.go b/internal/api/media_routes.go index 0c25ca398..57ffbc5d5 100644 --- a/internal/api/media_routes.go +++ b/internal/api/media_routes.go @@ -1,12 +1,10 @@ package api import ( - "net" "net/http" "time" - "github.com/Silo-Server/silo-server/internal/api/handlers" - "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -39,14 +37,8 @@ func nativeRoute(method, pattern string, class streamtelemetry.Class, capRelevan func nativeCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { return func(r *http.Request) streamtelemetry.CaptureSet { - client := handlers.PlaybackClientInfoFromRequest(r) - viewerIP := clientip.FromContext(r.Context()) - if viewerIP == "" { - viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) - if viewerIP == "" { - viewerIP = r.RemoteAddr - } - } + client := playback.ClientInfoFromRequest(r) + viewerIP := streamtelemetry.ViewerIP(r) return streamtelemetry.CaptureSet{ Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, DeviceID: r.Header.Get("X-Silo-Device-ID"), diff --git a/internal/envutil/bool.go b/internal/envutil/bool.go new file mode 100644 index 000000000..3f180823f --- /dev/null +++ b/internal/envutil/bool.go @@ -0,0 +1,29 @@ +// Package envutil holds the parsers shared by everything that reads +// configuration out of the environment. +// +// It exists because "is this environment variable on?" had been re-implemented +// nine times across the tree, each copy accepting a slightly different set of +// spellings — so whether SILO_X=enabled worked depended on which subsystem read +// it. New env flags belong here; the remaining ad hoc copies should migrate as +// the code around them is touched. +package envutil + +import ( + "os" + "strings" +) + +// Truthy reports whether a raw environment value means "on". Case and +// surrounding whitespace are ignored. Anything else, including an empty or +// unset value, is false — a flag has to be turned on deliberately. +func Truthy(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "true", "yes", "on", "enabled": + return true + default: + return false + } +} + +// Bool reports whether the named environment variable is set to a truthy value. +func Bool(name string) bool { return Truthy(os.Getenv(name)) } diff --git a/internal/jellycompat/media_routes.go b/internal/jellycompat/media_routes.go index ea3a22ee8..c646a0d5b 100644 --- a/internal/jellycompat/media_routes.go +++ b/internal/jellycompat/media_routes.go @@ -1,11 +1,9 @@ package jellycompat import ( - "net" "net/http" "time" - "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -47,13 +45,7 @@ func compatRoute(method, pattern string, class streamtelemetry.Class, capRelevan // deliberate; do not "fix" it by moving the wrapper. func compatCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { return func(r *http.Request) streamtelemetry.CaptureSet { - viewerIP := clientip.FromContext(r.Context()) - if viewerIP == "" { - viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) - if viewerIP == "" { - viewerIP = r.RemoteAddr - } - } + viewerIP := streamtelemetry.ViewerIP(r) return streamtelemetry.CaptureSet{ Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, DeviceID: stripCompatNUL(firstMediaBrowserAuthorizationValue(r, "DeviceId")), diff --git a/internal/proxy/media_routes.go b/internal/proxy/media_routes.go index fc3ae6e92..f26cf76f4 100644 --- a/internal/proxy/media_routes.go +++ b/internal/proxy/media_routes.go @@ -1,11 +1,9 @@ package proxy import ( - "net" "net/http" "time" - "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/streamtelemetry" ) @@ -35,13 +33,7 @@ func declareProxyMediaRoutes() { streamtelemetry.DeclareRoutes(proxyMediaRoutes. func proxyCapture(pattern string) func(*http.Request) streamtelemetry.CaptureSet { return func(r *http.Request) streamtelemetry.CaptureSet { client := playback.ClientInfoFromRequest(r) - viewerIP := clientip.FromContext(r.Context()) - if viewerIP == "" { - viewerIP, _, _ = net.SplitHostPort(r.RemoteAddr) - if viewerIP == "" { - viewerIP = r.RemoteAddr - } - } + viewerIP := streamtelemetry.ViewerIP(r) return streamtelemetry.CaptureSet{ Method: r.Method, Pattern: pattern, ViewerIP: viewerIP, DeviceID: r.Header.Get("X-Silo-Device-ID"), diff --git a/internal/streamtelemetry/codec.go b/internal/streamtelemetry/codec.go index 9f5376add..26f0e0a47 100644 --- a/internal/streamtelemetry/codec.go +++ b/internal/streamtelemetry/codec.go @@ -143,15 +143,14 @@ func timeFromUnixNano(value int64) time.Time { return time.Unix(0, value) } -func checkVersion(data []byte) error { - var header struct { - V int `json:"v"` - } - if err := json.Unmarshal(data, &header); err != nil { - return err - } - if header.V != codecVersion { - return errUnsupportedCodecVersion{Version: header.V} +// checkVersion validates the version a record already carries. It takes the +// decoded field rather than the raw bytes: every wire type embeds V, so parsing +// the payload a second time into a throwaway header struct doubled the JSON work +// of every record in a merged-view rebuild — measured at ~347 ms for 50 000 +// sessions, all of it decode. +func checkVersion(version int) error { + if version != codecVersion { + return errUnsupportedCodecVersion{Version: version} } return nil } @@ -185,13 +184,13 @@ func encodeSession(value SessionView) ([]byte, error) { } func decodeSession(data []byte) (SessionView, error) { - if err := checkVersion(data); err != nil { - return SessionView{}, err - } var w wireSession if err := json.Unmarshal(data, &w); err != nil { return SessionView{}, err } + if err := checkVersion(w.V); err != nil { + return SessionView{}, err + } if err := validateSessionWire(w); err != nil { return SessionView{}, err } @@ -264,13 +263,13 @@ func encodeTransfer(value TransferView) ([]byte, error) { } func decodeTransfer(data []byte) (TransferView, error) { - if err := checkVersion(data); err != nil { - return TransferView{}, err - } var w wireTransfer if err := json.Unmarshal(data, &w); err != nil { return TransferView{}, err } + if err := checkVersion(w.V); err != nil { + return TransferView{}, err + } if w.MediaFileID < 0 || w.BytesAccepted < 0 || w.OpenObservations < 0 || w.RequestCount < 0 { return TransferView{}, errors.New("negative transfer counter") } @@ -293,13 +292,13 @@ func encodeMeta(value publisherMeta) ([]byte, error) { } func decodeMeta(data []byte) (publisherMeta, error) { - if err := checkVersion(data); err != nil { - return publisherMeta{}, err - } var value publisherMeta if err := json.Unmarshal(data, &value); err != nil { return publisherMeta{}, err } + if err := checkVersion(value.V); err != nil { + return publisherMeta{}, err + } if value.DroppedObservations < 0 || value.DroppedBytes < 0 || value.UnattributedObservations < 0 || value.UnattributedBytes < 0 || value.SessionCount < 0 || value.TransferCount < 0 { return publisherMeta{}, errors.New("negative publisher metadata counter") } diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index e841121a2..fc7982f48 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -8,6 +8,8 @@ import ( "strings" "time" "unicode" + + "github.com/Silo-Server/silo-server/internal/envutil" ) const ( @@ -107,14 +109,14 @@ func DefaultConfig(nodeID string) Config { // telemetry; invalid distributed-only settings retain local telemetry. func ConfigFromEnv(nodeID string) Config { cfg := DefaultConfig(nodeID) - cfg.Enabled = envEnabled(os.Getenv(enabledEnv)) + cfg.Enabled = envutil.Bool(enabledEnv) coreInvalid := make([]string, 0) distributedInvalid := make([]string, 0) // The operator only owns the variables they actually set. The cross-checks // below relate two knobs, and a violation involving an unset knob is not the // operator's mistake — it is a default that has to move. explicit := make(map[string]bool) - cfg.Distributed = envEnabled(os.Getenv(distributedEnv)) + cfg.Distributed = envutil.Bool(distributedEnv) parseDuration := func(name string, dst *time.Duration) { value := strings.TrimSpace(os.Getenv(name)) if value == "" { @@ -308,8 +310,3 @@ func parseFamilies(value string) (map[Family]bool, bool) { } return families, true } - -func envEnabled(value string) bool { - value = strings.TrimSpace(strings.ToLower(value)) - return value == "1" || value == "true" || value == "yes" || value == "on" -} diff --git a/internal/streamtelemetry/registry.go b/internal/streamtelemetry/registry.go index a831f856c..315d39e86 100644 --- a/internal/streamtelemetry/registry.go +++ b/internal/streamtelemetry/registry.go @@ -95,6 +95,16 @@ func NewRegistry(cfg Config, store SnapshotStore, logger *slog.Logger) *Registry func (r *Registry) Enabled() bool { return r != nil && r.cfg.Enabled } +// ViewTTL exposes the resolved bounded-staleness window so the view cache can be +// built from the config this registry already parsed, rather than reading and +// re-validating every SILO_STREAM_TELEMETRY_* variable a second time. +func (r *Registry) ViewTTL() time.Duration { + if r == nil { + return 0 + } + return r.cfg.ViewTTL +} + func (r *Registry) Store() SnapshotStore { if r == nil { return nil diff --git a/internal/streamtelemetry/route.go b/internal/streamtelemetry/route.go index 2d9873e58..e2b041204 100644 --- a/internal/streamtelemetry/route.go +++ b/internal/streamtelemetry/route.go @@ -117,6 +117,13 @@ func genericCapture(r *http.Request) CaptureSet { } } +// ViewerIP resolves the address to record for the person on the other end of a +// media route: the resolved client IP when clientip.Middleware has run, and the +// transport peer otherwise. Exported because every family's Capture builds the +// same fallback chain, and four copies of it would diverge the moment one is +// fixed (IPv6 bracket handling, say) and the others are not. +func ViewerIP(r *http.Request) string { return viewerIP(r) } + func viewerIP(r *http.Request) string { if r == nil { return "" diff --git a/internal/telemetry/config.go b/internal/telemetry/config.go index a4342dac4..5668ed220 100644 --- a/internal/telemetry/config.go +++ b/internal/telemetry/config.go @@ -11,6 +11,8 @@ import ( "os" "strconv" "strings" + + "github.com/Silo-Server/silo-server/internal/envutil" ) // Protocol identifies the OTLP exporter wire protocol. @@ -93,7 +95,7 @@ type Config struct { // attribute. func LoadConfig(nodeID string) Config { endpoint := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) - enabled := truthy(os.Getenv("SILO_OTEL_ENABLED")) || endpoint != "" + enabled := envutil.Bool("SILO_OTEL_ENABLED") || endpoint != "" serviceName := strings.TrimSpace(os.Getenv("OTEL_SERVICE_NAME")) if serviceName == "" { @@ -156,13 +158,3 @@ func parseProtocol(raw string, fallback Protocol) Protocol { return fallback } } - -// truthy reports whether an env value should be treated as a boolean true. -func truthy(v string) bool { - switch strings.ToLower(strings.TrimSpace(v)) { - case "1", "true", "yes", "on": - return true - default: - return false - } -} From 88451eebe114f4cecd42dc70bd586e684ef7f498 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:54:12 -0400 Subject: [PATCH 22/44] docs: document the admin stream-telemetry parity endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md requires a docs/*-api.md entry and a changelog entry for a client-visible API change. No admin-API document existed — the ~20 sibling routes in the same router block are undocumented too — so this adds one, scoped honestly to what it covers, with the full response shape for GET /api/v1/admin/stream-telemetry/parity and the caveats an operator needs to read a report correctly. Co-Authored-By: Claude Opus 5 (1M context) --- docs/admin-api.md | 77 +++++++++++++++++++++++++++++++++++++++ docs/feature-changelog.md | 9 +++++ 2 files changed, 86 insertions(+) create mode 100644 docs/admin-api.md diff --git a/docs/admin-api.md b/docs/admin-api.md new file mode 100644 index 000000000..57ab7beb0 --- /dev/null +++ b/docs/admin-api.md @@ -0,0 +1,77 @@ +# Admin API + +Server-administration endpoints under `/api/v1/admin`. Every route here requires +an authenticated account with the server-wide `admin` role — the same +authorization as `/api/v1/admin/sessions` — and none of them are part of the +client-facing contract that third-party apps build against. + +This document is new and covers only the routes listed below. The rest of the +admin surface predates it and is currently documented by the code and by the +design documents under `docs/design/`. + +## `GET /api/v1/admin/stream-telemetry/parity` + +Returns the merged stream-telemetry view beside the two legacy live-session +projections an admin reads today, plus the diff between them. + +It is a diagnostic: it compares and does not cut over. No existing admin read has +been repointed onto telemetry, and nothing here blocks, throttles or ends a +session. Design: [`docs/design/2026-08-17-stream-telemetry.md`](design/2026-08-17-stream-telemetry.md). + +The view is served from a bounded-staleness cache with single-flight refresh, so +several admins polling this route pay at most one rebuild per TTL. + +### Response + +Always `200 OK`. "Nothing to compare" is expressed in the body rather than as an +error status, because an empty report with a success status would read as +agreement. + +| Field | Type | Meaning | +|---|---|---| +| `enabled` | bool | Stream telemetry is running in this process. | +| `reason` | string | Present when there is nothing to compare (telemetry disabled, or no view built yet). | +| `view` | object | State of the merged view the comparison was built from. | +| `sources` | array | One report per legacy projection. Empty when `enabled` is false. | + +`view`: + +| Field | Type | Meaning | +|---|---|---| +| `available` | bool | A merged view exists. | +| `built_at` | RFC3339 string | When it was built. Omitted if never. | +| `age_ms`, `stale` | int, bool | Age of the cached view, and whether it exceeded the TTL. | +| `build_took_ms` | int | Cost of the last rebuild. | +| `refreshes`, `failures`, `last_error` | int, int, string | Cache counters since process start. | +| `complete` | bool | No publisher was stale, degraded or truncated. | +| `incomplete_reasons` | string[] | Why `complete` is false — e.g. `missing_publisher`, `publisher_truncated`, `decode_errors`, `truncated`. | +| `missing_publishers` | string[] | Publisher ids present in the roster but with no usable snapshot. | +| `clock_skew_suspected` | bool | A publisher stamped a time in the future. A clock running *behind* is indistinguishable from a stalled publisher in one sample; compare `publishers` sequence across two reads to tell them apart. | +| `publishers` | string[] | `=`, where state is `fresh`, `degraded`, `stale` or `departed`. | +| `session_count`, `transfer_count` | int | Sizes of the merged view. | + +Each entry in `sources`: + +| Field | Type | Meaning | +|---|---|---| +| `source` | string | `playback_sessions_sync` or `node_sessions`. | +| `available` | bool | The projection could be read. | +| `error` | string | Why it could not. | +| `notes` | string[] | Caveats that apply to this comparison. | +| `report` | object | The diff, when available. | + +`report`: + +| Field | Type | Meaning | +|---|---|---| +| `telemetry_count`, `legacy_count`, `in_both` | int | Session counts on each side and their intersection. | +| `agrees` | bool | Same session set, and no field both sides express disagrees. Read `fields_absent` before treating this as clearance to cut over. | +| `telemetry_only`, `legacy_only` | string[] | Session ids present on one side only, capped. | +| `telemetry_only_truncated`, `legacy_only_truncated` | int | How many ids the cap dropped. | +| `mismatches` | object[] | Per-session field disagreements, capped. | +| `mismatches_truncated` | int | How many the cap dropped. | +| `fields_absent` | object | Per field, sessions both sides know where one side carries no value. A gap in a projection, not a disagreement. | + +A single report samples three independently updated stores, so one-sided +differences are normal and are not on their own evidence of a defect. Repeated +agreement over time is what the legacy-retirement project is gated on. diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index 8fd5a0fdd..fce8a9f92 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -1,5 +1,14 @@ # Feature Changelog +## 2026-08-22 + +### Measure delivered bytes on every serving path +Silo now measures what it actually sends, rather than trusting what a client reports it is watching. +- Every byte-serving route across the API server, Jellyfin-compatibility layer, standalone proxy, audiobook listener and transcode nodes reports what it served, to whom and how fast, off the hot path. +- Adds `GET /api/v1/admin/stream-telemetry/parity`, which puts the merged measurement beside the two live-session views admins read today and diffs them. See [docs/admin-api.md](admin-api.md). +- Makes no decisions: nothing is blocked, throttled or ended, and no existing admin view was repointed onto it. +- Fixes four defects on the byte paths themselves — proxied streams recorded against no owner, the proxy's own address recorded as the viewer's, the kernel sendfile fast path dead through the proxy chain, and stream tokens with no reliable creation time. + ## 2026-04-09 Covers commits from 2026-04-08 22:32 EDT through 2026-04-09 20:02 EDT. From a1ba4d648fdd94956489657c36bbe9bdbd2408de Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:42:26 -0400 Subject: [PATCH 23/44] docs: distill the streaming write-deadline design into architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #675 pruned docs/superpowers and the shipped design artifacts, distilling the durable content into docs/architecture first. The streaming write-deadline document was deleted on main under that rollup while this branch was extending its writer-chain conformance section, which is the whole of the conflict between the two. This carries the durable half forward on main's own pattern: the invariants a future change has to respect — the rolling-deadline contract, why slice size is a correctness constraint rather than a knob, the two rules every ResponseWriter wrapper on a media route must follow, the one-limiter sendfile trap, why chi's compressor is bypassed rather than repaired, and how conformance is actually verified. The one-shot half — the 2026-07-09 debugging session, the per-file application table, the rollout plan, the silo-apple follow-up list — goes with the deletion. Also records the two rules this branch's review turned up: the bump throttle belongs to Write and never to a ReadFrom slice, and the proxy egress meter has the same shape of constraint at a different value. Co-Authored-By: Claude Opus 5 (1M context) --- docs/architecture/streaming-write-deadline.md | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docs/architecture/streaming-write-deadline.md diff --git a/docs/architecture/streaming-write-deadline.md b/docs/architecture/streaming-write-deadline.md new file mode 100644 index 000000000..1262d191a --- /dev/null +++ b/docs/architecture/streaming-write-deadline.md @@ -0,0 +1,124 @@ +# Streaming write deadlines and writer-chain conformance + +The main API `http.Server` sets `WriteTimeout: 120s`. Go's `WriteTimeout` is an +**absolute deadline from the start of each request**, not an idle timeout, so every +response still being written at T+120s is killed mid-body — including a perfectly +healthy multi-gigabyte media stream. + +`internal/httpstream.RollingDeadlineWriter` replaces that contract for streaming +responses only. The per-response deadline is pushed forward as the body makes +progress, so the semantics change from "must complete within 120s" to **"must make +progress at least every `window` seconds"**. A response that keeps moving lives +indefinitely; a stalled one is still reaped inside the window. + +The server-level 120s guard stays exactly as it is. Streaming handlers opt out +per-response; every JSON, image and other ordinary API route keeps it. + +## The contract + +- The deadline is set through `http.NewResponseController(w).SetWriteDeadline`. A + per-request controller deadline overrides the server-level `WriteTimeout` for that + response — the mechanism the stdlib provides for exactly this case. +- `window` defaults to 180s, overridable via `SILO_STREAM_WRITE_STALL_TIMEOUT` + (integer seconds). +- If the transport does not support per-response write deadlines, the wrapper degrades + to a plain pass-through and the server-level `WriteTimeout` stays in effect. +- A paused client that stops reading for longer than `window` has its connection + reaped. That is intended: the client's reconnect ladder resumes at its byte cursor, + and this is a strict improvement on the 120s that applied to *all* streams before. + +### Bumps are throttled on `Write`, never on `ReadFrom` + +`Write` bumps at most once per `bumpStep` (~15s) so a fast stream issues one +`SetWriteDeadline` per step rather than one per 32 KB chunk. + +**That throttle must not be applied to `ReadFrom` slices.** A slice is already bounded +at `readFromChunk`, so throttling around one saves at most a syscall per 4 MiB and +costs correctness: a slice completing less than a step after the last bump would get no +refresh, and the next slice would start with as little as `window - step` remaining. +That raises the sustained rate a client must hold from the documented floor to roughly +203 kbit/s and reaps healthy slow clients. Every slice — including the first — gets a +full window. + +### Slice size is a correctness constraint, not a tuning knob + +The deadline is an *absolute* time, so a write attempted after it fails immediately. +Slice size divided by window is therefore a **hard floor on the sustained client rate**. + +The original 64 MiB slice against a 180s window implied ~3 Mbit/s: any slower client had +its deadline expire part-way through a single slice and was reaped despite continuous +progress. The slice is `httpstream.ReadFromChunkDefault` — 4 MiB, a ~186 kbit/s floor. +Raising it re-introduces the reap. + +The proxy's egress meter has the *same* shape of constraint for a different reason and +therefore a different value: it credits a rolling per-second ring only when a slice +completes, so `internal/proxy.meterChunk` is 256 KiB. At 4 MiB a 200–500 kbit/s viewer +takes 60–170s per slice and reads as zero egress for most samples of the 60s window, +which under-reports committed bandwidth and lets the planner over-admit. + +## Writer-chain conformance + +`RollingDeadlineWriter` is not the only `http.ResponseWriter` on a media route, and the +ones above it in the chain can silently defeat it. Two rules apply to **every** wrapper +mounted on a path that serves media: + +- **Forward `ReadFrom`.** `io.Copy` discovers `io.ReaderFrom` by direct type assertion + and never consults `Unwrap()`, so a single wrapper without `ReadFrom` disables + sendfile for everything below it. Wrappers that count bytes must transfer in bounded + slices and credit each one, or a large transfer lands in a single accounting bucket. + Use `httpstream.ForwardReadFrom`, which is the one implementation of this tail — + hand-rolling it is how sites drift apart. +- **Implement `Unwrap()`.** Without it, `http.ResponseController` dead-ends at that + wrapper and `SetWriteDeadline` fails, degrading the rolling writer to a plain + pass-through — the deadline is silently gone. Preserve `Hijacker` too wherever a + wrapper could sit over an upgradable route (ABS socket.io, the playback control + websocket). + +### Forwarding `ReadFrom` is necessary but not sufficient + +Go's kernel sendfile path unwraps **exactly one** `io.LimitedReader` before it looks for +the `*os.File`, and `http.ServeContent`'s `io.CopyN` already contributes that one. An +accounting layer that hands down a *freshly nested* limiter therefore forfeits sendfile +even though it forwards `ReadFrom` correctly. + +`httpstream.CopyChunked` slices the caller's limiter over the same underlying reader +instead of nesting a new one. **If you change it, re-run the +`strace -f -e trace=sendfile` comparison over a mounted router.** A byte-exact body and +a correct `Range` status prove HTTP correctness, not sendfile. + +### chi's compressor is bypassed, not repaired + +chi's `middleware.Compress` cannot be fixed in place: `compressResponseWriter` +implements `Unwrap`/`Flush`/`Hijack`/`Push` but **not** `ReadFrom`, and its handler +wraps unconditionally — the encoder is chosen later, so even a non-compressible content +type still gets a sendfile-killing wrapper. + +It is bypassed on exact bulk-media routes via `httpstream.CompressExcept`, matching only +the registered GET/HEAD methods with exact segment counts and exact casing. A *blanket* +bypass would be wrong: subtitle font bundles are JSON served under the same global +compressor, and bypassing them would drop `Content-Encoding`/`Vary` and change the wire +contract. + +### Stream telemetry sits inside the same contract + +`streamtelemetry.observedWriter` is inserted between an enrolled route handler's +`RollingDeadlineWriter` and the real response writer, on every enrolled route family. It +follows the same rules: bounded `ReadFrom` forwarding preserves sendfile, `Unwrap` +preserves deadline traversal, and `Flush`, `Hijack` and `Push` retain their +optional-interface behavior. The outer compressor still bypasses only exact bulk routes; +subtitle-font JSON remains compressible. + +## How conformance is verified + +Handler-level tests bypass exactly the middleware this concerns, so they cannot prove +any of the above. Conformance is verified by driving the **mounted routers over real +sockets** (`internal/api`, `internal/jellycompat`, `internal/audiobooks/abs`, +`internal/proxy`), covering GET/HEAD, single and multi-range, conditional responses, +`Accept-Encoding` present and absent, HTTP/2, the proxy→node hop, and the socket.io +upgrade. + +The deadline behavior itself is pinned by `internal/httpstream/readfrom_deadline_test.go`, +which asserts that a slow-but-progressing stream survives a window far shorter than the +whole transfer, that an oversized slice is still reaped (so nobody restores a large slice +without noticing), and that neither the production bump throttle nor a long pause before +the first write shortens the window a slice runs against. From 6b3bca2d729e5f11fffcd8e0f1627a6e3d77d800 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:32:31 -0400 Subject: [PATCH 24/44] chore: ignore skill secrets and state paths - Ignore `.secrets` and `.state` paths regardless of whether they are files or directories --- .claude/skills/silo-discord-triage/.gitignore | 4 ++-- .gitignore | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.claude/skills/silo-discord-triage/.gitignore b/.claude/skills/silo-discord-triage/.gitignore index 88de7c7b1..23a337910 100644 --- a/.claude/skills/silo-discord-triage/.gitignore +++ b/.claude/skills/silo-discord-triage/.gitignore @@ -1,5 +1,5 @@ -.secrets/ -.state/ +.secrets +.state __pycache__/ *.pyc *.env diff --git a/.gitignore b/.gitignore index 4811c7b8a..9df4fa4fe 100644 --- a/.gitignore +++ b/.gitignore @@ -63,8 +63,8 @@ docs/superpowers/ !/.claude/skills/ # Skills may carry per-developer state next to their instructions (the global # *.local.md rule above covers notes; these cover tokens and run state). -**/.secrets/ -**/.state/ +**/.secrets +**/.state .cursor/ .superpowers/ .codex/ From 5b7cb638f99a32e4dc062d41dae615178953ce9f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 01:37:18 -0400 Subject: [PATCH 25/44] feat(playback): tokenless V3 playback, DV7 client transforms, admin transcode honesty Playback protocol V3: - Tokenless playback: header-authenticated media with signed stream URL reconstruction, sticky per-attempt feature set, and tokenless subtitle delivery (playback_v3, resolver, transcode manager, protocol_v3). - Downloads and auth updates supporting the same flow; access-group clause coverage for repository queries. Admin activity honesty: - Plumb target_audio_channels end to end (new migration, session sync, reconciler, admin session payload, web types) so a transcode target renders its real output layout ("AAC 5.1"), falling back to the bare codec when unknown - never the source channel count. - Rename the "Audio SW" chip to "Audio Transcode"; it labels a plan decision (video copied, audio transcoded), not a client capability. Client counterpart: silo-apple branch t3code/replace-custom-engine-aether (AetherEngine player). This server branch is required for that client - AetherEngine playback negotiation (tokenless media, DV Profile 7 client-transform grants) does not work against older servers. Co-Authored-By: Claude Fable 5 --- cmd/silo/session_sync.go | 1 + docs/architecture/playback-protocol-v3.md | 63 +++- docs/architecture/v1-scope.md | 1 + docs/downloads-api.md | 71 +++++ internal/api/handlers/downloads.go | 42 +-- internal/api/handlers/downloads_test.go | 78 ++++- internal/api/handlers/playback.go | 42 ++- internal/api/handlers/playback_sessions.go | 126 ++++---- internal/api/handlers/playback_v3.go | 273 ++++++++++++----- internal/api/handlers/playback_v3_test.go | 121 +++++++- .../handlers/playback_v3_tokenless_test.go | 284 ++++++++++++++++++ .../api/handlers/playback_v3_union_test.go | 2 +- internal/auth/repository.go | 111 ++++--- .../repository_access_group_clause_test.go | 121 ++++++++ internal/downloads/policy_test.go | 61 ++++ internal/nodepool/planner.go | 49 ++- .../attempt_sticky_features_v3_test.go | 64 ++++ internal/playback/protocol_v3.go | 50 ++- internal/playback/resolver.go | 52 +++- internal/playback/resolver_test.go | 146 ++++++++- internal/playback/transcode_manager.go | 16 +- internal/playback/transcode_manager_test.go | 19 ++ internal/worker/reconciler.go | 32 +- ...playback_session_target_audio_channels.sql | 13 + web/src/api/types.ts | 4 + web/src/components/UserPolicyFields.tsx | 7 + web/src/pages/AdminUserDetail.tsx | 5 +- web/src/pages/AdminUsers.tsx | 5 +- .../pages/admin-settings/InvitationsTab.tsx | 3 +- .../pages/adminActivityPresentation.test.ts | 34 ++- web/src/pages/adminActivityPresentation.ts | 40 ++- 31 files changed, 1623 insertions(+), 313 deletions(-) create mode 100644 internal/api/handlers/playback_v3_tokenless_test.go create mode 100644 internal/auth/repository_access_group_clause_test.go create mode 100644 internal/playback/attempt_sticky_features_v3_test.go create mode 100644 migrations/sql/20260823101500_add_playback_session_target_audio_channels.sql diff --git a/cmd/silo/session_sync.go b/cmd/silo/session_sync.go index 533c45e74..b3879a65a 100644 --- a/cmd/silo/session_sync.go +++ b/cmd/silo/session_sync.go @@ -35,6 +35,7 @@ func buildLiveSessionSync(s *playback.Session, reportingNode string) worker.Sess TargetResolution: s.TargetResolution, TargetVideoCodec: s.TargetVideoCodec, TargetAudioCodec: s.TargetAudioCodec, + TargetAudioChannels: s.TargetAudioChannels, TargetBitrateKbps: s.TargetBitrateKbps, TranscodeHWAccel: s.TranscodeHWAccel, StartedAt: s.StartedAt, diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index af3677414..220d2867a 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -486,7 +486,13 @@ A pooled transcode node may still execute HLS behind the API server; the API relays its manifest and segments over the same authenticated client route. Direct-play and progressive-remux proxy routes are bypassed because those nodes accept a signed URL token rather than the user's API credential, so the -normal local-remux fallback policy still applies. +normal local-remux fallback policy still applies. On a server that disables +`playback.local_transcode_fallback`, a progressive remux needing a server +transformation therefore has no executor at all: the server plans the same +recipe as `server_remux_hls` instead, which a pooled transcode node can run +behind the API. A client that advertises no HLS delivery gets the non-retryable +terminal `local_transcode_disabled` rather than a retryable capacity error it +could only retry forever. The client must attach its current `Authorization: Bearer ...` header to the manifest/file request and every derived request, including HLS segments, @@ -505,6 +511,34 @@ sticky for the lifetime of the attempt; a client that can no longer honor it must stop and start a new attempt rather than downgrade a replan to a credential-bearing URL. +### 4.2 Media and subtitle URL query parameters + +Every URL a plan publishes belongs to one of two route families, and the query +parameters each family accepts are part of the contract. A client replays the +URL it was handed byte-for-byte; it never composes one, never drops a +parameter, and never carries a parameter across families. + +| Route family | Routes | Query parameters | +| --- | --- | --- | +| Media | `/stream/{session_id}`, `/playback/transcode/{session_id}/master.m3u8` and its segments | `seek` only — the progressive-remux start offset in seconds, present only when it is non-zero | +| Subtitle artifact | `/stream/{session_id}/subtitles/{combined_index}{.ext}`, `/stream/{session_id}/subtitles/{combined_index}/fonts` | `file_id`, always; plus `downloaded_subtitle_id` when the track is a downloaded or AI-generated one (§8) | + +A media route never carries `file_id` or `downloaded_subtitle_id` — the session +already names the file it plays, and the media timeline is anchored by `seek` +plus the fields in §5. A subtitle route never carries `seek`: a sidecar is +fetched whole and timed against `subtitle.artifact.timing_origin_seconds`. + +`file_id` is required on a subtitle route because a plan can fall back to an +alternate edition, so the session id alone does not fix which file's ordinal +space `{combined_index}` addresses. `downloaded_subtitle_id` pins the exact +downloaded row behind that ordinal, which is what keeps the URL stable when the +downloaded segment of the inventory is reordered or grows mid-session (§8). + +An attempt that did not opt into `header_authenticated_media_v1` additionally +carries the signed stream token `st` on its media URLs — never on subtitle or +font-bundle routes. It is an opaque transport credential rather than a playback +parameter, and it is outside the table above. + --- ## 5. The timeline model @@ -635,6 +669,20 @@ A seek-scoped recovery refuses to accept new capability or device evidence: a seek is not an authority boundary for replacing the client's declared abilities mid-session. +**Attempt-sticky features.** `client_features` is otherwise refreshed by any +replan that sends it, but two entries are fixed by the start negotiation and a +replan can neither add nor drop them: + +| Feature | Why it is fixed | +| --- | --- | +| `header_authenticated_media_v1` | It selects the media security contract. A signed URL from an earlier plan stays usable until its recipe expires, so a mid-attempt switch would leave two contracts alive for one session (§4.1) | +| `software_video_decode_v1` | It widens the direct-play evidence tiers. Dropping it converts a direct route into a transcode and persists that downgrade into the durable request | + +The server silently restores the negotiated state of both, whatever the replan +sends — including an explicit list that omits one, which is otherwise a valid +way to drop a feature. Seek replans never replace the feature list at all. +Changing either mode means stopping and starting a new attempt. + --- ## 7. Registries @@ -690,6 +738,7 @@ HDR, 4K, or transcode-policy reason — deselecting the subtitle restores playba *Transport and session:* `internal_error`, `session_expired`, `subtitle_artifact_unavailable`, `capacity_unavailable`, +`local_transcode_disabled`, `audio_transcoding_disabled`, `transcode_start_failed`, `transcode_node_unavailable`, `transcode_node_capability_unavailable`, `track_unavailable`, @@ -764,6 +813,18 @@ transcodes it to a client-renderable format first — always to WebVTT, served a `text/vtt` at a `.vtt` URL), or `burn_in` (rendered into the video, which forces a transcode). +`subtitle.artifact` is the one track the plan tells the client to draw, and it +is present **only** under `render` and `convert`. Under `off` and `burn_in` it +is absent, and every plan states this afresh: an artifact is never carried over +from an earlier plan of the same session, so a client must take the current +plan's `subtitle` block literally rather than remembering the previous one. +`off` also carries no `subtitle.track_id`. The inventory `url`s are unaffected +— they describe what is fetchable, not what is selected, and stay published in +every mode. + +Subtitle artifact, inventory and font-bundle URLs are session-scoped and carry +their own query parameters; see §4.2 for the per-route-family contract. + The sidecar URL suffix is part of the representation contract, not decoration. An embedded `hdmv_pgs_subtitle`/PGS sidecar is lossless binary PGS at a `.sup` URL with `application/octet-stream`; cached full-track responses support `HEAD` diff --git a/docs/architecture/v1-scope.md b/docs/architecture/v1-scope.md index 90be03dbb..039c2d94b 100644 --- a/docs/architecture/v1-scope.md +++ b/docs/architecture/v1-scope.md @@ -38,6 +38,7 @@ justification and falls back to the Deprecation/Sunset flow like anything else. | `PATCH /api/v1/playback/{session_id}/audio` | Playback protocol v3, [spec](playback-protocol-v3.md) | Superseded by the `track_change` replan operation, which changes the audio track *and* returns the resulting plan. The PATCH mutated the session without re-planning, so a track change that invalidated the route left the client playing a plan the server no longer agreed with. | | `409 protocol_disabled` on `POST /api/v1/playback/route-events`, and the `"enabled": false` shape of `GET /api/v1/playback/capability` | Playback protocol v3, [spec](playback-protocol-v3.md) | Both described a server with v3 switched off. With v3 the only playback protocol that state cannot exist — "disabled" would mean "no playback at all". The `enabled` field itself is kept and is constant `true`, so clients that feature-detect against it keep working; only the negative shape and the status code go. | | The draft-v3 platform-specific wire vocabulary: `ClientPlaybackContextV3.features`, `.platform`, and `.engines`; `PlanV3.engine`; `output_route_generation` in start, replan, output-context, and route-event bodies; Android device/build fields (`brand`, `device`, `product`, `soc_*`, `build_*`, `security_patch`, `sdk_int`, `abis`); and the `media3_only` / `detailed_decode_capabilities` feature tokens | Platform-neutral playback protocol v3, [spec](playback-protocol-v3.md) | These names exposed one client's implementation as the cross-platform contract. Before v1 lock they are replaced by neutral delivery classes, evidence tiers, `device.platform` / `device.os_version` / bounded `platform_details`, opaque `output_context_id`, and top-level feature advertisement. Carrying both drafts through lock would force every client to translate Media3-specific aliases indefinitely and leave two conflicting sources of capability truth. | +| Accepting `access_group_id` alongside the admin role on `POST /api/v1/admin/users`, `PUT /api/v1/admin/users/{id}`, and `POST /api/v1/admin/invitations` — all three now reject the combination with `422` (`ErrAdminGrouped`, "Admin accounts cannot belong to an access group") | Admin-ungrouped constraint, 2026-08-22 | Admins are never grouped: the household access-group ceiling has no meaning for an account that already has server-wide admin rights, and silently accepting a group on an admin account left a stored value that read as a policy nobody enforced. Rejecting the combination at write time is cheaper to carry than a deprecation window for a field whose only valid value on an admin account was already `null`. | Feature-detection precedent: clients discover which metadata providers (including the built-in NFO provider, #216) apply to a library type via diff --git a/docs/downloads-api.md b/docs/downloads-api.md index 9d03faeef..8c5d107d7 100644 --- a/docs/downloads-api.md +++ b/docs/downloads-api.md @@ -220,6 +220,43 @@ Capabilities mirror streaming playback caps: } ``` +`caps` fields: + +| Field | Type | Notes | +| -------------------------- | -------- | ------------------------------------------------------------------------------------------------------- | +| `codecs_video` | string[] | Flat list of video codecs the device can decode. | +| `codecs_audio` | string[] | Flat list of audio codecs the device can decode. | +| `audio_passthrough_codecs` | string[] | Codecs the connected sink accepts bit-exact. | +| `containers` | string[] | Containers the device can open. | +| `max_resolution` | string | Coarse device ceiling (`480p`…`2160p`). See the note under detailed evidence below. | +| `hdr` | bool | Whether the display can present HDR. | +| `client_features` | string[] | Optional protocol-v3 feature tokens, e.g. `software_video_decode_v1`. Same vocabulary as playback start. | +| `video_evidence` | string | Optional provenance of the video facts: `declared`, `platform_attested`, or `exact`. | +| `video_decode` | object[] | Optional per-decoder entries; same shape and bounds as the protocol-v3 `video_decode[]`. | + +The last three fields are additive and optional. They carry the same meaning as +on the v3 playback start request — see +[docs/architecture/playback-protocol-v3.md](architecture/playback-protocol-v3.md) +— and download creation accepts exactly the shapes playback accepts: + +- Flat lists alone, with or without `video_evidence`, are always valid. A + `declared` payload and a payload carrying only `client_features` both resolve + from the flat codec lists. +- `video_decode` entries are only honoured at `video_evidence` of `exact` or + `platform_attested`, because no weaker tier can validate them. Sending + `video_decode` entries with `declared`, or with `video_evidence` omitted, is a + partial opt-in the server will not silently ignore: it returns `400` + `bad_request`. Malformed entries (empty `codec`, negative bounds, oversized + lists) are rejected with `400` at any tier. + +When a strict tier does supply entries, they decide whether a particular +original file is safe to hand over as-is, and they supersede the coarse +`max_resolution` ceiling — a `max_width: 3840` hardware entry preserves a 4K +original even when `max_resolution` says `1080p`. If the file's stored probe +metadata is too sparse to check against those bounds (missing bit depth, +dimensions, frame rate, or bitrate), the server falls back to the flat codec +lists rather than forcing a transcode of an original-quality download. + Single-item response (`202 Accepted`): ```json @@ -902,6 +939,40 @@ For Apple TV 4K or modern HDR-capable devices, the client may advertise `4k` and `hdr: true`; older phones/tablets should stay conservative. These caps affect only server-side compatibility decisions and bitrate transcode targets. +A client with real decoder facts can send `video_evidence` and `video_decode` +alongside the flat lists. Note what that changes: detailed entries supersede the +coarse `max_resolution` ceiling, so a conservative `"max_resolution": "1080p"` +no longer bounds anything once `video_decode` describes a decoder that reaches +higher. If a hard 1080p cap is the intent, bound the entries themselves +(`max_width: 1920`, `max_height: 1080`, and the matching frame-rate and bitrate +limits) rather than relying on `max_resolution`. + +```json +{ + "caps": { + "client_features": ["software_video_decode_v1"], + "video_evidence": "platform_attested", + "codecs_video": ["h264", "hevc"], + "codecs_audio": ["aac", "ac3", "eac3"], + "audio_passthrough_codecs": ["ac3", "eac3"], + "containers": ["mp4", "mov", "m4v"], + "max_resolution": "1080p", + "hdr": false, + "video_decode": [ + { + "codec": "hevc", + "bit_depths": [8, 10], + "max_width": 1920, + "max_height": 1080, + "max_frame_rate": 60, + "max_bitrate_kbps": 40000, + "hardware": true + } + ] + } +} +``` + ### 10.5 Download orchestration For a single movie or episode: diff --git a/internal/api/handlers/downloads.go b/internal/api/handlers/downloads.go index f76b1b288..66c786776 100644 --- a/internal/api/handlers/downloads.go +++ b/internal/api/handlers/downloads.go @@ -99,26 +99,16 @@ const ( // downloadRequest represents the JSON body for POST /downloads. type downloadRequest struct { - ContentID string `json:"content_id"` - EpisodeID string `json:"episode_id,omitempty"` - FileID int `json:"file_id,omitempty"` - Quality string `json:"quality,omitempty"` // original (default) | 20mbps | 10mbps | 5mbps | 2mbps | 1mbps - Series bool `json:"series,omitempty"` // if true, downloads all episodes - Season *int `json:"season_number,omitempty"` // with series=true, downloads only this season (0 = Specials) - Caps *downloadCaps `json:"caps,omitempty"` // device decode capability (original fallback / transcode target) -} - -// downloadCaps mirrors playback.ClientCapabilities for the request body. -type downloadCaps struct { - ClientFeatures []string `json:"client_features,omitempty"` - VideoEvidence playback.CapabilityEvidenceV3 `json:"video_evidence,omitempty"` - CodecsVideo []string `json:"codecs_video,omitempty"` - CodecsAudio []string `json:"codecs_audio,omitempty"` - AudioPassthroughCodecs []string `json:"audio_passthrough_codecs,omitempty"` - Containers []string `json:"containers,omitempty"` - MaxResolution string `json:"max_resolution,omitempty"` - HDR bool `json:"hdr,omitempty"` - VideoDecode []playback.VideoDecodeCapabilityV3 `json:"video_decode,omitempty"` + ContentID string `json:"content_id"` + EpisodeID string `json:"episode_id,omitempty"` + FileID int `json:"file_id,omitempty"` + Quality string `json:"quality,omitempty"` // original (default) | 20mbps | 10mbps | 5mbps | 2mbps | 1mbps + Series bool `json:"series,omitempty"` // if true, downloads all episodes + Season *int `json:"season_number,omitempty"` // with series=true, downloads only this season (0 = Specials) + // Caps is the device decode capability (original fallback / transcode + // target). The playback type is decoded directly: its JSON contract is the + // download `caps` contract, and a mirror struct here could only drift. + Caps *playback.ClientCapabilities `json:"caps,omitempty"` } // patchDownloadRequest is the JSON body for PATCH /downloads/{id}. @@ -278,17 +268,7 @@ func (h *DownloadHandler) HandleCreateDownload(w http.ResponseWriter, r *http.Re DevicePlatform: devicePlatform, } if req.Caps != nil { - createReq.Caps = playback.ClientCapabilities{ - ClientFeatures: req.Caps.ClientFeatures, - VideoEvidence: req.Caps.VideoEvidence, - CodecsVideo: req.Caps.CodecsVideo, - CodecsAudio: req.Caps.CodecsAudio, - AudioPassthroughCodecs: req.Caps.AudioPassthroughCodecs, - Containers: req.Caps.Containers, - MaxResolution: req.Caps.MaxResolution, - HDR: req.Caps.HDR, - VideoDecode: req.Caps.VideoDecode, - } + createReq.Caps = *req.Caps if err := createReq.Caps.NormalizeAndValidateVideoDecode(); err != nil { writeError(w, http.StatusBadRequest, "bad_request", err.Error()) return diff --git a/internal/api/handlers/downloads_test.go b/internal/api/handlers/downloads_test.go index 0ce195744..558a569a9 100644 --- a/internal/api/handlers/downloads_test.go +++ b/internal/api/handlers/downloads_test.go @@ -435,15 +435,82 @@ func TestHandleCreateDownloadRejectsUnboundedDetailedDecoderInput(t *testing.T) } } -func TestHandleCreateDownloadRejectsSoftwareOptInWithoutDetailedEvidence(t *testing.T) { +// Flat-list payloads are legal at every evidence tier on the v3 playback start +// path, so download creation must accept the same shapes rather than 400 on +// them. Only video_decode entries the tier cannot validate are refused. +func TestHandleCreateDownloadAcceptsFlatCapabilityPayloads(t *testing.T) { + tests := []struct { + name string + caps string + }{ + { + name: "declared evidence with flat lists", + caps: `{ + "video_evidence":"declared", + "codecs_video":["h264","hevc"], + "codecs_audio":["aac"], + "containers":["mp4"], + "max_resolution":"1080p" + }`, + }, + { + name: "feature token only", + caps: `{ + "client_features":["software_video_decode_v1"], + "codecs_video":["av1"], + "codecs_audio":["aac"], + "containers":["mp4"] + }`, + }, + { + name: "platform attested without entries", + caps: `{ + "client_features":["software_video_decode_v1"], + "video_evidence":"platform_attested", + "codecs_video":["av1"] + }`, + }, + { + name: "legacy flat payload", + caps: `{ + "codecs_video":["h264"], + "codecs_audio":["aac"], + "containers":["mp4"], + "max_resolution":"1080p" + }`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + svc := &fakeDownloadService{created: &downloads.Download{ + ID: "dl1", ContentID: "c1", Status: downloads.StatusQueued, + Format: downloads.FormatOriginal, Quality: downloads.QualityOriginal, + EffectiveQuality: downloads.QualityOriginal, + }} + h := NewDownloadHandler(svc) + body := []byte(`{"content_id":"c1","quality":"original","caps":` + tc.caps + `}`) + rec := httptest.NewRecorder() + h.HandleCreateDownload(rec, downloadTestRequest(http.MethodPost, "/downloads", body, 7, "", "")) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202 (body: %s)", rec.Code, rec.Body.String()) + } + if len(svc.gotCreateReq.Caps.CodecsVideo) == 0 { + t.Fatalf("service received no flat codec list: %+v", svc.gotCreateReq.Caps) + } + }) + } +} + +func TestHandleCreateDownloadRejectsDetailedEntriesWithoutStrictEvidence(t *testing.T) { svc := &fakeDownloadService{} h := NewDownloadHandler(svc) body := []byte(`{ "content_id":"c1", "caps":{ - "client_features":["software_video_decode_v1"], - "video_evidence":"platform_attested", - "codecs_video":["av1"] + "video_evidence":"declared", + "codecs_video":["av1"], + "video_decode":[{"codec":"av1","max_width":1920,"hardware":true}] } }`) rec := httptest.NewRecorder() @@ -452,6 +519,9 @@ func TestHandleCreateDownloadRejectsSoftwareOptInWithoutDetailedEvidence(t *test if rec.Code != http.StatusBadRequest { t.Fatalf("status = %d, want 400 (body: %s)", rec.Code, rec.Body.String()) } + if svc.gotCreateReq.ContentID != "" { + t.Fatal("unvalidatable video_decode entries reached the download service") + } } func TestHandleCreateDownloadSeriesThreadsQuality(t *testing.T) { diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 8cb80bbf8..26c7358ba 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -396,8 +396,18 @@ const streamTokenParam = "st" // signSessionToken mints a stream token carrying the session's full // reconstruction recipe. Returns "" when no signing secret is configured -// (reconstruct effectively disabled, e.g. in tests). -func (h *PlaybackHandler) signSessionToken(card playback.RecipeCard) string { +// (reconstruct effectively disabled, e.g. in tests), or when the attempt +// negotiated header-authenticated media. +// +// requireMediaAuth is the attempt's negotiated media-auth mode, threaded from +// the session/recipe state the caller holds. It is refused here, at the mint, +// rather than only at the call sites that build URLs: a token that is never +// signed cannot leak into a client-visible URL by way of a builder that forgot +// to ask. Call sites keep their own checks as defense in depth. +func (h *PlaybackHandler) signSessionToken(card playback.RecipeCard, requireMediaAuth bool) string { + if requireMediaAuth { + return "" + } return h.signStreamClaims(card.ToClaims()) } @@ -441,6 +451,8 @@ func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID s requestUserID := apimw.GetUserID(r.Context()) session, err := h.sessionMgr.GetSession(sessionID) if err == nil { + // Defense in depth: LoadOrReconstructSession enforces the same rule for + // every serve handler, but this fast path never reaches it. if session.RequireMediaAuthorization && requestUserID == 0 { return nil, playback.SessionUnauthorized, nil } @@ -495,6 +507,12 @@ func appendStreamToken(rawURL, token string) string { // client re-supplies its byte position). Transcode sessions are told which URL // to play by their v3 plan; the URL here is an informational placeholder that // the plan's delivery URL supersedes. +// +// A session that requires media authorization gets the bare relative URL: it +// authenticates every media request with the caller's own access token, so no +// client-visible URL may carry a playback credential. Losing the token also +// means losing transparent reconstruction after a restart, which is the +// documented trade of that mode. func (h *PlaybackHandler) playbackStreamURL(s *playback.Session) string { if s == nil { return "" @@ -502,8 +520,12 @@ func (h *PlaybackHandler) playbackStreamURL(s *playback.Session) string { if s.PlayMethod == playback.PlayTranscode { return fmt.Sprintf("/playback/transcode/%s/master.m3u8", s.ID) } + streamURL := fmt.Sprintf("/stream/%s", s.ID) + if s.RequireMediaAuthorization { + return streamURL + } card := identityRecipeCard(s) - return appendStreamToken(fmt.Sprintf("/stream/%s", s.ID), h.signSessionToken(card)) + return appendStreamToken(streamURL, h.signSessionToken(card, s.RequireMediaAuthorization)) } // identityRecipeCard builds the identity-only recipe for a direct-play or remux @@ -1557,15 +1579,17 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht // reconstruction recipe and builds the manifest URL. proxyNode is the planner's // pick; when nil the URL falls back to the API-local path, where the token rides // the ?st= query parameter so the integrated server can reconstruct from it. -func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyNode *nodepool.Node) string { - token := h.signSessionToken(card) +// +// requireMediaAuth is the attempt's negotiated media-auth mode: such a session +// never receives a token, and therefore never a proxy origin either, since a +// proxy authenticates from the token in the URL path alone. It gets the +// API-local manifest path, which the client fetches with its own credential. +func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyNode *nodepool.Node, requireMediaAuth bool) string { + token := h.signSessionToken(card, requireMediaAuth) localURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", card.SessionID) - if proxyNode == nil { + if proxyNode == nil || token == "" { return appendStreamToken(localURL, token) } - if token == "" { - return localURL - } return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" } diff --git a/internal/api/handlers/playback_sessions.go b/internal/api/handlers/playback_sessions.go index 97f4cefc9..bddbd7429 100644 --- a/internal/api/handlers/playback_sessions.go +++ b/internal/api/handlers/playback_sessions.go @@ -21,63 +21,69 @@ import ( // method for the active stream; component-level behavior is exposed separately // via video_decision and audio_decision. type playbackSessionRow struct { - SessionID string `json:"session_id"` - UserID int `json:"user_id"` - Username string `json:"username"` - ProfileID string `json:"profile_id"` - ProfileName string `json:"profile_name,omitempty"` - MediaFileID int `json:"media_file_id"` - RequestedMediaFileID int `json:"requested_media_file_id"` - ContentID string `json:"content_id,omitempty"` - MediaTitle string `json:"media_title"` - MediaType string `json:"media_type"` - SeriesName string `json:"series_name,omitempty"` - EpisodeName string `json:"episode_name,omitempty"` - SeasonNumber *int `json:"season_number,omitempty"` - EpisodeNumber *int `json:"episode_number,omitempty"` - PosterURL string `json:"poster_url,omitempty"` - PlayMethod string `json:"play_method"` - ReportingNode string `json:"reporting_node"` - NodeDisplayName string `json:"node_display_name,omitempty"` - FileDuration *int `json:"file_duration"` - StartedAt time.Time `json:"started_at"` - UpdatedAt time.Time `json:"updated_at"` - PositionSeconds float64 `json:"position_seconds"` - IsPaused bool `json:"is_paused"` - HasPlaybackControl bool `json:"has_playback_control"` - ClientIP string `json:"client_ip,omitempty"` - ClientName string `json:"client_name,omitempty"` - ClientVersion string `json:"client_version,omitempty"` - ClientBuild string `json:"client_build,omitempty"` - ClientChannel string `json:"client_channel,omitempty"` - ClientLabel string `json:"client_label,omitempty"` - ClientLabelFull string `json:"client_label_full,omitempty"` - ClientUserAgent string `json:"client_user_agent,omitempty"` - AudioTrackIndex int `json:"audio_track_index"` - TranscodeAudio bool `json:"transcode_audio"` - StreamBitrateKbps *int `json:"stream_bitrate_kbps"` - TranscodeNodeURL string `json:"-"` - TargetResolution string `json:"target_resolution,omitempty"` - TargetVideoCodec string `json:"target_video_codec,omitempty"` - TargetAudioCodec string `json:"target_audio_codec,omitempty"` - TargetBitrateKbps *int `json:"target_bitrate_kbps"` - TranscodeHWAccel string `json:"transcode_hw_accel,omitempty"` - SourceContainer string `json:"source_container,omitempty"` - SourceBitrateKbps *int `json:"source_bitrate_kbps"` - SourceVideoCodec string `json:"source_video_codec,omitempty"` - SourceVideoResolution string `json:"source_video_resolution,omitempty"` - SourceAudioCodec string `json:"source_audio_codec,omitempty"` - SourceAudioChannels *int `json:"source_audio_channels"` - SourceAudioLanguage string `json:"source_audio_language,omitempty"` - SourceAudioTitle string `json:"source_audio_title,omitempty"` - SourceAudioLayout string `json:"source_audio_layout,omitempty"` - RequestedVideoCodec string `json:"requested_video_codec,omitempty"` - RequestedVideoResolution string `json:"requested_video_resolution,omitempty"` - VideoDecision string `json:"video_decision,omitempty"` - AudioDecision string `json:"audio_decision,omitempty"` - EffectivePlayMethod string `json:"effective_play_method,omitempty"` - IsJellyfinClient bool `json:"is_jellyfin_client,omitempty"` - CompatOrigin bool `json:"-"` + SessionID string `json:"session_id"` + UserID int `json:"user_id"` + Username string `json:"username"` + ProfileID string `json:"profile_id"` + ProfileName string `json:"profile_name,omitempty"` + MediaFileID int `json:"media_file_id"` + RequestedMediaFileID int `json:"requested_media_file_id"` + ContentID string `json:"content_id,omitempty"` + MediaTitle string `json:"media_title"` + MediaType string `json:"media_type"` + SeriesName string `json:"series_name,omitempty"` + EpisodeName string `json:"episode_name,omitempty"` + SeasonNumber *int `json:"season_number,omitempty"` + EpisodeNumber *int `json:"episode_number,omitempty"` + PosterURL string `json:"poster_url,omitempty"` + PlayMethod string `json:"play_method"` + ReportingNode string `json:"reporting_node"` + NodeDisplayName string `json:"node_display_name,omitempty"` + FileDuration *int `json:"file_duration"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` + PositionSeconds float64 `json:"position_seconds"` + IsPaused bool `json:"is_paused"` + HasPlaybackControl bool `json:"has_playback_control"` + ClientIP string `json:"client_ip,omitempty"` + ClientName string `json:"client_name,omitempty"` + ClientVersion string `json:"client_version,omitempty"` + ClientBuild string `json:"client_build,omitempty"` + ClientChannel string `json:"client_channel,omitempty"` + ClientLabel string `json:"client_label,omitempty"` + ClientLabelFull string `json:"client_label_full,omitempty"` + ClientUserAgent string `json:"client_user_agent,omitempty"` + AudioTrackIndex int `json:"audio_track_index"` + TranscodeAudio bool `json:"transcode_audio"` + StreamBitrateKbps *int `json:"stream_bitrate_kbps"` + TranscodeNodeURL string `json:"-"` + TargetResolution string `json:"target_resolution,omitempty"` + TargetVideoCodec string `json:"target_video_codec,omitempty"` + TargetAudioCodec string `json:"target_audio_codec,omitempty"` + // TargetAudioChannels is the channel count the transcode actually encodes. + // Absent when the reporting node did not know it — clients must then show + // the target codec with no channel layout rather than reusing + // SourceAudioChannels, which is what made a 7.1 source downmixed to AAC 5.1 + // read as "AAC 7.1". + TargetAudioChannels *int `json:"target_audio_channels,omitempty"` + TargetBitrateKbps *int `json:"target_bitrate_kbps"` + TranscodeHWAccel string `json:"transcode_hw_accel,omitempty"` + SourceContainer string `json:"source_container,omitempty"` + SourceBitrateKbps *int `json:"source_bitrate_kbps"` + SourceVideoCodec string `json:"source_video_codec,omitempty"` + SourceVideoResolution string `json:"source_video_resolution,omitempty"` + SourceAudioCodec string `json:"source_audio_codec,omitempty"` + SourceAudioChannels *int `json:"source_audio_channels"` + SourceAudioLanguage string `json:"source_audio_language,omitempty"` + SourceAudioTitle string `json:"source_audio_title,omitempty"` + SourceAudioLayout string `json:"source_audio_layout,omitempty"` + RequestedVideoCodec string `json:"requested_video_codec,omitempty"` + RequestedVideoResolution string `json:"requested_video_resolution,omitempty"` + VideoDecision string `json:"video_decision,omitempty"` + AudioDecision string `json:"audio_decision,omitempty"` + EffectivePlayMethod string `json:"effective_play_method,omitempty"` + IsJellyfinClient bool `json:"is_jellyfin_client,omitempty"` + CompatOrigin bool `json:"-"` } // playbackSessionsCapabilitiesResponse advertises the additive fields of the @@ -202,6 +208,7 @@ func (l *PlaybackSessionsLoader) Load( COALESCE(s.target_resolution, ''), COALESCE(s.target_video_codec, ''), COALESCE(s.target_audio_codec, ''), + s.target_audio_channels, s.target_bitrate_kbps, COALESCE(s.transcode_hw_accel, ''), COALESCE(mf.container, ''), @@ -241,6 +248,7 @@ func (l *PlaybackSessionsLoader) Load( var s playbackSessionRow var posterPath string var streamBitrateKbps *int + var targetAudioChannels *int var targetBitrateKbps *int var sourceBitrateKbps *int var sourceAudioChannels *int @@ -253,7 +261,8 @@ func (l *PlaybackSessionsLoader) Load( &s.PositionSeconds, &s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.ClientName, &s.ClientVersion, &s.ClientBuild, &s.ClientChannel, &s.ClientUserAgent, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps, - &s.TranscodeNodeURL, &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, &targetBitrateKbps, + &s.TranscodeNodeURL, &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, + &targetAudioChannels, &targetBitrateKbps, &s.TranscodeHWAccel, &s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution, &s.SourceAudioCodec, &sourceAudioChannels, &audioTracksJSON, &s.RequestedVideoCodec, &s.RequestedVideoResolution, &s.CompatOrigin, @@ -262,6 +271,7 @@ func (l *PlaybackSessionsLoader) Load( } s.PosterURL = l.presignPosterURL(r, posterPath) s.StreamBitrateKbps = streamBitrateKbps + s.TargetAudioChannels = targetAudioChannels s.TargetBitrateKbps = targetBitrateKbps s.SourceBitrateKbps = sourceBitrateKbps s.SourceAudioChannels = sourceAudioChannels diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index c15ce8c5e..8d28fce49 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -77,48 +77,15 @@ type preparedTimelineV3 struct { copySeekAnchorResolved bool } -type headerAuthenticatedMediaContextKeyV3 struct{} - -// withHeaderAuthenticatedMediaV3 records the request's negotiated media-auth -// mode without carrying a credential. Transport preparation happens several -// layers below the v3 request decoder; keeping the bounded boolean on the -// request preserves that negotiation across direct, remux, HLS and replan -// paths while leaving the durable request body as the source of truth. -func withHeaderAuthenticatedMediaV3(r *http.Request, clientFeatures []string) *http.Request { - if r == nil { - return nil - } - enabled := playback.HasFeatureV3(clientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) - return r.WithContext(context.WithValue(r.Context(), headerAuthenticatedMediaContextKeyV3{}, enabled)) -} - -func headerAuthenticatedMediaV3(r *http.Request) bool { - if r == nil { - return false - } - return headerAuthenticatedMediaContextV3(r.Context()) -} - -func headerAuthenticatedMediaContextV3(ctx context.Context) bool { - if ctx == nil { - return false - } - enabled, _ := ctx.Value(headerAuthenticatedMediaContextKeyV3{}).(bool) - return enabled -} - -func pinHeaderAuthenticatedMediaFeatureV3(clientFeatures []string, enabled bool) []string { - pinned := make([]string, 0, len(clientFeatures)+1) - for _, feature := range clientFeatures { - if strings.EqualFold(strings.TrimSpace(feature), playback.FeatureHeaderAuthenticatedMediaV3) { - continue - } - pinned = append(pinned, feature) - } - if enabled { - pinned = append(pinned, playback.FeatureHeaderAuthenticatedMediaV3) - } - return pinned +// headerAuthenticatedMediaV3 reports whether a client's advertised feature set +// opted into the tokenless, header-authenticated media transport. +// +// The negotiated mode is a bounded boolean threaded from the v3 request decoder +// down through transport preparation and into the session's stream state, the +// same way local egress is. It deliberately carries no credential, and the +// durable normalized request stays the source of truth for the attempt. +func headerAuthenticatedMediaV3(clientFeatures []string) bool { + return playback.HasFeatureV3(clientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) } type transportErrorV3 struct { @@ -341,7 +308,15 @@ type localEgressSessionPlannerV3 interface { // transcode node and no client-facing proxy is selected or returned. func (h *PlaybackHandler) planNodeSessionV3(ctx context.Context, session *playback.Session, result playback.PlannerResultV3, localEgress bool) nodepool.Plan { var eligible func(*nodepool.Node) bool - if enumerator, ok := h.NodePlanner.(transcodeNodeEnumeratorV3); ok && planRequiresServerTransformationsV3(result.Plan) { + // The per-node capability fan-out only pays for itself when a planner can + // actually consume the predicate it produces: PlanSessionWith for an ordinary + // selection, PlanTranscodeSessionWithLocalEgress for a local-egress one. A + // planner that implements neither would spend a round of capability lookups + // on a filter nothing reads. + _, capabilitySelectable := h.NodePlanner.(capabilitySessionPlannerV3) + _, localEgressSelectable := h.NodePlanner.(localEgressSessionPlannerV3) + predicateConsumed := capabilitySelectable || (localEgress && localEgressSelectable) + if enumerator, ok := h.NodePlanner.(transcodeNodeEnumeratorV3); ok && predicateConsumed && planRequiresServerTransformationsV3(result.Plan) { capable := make(map[string]struct{}) for nodeURL, advertised := range h.pooledNodeTransformationsV3(ctx, enumerator.TranscodeNodeURLs()) { if validateAdvertisedTransformationsV3(result.Plan, advertised) == nil { @@ -551,6 +526,22 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R writeJSON(w, http.StatusCreated, response) return } + // A refused progressive remux is escalated before the decision is logged or + // a session is opened, so the logged route is the one that will actually run. + escalated, escalateErr := h.escalateRefusedProgressiveRemuxV3(r.Context(), headerAuthenticatedMediaV3(req.ClientFeatures), + func() playback.PlannerInputV3 { + return h.plannerInputV3(r.Context(), req, requestedFile, effectiveFile, audioIndex, nil) + }, result) + if escalateErr != nil { + persistedResponse, persistErr := h.startFailureDecisionV3(r.Context(), userID, profileID, req, requestDigests, requestedFile.ID, effectiveFile.ID, escalateErr) + if persistErr != nil { + writeStartAttemptPersistenceErrorV3(w, persistErr) + return + } + writeJSON(w, http.StatusCreated, persistedResponse) + return + } + result = escalated // One line per plan decision so route selection is reconstructible from // server logs alone (finding a mis-planned route previously required // correlating client logcat, ffmpeg commands, and session rows). @@ -683,7 +674,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr if result.Plan == nil { return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "The server produced no playback plan."} } - r = withHeaderAuthenticatedMediaV3(r, req.ClientFeatures) + headerAuth := headerAuthenticatedMediaV3(req.ClientFeatures) if checker, ok := h.sessionMgr.(transcodePermissionChecker); ok && (result.PlayMethod == playback.PlayTranscode || result.TranscodeAudio) { if err := checker.CheckTranscodingAllowed(r.Context(), userID, result.PlayMethod == playback.PlayTranscode); err != nil { reason := "transcoding_disabled" @@ -729,7 +720,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr abort() return playback.DecisionResponseV3{}, subtitleArtifactErrorV3("Failed to freeze the selected subtitle identity.", frozenErr) } - transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result) + transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result, headerAuth) if transportErr != nil { abort() return playback.DecisionResponseV3{}, transportErr @@ -742,7 +733,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr } response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: playback.ServerFeaturesV3(), Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan} record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, FrozenRecipe: frozenRecipe, NormalizedRequest: req, StartResponse: response, RequestDigest: requestDigests.current, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)} - if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport); err != nil { + if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport, headerAuth); err != nil { transport.rollback() abort() return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to commit the live playback session.", cause: err} @@ -807,23 +798,26 @@ func (h *PlaybackHandler) persistSeriesSelectionsV3(ctx context.Context, userID h.persistAudioPreference(ctx, userID, profileID, file, audioTrackIndex) } -func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTransportV3, *transportErrorV3) { +// prepareTransportV3 resolves the plan into a live transport. headerAuth is the +// attempt's negotiated media-auth mode, resolved once by the caller and threaded +// down every branch (like localEgress) rather than re-derived per URL builder. +func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { timeline, timelineErr := h.prepareTransportTimelineV3(r.Context(), session, file, result) if timelineErr != nil { return preparedTransportV3{}, timelineErr } if result.Plan.Delivery != playback.DeliveryTranscodeHLSV3 && result.Plan.Delivery != playback.DeliveryRemuxHLSV3 { - return h.prepareIdentityTransportV3(r, session, file, result, timeline) + return h.prepareIdentityTransportV3(r, session, file, result, timeline, headerAuth) } if h.NodePlanner != nil { - plan := h.planNodeSessionV3(r.Context(), session, result, headerAuthenticatedMediaV3(r)) + plan := h.planNodeSessionV3(r.Context(), session, result, headerAuth) if plan.TranscodeNode != nil { transformations, err := h.remoteTransformationsV3(r.Context(), plan.TranscodeNode.URL) if err == nil { err = validateAdvertisedTransformationsV3(result.Plan, transformations) } if err == nil { - transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan, timeline) + transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan, timeline, headerAuth) if transportErr != nil { if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { releaser.ReleaseSession(session.ID) @@ -853,7 +847,7 @@ func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback. return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_capability_unavailable", message: "No available transcode executor can run the selected playback recipe.", retryable: true, cause: err} } } - return h.prepareLocalTransportV3(r, session, file, result, timeline) + return h.prepareLocalTransportV3(r, session, file, result, timeline, headerAuth) } func (h *PlaybackHandler) prepareTransportTimelineV3(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTimelineV3, *transportErrorV3) { @@ -918,8 +912,12 @@ func planRequiresServerTransformationsV3(plan *playback.PlanV3) bool { return false } -func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { routeSession := *session + // The URL builders below refuse to mint a stream token for a session that + // requires media authorization. The live session only learns the mode when + // its stream state is committed, so stamp the route copy the builders see. + routeSession.RequireMediaAuthorization = headerAuth routeSession.PlayMethod = result.PlayMethod routeSession.BasePlayMethod = result.PlayMethod routeSession.MediaFileID = result.Plan.EffectiveMediaFileID @@ -931,10 +929,14 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p routeSession.RemuxDVMode = remuxDVModeForPlanV3(result.Plan) var proxyNode *nodepool.Node - if headerAuthenticatedMediaV3(r) { + if headerAuth { // Proxy identity routes authenticate with a signed token in the URL path. // Keep this negotiated mode on the authenticated API origin instead, so // no client-visible URL can carry or disclose that credential. + // + // A remux that must run ffmpeg here has already been escalated onto an + // HLS delivery (or refused outright) before the session started; this + // call only refuses the residual cases. if localErr := h.refuseLocalIdentityWorkV3(r, result); localErr != nil { return preparedTransportV3{}, localErr } @@ -947,7 +949,7 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p } streamURL := fmt.Sprintf("/stream/%s", routeSession.ID) servedByProxy := false - if !headerAuthenticatedMediaV3(r) { + if !headerAuth { streamURL, servedByProxy = h.identityStreamURLV3(&routeSession, file, proxyNode) } releaseProxyReservation := func() { @@ -1098,6 +1100,81 @@ func (h *PlaybackHandler) refuseLocalIdentityWorkV3(r *http.Request, result play return &transportErrorV3{reason: "capacity_unavailable", message: "No proxy node is available and local fallback is disabled.", retryable: true} } +// plannerInputV3 assembles the planner input for one route decision. The +// escalation below re-plans with the same inputs the original decision used, +// plus the refused route's attempt key. +func (h *PlaybackHandler) plannerInputV3(ctx context.Context, req playback.StartRequestV3, requestedFile, effectiveFile *models.MediaFile, audioIndex int, attemptedKeys []string) playback.PlannerInputV3 { + return playback.PlannerInputV3{ + Request: req, + RequestedFile: requestedFile, + EffectiveFile: effectiveFile, + AudioTrackIndex: audioIndex, + Settings: h.plannerSettingsV3(ctx), + Registry: h.transformationRegistryV3(ctx), + HLSRegistry: h.lazyHLSPlanningRegistryV3(ctx), + DVRPUStrippable: h.lazyDVRPUStrippableV3(ctx, effectiveFile), + Now: time.Now(), + AttemptedKeys: attemptedKeys, + AdditionalSubtitles: h.downloadedSubtitleInventoryV3(ctx, effectiveFile), + } +} + +// escalateRefusedProgressiveRemuxV3 replaces a progressive remux that the +// header-authenticated transport is guaranteed to refuse. +// +// That mode bypasses the proxy identity routes (a proxy authenticates from the +// signed URL token this mode exists to remove), so a remux carrying server +// transformations is ffmpeg work with nowhere to run once +// playback.local_transcode_fallback is off — refuseLocalIdentityWorkV3 turns it +// into a retryable capacity_unavailable that nothing will ever satisfy. HLS is +// the same recipe on a delivery the API can relay from a pooled transcode node, +// so plan it here rather than making the client discover the refusal and +// recover through a replan round trip. +// +// A client that cannot execute an HLS delivery has no such alternative: it gets +// a non-retryable error naming the policy, because retrying is exactly what it +// must not do. +// +// plannerInput is evaluated only on the escalation path: rebuilding it costs a +// settings resolution and a downloaded-subtitle listing, which the overwhelming +// majority of starts must not pay for a route they never take. +func (h *PlaybackHandler) escalateRefusedProgressiveRemuxV3(ctx context.Context, headerAuth bool, plannerInput func() playback.PlannerInputV3, result playback.PlannerResultV3) (playback.PlannerResultV3, *transportErrorV3) { + if !headerAuth || result.Terminal != nil || result.Plan == nil || + result.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 || + !planRequiresServerTransformationsV3(result.Plan) || + nodepool.LocalTranscodeFallbackAllowed(ctx, h.SettingsRepo) { + return result, nil + } + input := plannerInput() + outputContextID := input.Request.ClientPlaybackContext.Output.OutputContextID + next := input + next.AttemptedKeys = append(append([]string(nil), input.AttemptedKeys...), + playback.PlanAttemptKeyV3(*result.Plan, outputContextID, nil)) + next.Now = time.Now() + escalated := playback.PlanPlaybackV3(next) + if escalated.Terminal != nil || escalated.Plan == nil || escalated.Plan.Delivery == playback.DeliveryRemuxProgressiveV3 { + reason := "" + if escalated.Terminal != nil { + reason = escalated.Terminal.Reason + } + slog.WarnContext(ctx, "protocol v3 header-authenticated remux has no executable delivery", + "component", "playback", + "delivery", result.Plan.Delivery, + "replanned_terminal_reason", reason, + ) + return result, &transportErrorV3{ + reason: "local_transcode_disabled", + message: "This server does not run playback conversions locally, and the client accepts no delivery that a transcode node can serve.", + } + } + slog.InfoContext(ctx, "protocol v3 escalated refused progressive remux", + "component", "playback", + "delivery", escalated.Plan.Delivery, + "decision_reason", escalated.Plan.DecisionReason, + ) + return escalated, nil +} + // identityStreamBitrateKbpsV3 estimates the bitrate a proxy will egress for an // identity delivery, so bandwidth-capped proxies admit it accurately. The plan's // effective recipe is authoritative (a remux that downmixes audio egresses less @@ -1127,8 +1204,13 @@ func identityStreamBitrateKbpsV3(result playback.PlannerResultV3) int { // // The bool reports whether the returned URL is actually a proxy URL, so the // caller can release the planner reservation when it is not. +// +// A session that requires media authorization never gets a proxy URL: the proxy +// serves from the signed token alone, which is exactly the credential that mode +// keeps out of client-visible URLs. It falls back to the API-local path, whose +// builder omits the token for the same reason. func (h *PlaybackHandler) identityStreamURLV3(s *playback.Session, file *models.MediaFile, proxyNode *nodepool.Node) (string, bool) { - if proxyNode == nil || file == nil { + if proxyNode == nil || file == nil || (s != nil && s.RequireMediaAuthorization) { return h.playbackStreamURL(s), false } card := identityRecipeCard(s) @@ -1288,7 +1370,7 @@ func appendPlaybackQueryV3(rawURL, key, value string) string { return rawURL + separator + key + "=" + value } -func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { cfg := h.playbackConfig() if err := os.MkdirAll(cfg.TranscodeDir, 0o755); err != nil { return preparedTransportV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to prepare the transcode directory.", cause: err} @@ -1343,9 +1425,9 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play } } url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) - if !headerAuthenticatedMediaV3(r) { + if !headerAuth { card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) - url = appendStreamToken(url, h.signSessionToken(card)) + url = appendStreamToken(url, h.signSessionToken(card, headerAuth)) } committed := false previousNodeURL := session.TranscodeNodeURL @@ -1392,7 +1474,7 @@ func manifestStartupTransportErrorV3(running bool, cause error) *transportErrorV return &transportErrorV3{reason: transcodeStartFailedReasonV3, message: message, retryable: running, cause: cause} } -func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { node := nodePlan.TranscodeNode transportID := transportGenerationV3(session.ID, result.Plan.PlanID) videoCodec := result.TargetVideoCodec @@ -1415,15 +1497,15 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "The selected transcode node rejected the playback transport.", retryable: true} } url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) - if !headerAuthenticatedMediaV3(r) { + if !headerAuth { hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) - url = h.buildProxyManifestURL(card, nodePlan.ProxyNode) + url = h.buildProxyManifestURL(card, nodePlan.ProxyNode, headerAuth) } // buildProxyManifestURL only returns an absolute proxy URL when a proxy was // planned and the token could be signed; otherwise the client fetches the // manifest from this server and the local liveness path applies. - servedByProxy := !headerAuthenticatedMediaV3(r) && nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") + servedByProxy := !headerAuth && nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") committed := false previousNodeURL := session.TranscodeNodeURL previousTransportID := remoteTransportID(session) @@ -1478,7 +1560,7 @@ func sourceVideoTranscodeFactsV3(file *models.MediaFile, result playback.Planner return profile, bitDepth } -func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3) playback.SessionStreamState { +func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, headerAuth bool) playback.SessionStreamState { state := playback.SessionStreamState{ PlayMethod: result.PlayMethod, BasePlayMethod: result.PlayMethod, @@ -1488,7 +1570,7 @@ func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *pla TranscodeNodeURL: transport.nodeURL, TranscodeTransportID: transport.transportID, TranscodeRouteSet: true, - RequireMediaAuthorization: headerAuthenticatedMediaContextV3(ctx), + RequireMediaAuthorization: headerAuth, MediaAuthorizationSet: true, ClientIP: clientip.FromContext(ctx), ClientName: session.ClientName, @@ -1515,8 +1597,8 @@ func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *pla return state } -func (h *PlaybackHandler) updateV3SessionState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3) error { - return h.sessionMgr.UpdateStreamState(session.ID, h.v3SessionStreamState(ctx, session, file, result, transport)) +func (h *PlaybackHandler) updateV3SessionState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, headerAuth bool) error { + return h.sessionMgr.UpdateStreamState(session.ID, h.v3SessionStreamState(ctx, session, file, result, transport, headerAuth)) } func plannedAudioTrackIndexV3(result playback.PlannerResultV3, fallback int) int { @@ -1580,7 +1662,20 @@ func (h *PlaybackHandler) attachSubtitleArtifactV3(ctx context.Context, sessionI inventory = playback.SubtitleInventoryV3(sessionID, file, downloadedSubtitleEntriesV3(file, downloaded)) } plan.Subtitle.Inventory = inventory + // Only render and convert publish a client-fetchable artifact; off and + // burn_in have none by definition. Clear rather than leave whatever the plan + // arrived with: a seek reanchor replays record.CurrentPlan verbatim + // (frozenSeekReanchorResultV3), so a plan that once rendered a sidecar would + // otherwise keep republishing that artifact after the selection changed — + // which is exactly the stale `mode: "off"` plus artifact pair observed in + // the field, and enough for a client to alias the artifact onto the track it + // is actually playing. An off decision carries no track either, so its + // track_id goes with the artifact; burn_in keeps the track it burns in. if selectedIndex < 0 || (plan.Subtitle.Mode != playback.SubtitleRenderV3 && plan.Subtitle.Mode != playback.SubtitleConvertV3) { + plan.Subtitle.Artifact = nil + if plan.Subtitle.Mode == playback.SubtitleOffV3 { + plan.Subtitle.TrackID = "" + } return nil } item, ok := playback.SubtitleInventoryItemAtV3(inventory, selectedIndex) @@ -1732,13 +1827,18 @@ func (h *PlaybackHandler) HandleReplanPlaybackV3(w http.ResponseWriter, r *http. if req.ClientFeatures == nil { req.ClientFeatures = append([]string(nil), record.NormalizedRequest.ClientFeatures...) } - // Media authentication is fixed at start. Neither an omitted/empty feature - // list nor a later opt-in can switch modes mid-attempt: a legacy URL from an - // earlier plan may remain usable until its signed recipe expires, so allowing - // legacy-to-header-auth upgrades would leave two different security contracts - // alive for the same session. Stop/start is the explicit mode boundary. - headerAuthenticatedAttempt := playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) - req.ClientFeatures = pinHeaderAuthenticatedMediaFeatureV3(req.ClientFeatures, headerAuthenticatedAttempt) + // The attempt-sticky features are fixed at start. Neither an omitted/empty + // feature list nor a later opt-in can add or drop one mid-attempt: media + // authentication would leave two security contracts alive for one session + // (a legacy URL from an earlier plan stays usable until its signed recipe + // expires), and a dropped software-decode opt-in would silently convert a + // direct route into a transcode and persist that downgrade into the durable + // normalized request. Stop/start is the explicit boundary for both. + // + // This is the single place attempt stickiness is enforced: everything + // downstream — the durable normalized request, the transport's media-auth + // mode, the planner's evidence tiers — reads the pinned list. + req.ClientFeatures = playback.PinAttemptStickyFeaturesV3(req.ClientFeatures, record.NormalizedRequest.ClientFeatures) if err := req.Validate(); err != nil { writeError(w, http.StatusBadRequest, "bad_request", "Invalid replan request") return @@ -2146,6 +2246,24 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte } } h.clarifyOriginalQuality4KTerminalV3(r.Context(), result.Terminal, requestedFile, replanAlternateFilePinnedByOriginalQualityV3(operation, start.QualityPreference)) + // Media authentication is attempt-sticky (pinned in HandleReplanPlaybackV3), + // so this mode always equals the one the attempt started under: a reused + // transport cannot change the session's media security contract. + headerAuth := headerAuthenticatedMediaV3(start.ClientFeatures) + if !seekReanchor { + // A freshly planned replan can land on the same refused progressive + // remux a start would have; escalate it identically. A seek reanchor + // replays the frozen recipe verbatim and must not change route identity, + // so it is excluded — its route was escalated when the attempt started. + escalated, escalateErr := h.escalateRefusedProgressiveRemuxV3(r.Context(), headerAuth, + func() playback.PlannerInputV3 { + return h.plannerInputV3(r.Context(), start, plannerRequestedFile, effectiveFile, audioIndex, attemptedKeys) + }, result) + if escalateErr != nil { + return playback.DecisionResponseV3{}, *record, nil, escalateErr + } + result = escalated + } if result.Terminal != nil { return playback.NewTerminalResponseV3(result.Terminal.Reason, result.Terminal.Message, result.Terminal.Retryable), *record, nil, nil } @@ -2179,10 +2297,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte } artifactRecipe = frozenRecipe } - currentHeaderAuthenticatedMedia := playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) - nextHeaderAuthenticatedMedia := playback.HasFeatureV3(start.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) - transportReused := currentHeaderAuthenticatedMedia == nextHeaderAuthenticatedMedia && trackChange && h.hasActiveHLSTransportV3(session) && sidecarOnlyHLSReplanV3(record, result.Plan, artifactRecipe, req.ClientPlaybackContext.Output.OutputContextID) - r = withHeaderAuthenticatedMediaV3(r, start.ClientFeatures) + transportReused := trackChange && h.hasActiveHLSTransportV3(session) && sidecarOnlyHLSReplanV3(record, result.Plan, artifactRecipe, req.ClientPlaybackContext.Output.OutputContextID) var transport preparedTransportV3 if transportReused { // A sidecar selection changes the plan and subtitle artifact, but it does @@ -2206,7 +2321,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte ) } else { var transportErr *transportErrorV3 - transport, transportErr = h.prepareTransportV3(r, session, effectiveFile, result) + transport, transportErr = h.prepareTransportV3(r, session, effectiveFile, result, headerAuth) if transportErr != nil { return playback.DecisionResponseV3{}, *record, nil, transportErr } @@ -2255,7 +2370,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte originalRollback := transport.rollback replacement := playback.SessionReplacement{ EffectiveMediaFileID: effectiveFile.ID, - StreamState: h.v3SessionStreamState(r.Context(), session, effectiveFile, result, transport), + StreamState: h.v3SessionStreamState(r.Context(), session, effectiveFile, result, transport, headerAuth), } if seekScopedRecovery { replacement.PositionSeconds = &req.PositionSeconds diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 6edde656f..178777647 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -2220,6 +2220,95 @@ func TestAttachSubtitleArtifactV3UsesFrozenDownloadedIdentityWithoutOrdinalLooku } } +// A plan struct can arrive at artifact attachment already carrying the previous +// plan's subtitle artifact — a seek reanchor replays record.CurrentPlan +// verbatim. Every mode without a client-fetchable artifact must therefore clear +// it rather than let the stale URL ride along, and an `off` decision must shed +// its track_id with it. The inventory URLs are selection-independent and stay. +func TestAttachSubtitleArtifactV3ClearsStaleArtifactWhenNoArtifactMode(t *testing.T) { + stale := func() *playback.SubtitleArtifactV3 { + return &playback.SubtitleArtifactV3{URL: "/stream/old-session/subtitles/1.vtt?file_id=42", MIMEType: "text/vtt", Format: "vtt", TimingOriginSeconds: 12} + } + tests := []struct { + name string + mode playback.SubtitleModeV3 + selectedIndex int + wantTrackID string + }{ + {name: "off", mode: playback.SubtitleOffV3, selectedIndex: -1}, + {name: "off with a carried-over ordinal", mode: playback.SubtitleOffV3, selectedIndex: 1}, + {name: "burn in keeps its track", mode: playback.SubtitleBurnInV3, selectedIndex: 1, wantTrackID: "file:42:subtitle:1"}, + {name: "render without a resolvable selection", mode: playback.SubtitleRenderV3, selectedIndex: -1, wantTrackID: "file:42:subtitle:1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.ExternalSubtitles = []models.ExternalSubtitle{{Path: "/subs/a.srt", Language: "eng", Format: "srt"}} + file.SubtitleTracks = []models.SubtitleTrack{{Index: 0, Codec: "subrip", Language: "fra"}} + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + plan := &playback.PlanV3{ + Subtitle: playback.SubtitleDecisionV3{ + Mode: test.mode, + TrackID: playback.TrackIDV3(file.ID, "subtitle", 1), + Artifact: stale(), + Inventory: playback.BuildSubtitleInventoryV3(file, nil), + }, + } + if err := handler.attachSubtitleArtifactV3(context.Background(), "session-current", file, plan, test.selectedIndex, nil); err != nil { + t.Fatalf("attach: %v", err) + } + if plan.Subtitle.Artifact != nil { + t.Fatalf("artifact = %#v, want nil for mode %q with selected index %d", plan.Subtitle.Artifact, test.mode, test.selectedIndex) + } + if plan.Subtitle.TrackID != test.wantTrackID { + t.Fatalf("track_id = %q, want %q", plan.Subtitle.TrackID, test.wantTrackID) + } + if len(plan.Subtitle.Inventory) != 2 { + t.Fatalf("inventory = %#v, want both tracks published", plan.Subtitle.Inventory) + } + for _, item := range plan.Subtitle.Inventory { + if !strings.HasPrefix(item.URL, "/stream/session-current/subtitles/") { + t.Fatalf("inventory entry %d url = %q, want a current-session sidecar url", item.CombinedIndex, item.URL) + } + } + }) + } +} + +// The regression this guards: a session that rendered a sidecar, then replanned +// with subtitles off, kept republishing the old artifact. +func TestAttachSubtitleArtifactV3DropsArtifactAcrossRenderToOffReplan(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.ExternalSubtitles = []models.ExternalSubtitle{{Path: "/subs/a.srt", Language: "eng", Format: "srt"}} + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + rendered := &playback.PlanV3{ + Subtitle: playback.SubtitleDecisionV3{ + Mode: playback.SubtitleRenderV3, + TrackID: playback.TrackIDV3(file.ID, "subtitle", 0), + Inventory: playback.BuildSubtitleInventoryV3(file, nil), + }, + } + if err := handler.attachSubtitleArtifactV3(context.Background(), "session-1", file, rendered, 0, nil); err != nil { + t.Fatalf("attach render: %v", err) + } + if rendered.Subtitle.Artifact == nil { + t.Fatal("render mode published no artifact") + } + // Replan reuses the accepted plan (the seek-reanchor shape) and turns + // subtitles off; the durable artifact must not survive the transition. + replanned := *rendered + replanned.Subtitle.Mode = playback.SubtitleOffV3 + if err := handler.attachSubtitleArtifactV3(context.Background(), "session-1", file, &replanned, -1, nil); err != nil { + t.Fatalf("attach off: %v", err) + } + if replanned.Subtitle.Artifact != nil || replanned.Subtitle.TrackID != "" { + t.Fatalf("off plan = %#v, want no artifact and no track_id", replanned.Subtitle) + } + if rendered.Subtitle.Artifact == nil { + t.Fatal("clearing the successor mutated the accepted plan's artifact") + } +} + func TestSubtitleArtifactStoreFailuresAreRetryable(t *testing.T) { storeErr := errors.New("database unavailable") wantRetryable := subtitleArtifactErrorV3("subtitle lookup failed", wrapSubtitleStoreErrorV3(storeErr)) @@ -2331,7 +2420,7 @@ func TestPrepareTransportV3ProgressiveRemuxUsesResolvedCopyAnchor(t *testing.T) EffectiveMediaFileID: 42, Timeline: playback.TimelineV3{SourceStartSeconds: requested, PlayerStartSeconds: requested, CanSeekAnywhere: true, SeekRestoration: "player_position"}, } - transport, transportErr := handler.prepareTransportV3(httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}) + transport, transportErr := handler.prepareTransportV3(httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, false) if transportErr != nil { t.Fatalf("prepare progressive transport: %v", transportErr) } @@ -2388,6 +2477,7 @@ func TestPrepareTransportV3AudioOnlyRemuxSkipsVideoCopyAnchor(t *testing.T) { &playback.Session{ID: "session-audio-only", MediaFileID: 42}, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, + false, ) if transportErr != nil { t.Fatalf("prepare audio-only transport: %v", transportErr) @@ -2422,6 +2512,7 @@ func TestPrepareTransportV3CopyAnchorFailureIsRetryable(t *testing.T) { &playback.Session{ID: "session-copy-failure"}, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, + false, ) if transportErr == nil || transportErr.reason != "transcode_start_failed" || !transportErr.retryable || transportErr.cause == nil || transportErr.cause.Error() != "probe failed" { t.Fatalf("transport error = %#v, want retryable copy anchor failure", transportErr) @@ -2455,7 +2546,7 @@ func TestPrepareTransportV3RejectsNodeMissingRequiredTransformation(t *testing.T }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" { t.Fatalf("transport error = %#v", transportErr) } @@ -2498,7 +2589,7 @@ func TestPrepareTransportV3RequiresRemoteManifestReadiness(t *testing.T) { }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-ready", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-ready", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) if transportErr != nil { t.Fatalf("prepare remote transport: %v", transportErr) } @@ -2549,8 +2640,8 @@ func TestPrepareTransportV3KeepsHeaderAuthenticatedRemoteHLSBehindAPI(t *testing {Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}, }, } - request := withHeaderAuthenticatedMediaV3(httptest.NewRequest(http.MethodPost, "/", nil), test.features) - transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-remote-auth", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + request := httptest.NewRequest(http.MethodPost, "/", nil) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-remote-auth", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, headerAuthenticatedMediaV3(test.features)) if transportErr != nil { t.Fatalf("prepare remote HLS: %v", transportErr) } @@ -2605,6 +2696,7 @@ func TestPrepareTransportV3SendsResolvedCopyAnchorToRemoteExecutor(t *testing.T) &playback.Session{ID: "session-remote-copy-anchor", UserID: 7, ProfileID: "profile-1"}, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv", CodecVideo: "h264"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, + false, ) if transportErr != nil { t.Fatalf("prepare remote copy transport: %v", transportErr) @@ -2662,7 +2754,7 @@ func TestPrepareTransportV3UsesFrozenSourceMetadataAfterProbeDrift(t *testing.T) file.Duration = 99 result := recipe.PlannerResult(plan) request := httptest.NewRequest(http.MethodPost, "/", nil) - transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-frozen-source", UserID: 7, ProfileID: "profile-1"}, file, result) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-frozen-source", UserID: 7, ProfileID: "profile-1"}, file, result, false) if transportErr != nil { t.Fatalf("prepare remote transport: %v", transportErr) } @@ -2718,7 +2810,7 @@ func TestPrepareLocalTransportV3ReturnsStableTerminalWhenFFmpegExitsBeforeReady( if timelineErr != nil { t.Fatalf("prepare timeline: %v", timelineErr) } - transport, transportErr := handler.prepareLocalTransportV3(request, &playback.Session{ID: "session-startup-failure", UserID: 7, ProfileID: "profile-1"}, file, result, timeline) + transport, transportErr := handler.prepareLocalTransportV3(request, &playback.Session{ID: "session-startup-failure", UserID: 7, ProfileID: "profile-1"}, file, result, timeline, false) if transportErr == nil { transport.rollback() t.Fatal("failed ffmpeg startup returned a playable transport") @@ -3887,6 +3979,7 @@ func TestPrepareTransportV3RoutesDirectPlayThroughProxyNode(t *testing.T) { &playback.Session{ID: "session-direct-proxy", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -3937,6 +4030,7 @@ func TestPrepareTransportV3RoutesProgressiveRemuxThroughProxyNodeWithSeekAndDV(t &playback.Session{ID: "session-remux-proxy", UserID: 7, ProfileID: "profile-1"}, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TranscodeAudio: true, TargetAudioCodec: "aac"}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -3979,13 +4073,14 @@ func TestPrepareTransportV3NegotiatesHeaderAuthenticatedProgressiveRemuxURL(t *t plan := identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3) plan.EffectiveMediaFileID = 42 plan.Timeline = playback.TimelineV3{SourceStartSeconds: 39.5} - request := withHeaderAuthenticatedMediaV3(httptest.NewRequest(http.MethodPost, "/", nil), test.features) + request := httptest.NewRequest(http.MethodPost, "/", nil) transport, transportErr := handler.prepareTransportV3( request, &playback.Session{ID: "session-remux-auth", UserID: 7, ProfileID: "profile-1", MediaFileID: 42}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, + headerAuthenticatedMediaV3(test.features), ) if transportErr != nil { t.Fatalf("prepare remux: %v", transportErr) @@ -4016,6 +4111,7 @@ func TestPrepareTransportV3FallsBackLocallyWithoutEligibleProxy(t *testing.T) { &playback.Session{ID: "session-direct-local", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -4043,6 +4139,7 @@ func TestPrepareTransportV3RefusesLocalRemuxWhenFallbackDisabled(t *testing.T) { PlayMethod: playback.PlayRemux, TranscodeAudio: true, }, + false, ) if transportErr == nil || transportErr.reason != "capacity_unavailable" { t.Fatalf("transport error = %#v, want capacity_unavailable when local remux work is disabled", transportErr) @@ -4062,6 +4159,7 @@ func TestPrepareTransportV3AllowsLocalDirectPlayWhenFallbackDisabled(t *testing. &playback.Session{ID: "session-direct-allowed", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("direct play refused with local fallback disabled: %#v", transportErr) @@ -4080,6 +4178,7 @@ func TestPrepareTransportV3ReleasesProxyReservationOnRollback(t *testing.T) { &playback.Session{ID: "session-rollback", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -4134,6 +4233,7 @@ func TestPrepareTransportV3KeepsRemuxLocalWhenProxyLacksTheRecipe(t *testing.T) PlayMethod: playback.PlayRemux, TranscodeAudio: true, }, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -4169,6 +4269,7 @@ func TestPrepareTransportV3DirectPlaySkipsProxyCapabilityProbe(t *testing.T) { &playback.Session{ID: "session-direct-noprobe", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -4216,6 +4317,7 @@ func TestPrepareTransportV3MarksProxySessionsAsRemotelyTransported(t *testing.T) session, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) @@ -4300,6 +4402,7 @@ func TestPrepareTransportV3PrefersACapableSiblingProxy(t *testing.T) { PlayMethod: playback.PlayRemux, TranscodeAudio: true, }, + false, ) if transportErr != nil { t.Fatalf("prepare identity transport: %#v", transportErr) @@ -4329,6 +4432,7 @@ func TestPrepareTransportV3ClearsRemoteTransportMarkWhenServingLocally(t *testin proxied, transportErr := handler.prepareTransportV3( httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare proxy transport: %v", transportErr) @@ -4340,6 +4444,7 @@ func TestPrepareTransportV3ClearsRemoteTransportMarkWhenServingLocally(t *testin local, transportErr := handler.prepareTransportV3( httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + false, ) if transportErr != nil { t.Fatalf("prepare local transport: %v", transportErr) diff --git a/internal/api/handlers/playback_v3_tokenless_test.go b/internal/api/handlers/playback_v3_tokenless_test.go new file mode 100644 index 000000000..9369b0c3e --- /dev/null +++ b/internal/api/handlers/playback_v3_tokenless_test.go @@ -0,0 +1,284 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// The negotiated mode's whole promise is that nothing the client can see +// carries a playback credential, so this asserts on the entire response body +// rather than on the individual URLs a reader remembered to check. +func TestHandleStartPlaybackV3HeaderAuthenticatedResponseCarriesNoStreamToken(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.ExternalSubtitles = []models.ExternalSubtitle{{Path: writePlaybackTestMediaFile(t, "movie.eng.srt"), Language: "eng", Format: "srt"}} + file.SubtitleTracks = []models.SubtitleTrack{{Index: 0, Codec: "subrip", Language: "fra"}} + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-stream-signing-secret" + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + start := v3HandlerStartRequest() + start.ClientFeatures = append(start.ClientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + subtitleIndex := 0 + start.SubtitleTrackID = playback.TrackIDV3(file.ID, "subtitle", subtitleIndex) + start.SubtitleTrackIndex = &subtitleIndex + + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, start))).WithContext(newAuthorizedPlaybackContext())) + if rr.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", rr.Code, rr.Body.String()) + } + body := rr.Body.String() + if strings.Contains(body, streamTokenParam+"=") { + t.Fatalf("header-authenticated response carries a stream token: %s", body) + } + + var response playback.DecisionResponseV3 + if err := json.Unmarshal([]byte(body), &response); err != nil || response.PlaybackPlan == nil { + t.Fatalf("response: err=%v body=%s", err, body) + } + urls := []string{response.PlaybackPlan.Stream.URL} + if artifact := response.PlaybackPlan.Subtitle.Artifact; artifact != nil { + urls = append(urls, artifact.URL) + } + for _, entry := range response.PlaybackPlan.Subtitle.Inventory { + if entry.URL != "" { + urls = append(urls, entry.URL) + } + } + if len(urls) < 3 { + t.Fatalf("response published %d URLs, want a stream plus subtitle artifact and inventory routes", len(urls)) + } + for _, raw := range urls { + parsed, err := url.Parse(raw) + if err != nil || parsed.IsAbs() || parsed.Query().Get(streamTokenParam) != "" { + t.Fatalf("URL %q is not a tokenless API-local route (parse error %v)", raw, err) + } + } + if len(response.PlaybackPlan.Stream.Headers) != 0 { + t.Fatalf("plan persisted credential material in stream headers: %#v", response.PlaybackPlan.Stream.Headers) + } +} + +// The call-site checks are defense in depth; the builders themselves must +// refuse, so a future caller cannot mint a credential by forgetting to ask. +func TestPlaybackURLBuildersRefuseTokensForMediaAuthorizedSessions(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-stream-signing-secret" + file := v3HandlerFixtureFile(t) + proxy := &nodepool.Node{URL: "http://proxy.example"} + + secure := &playback.Session{ID: "session-secure", UserID: 7, ProfileID: "profile-1", MediaFileID: file.ID, PlayMethod: playback.PlayDirect, RequireMediaAuthorization: true} + legacy := *secure + legacy.ID = "session-legacy" + legacy.RequireMediaAuthorization = false + + if got := handler.playbackStreamURL(secure); got != "/stream/session-secure" { + t.Fatalf("secure stream URL = %q, want the bare API-local route", got) + } + if got := handler.playbackStreamURL(&legacy); !strings.Contains(got, streamTokenParam+"=") { + t.Fatalf("legacy stream URL = %q, want a restart token", got) + } + + if got, servedByProxy := handler.identityStreamURLV3(secure, file, proxy); servedByProxy || got != "/stream/session-secure" { + t.Fatalf("secure identity URL = %q (proxy %v), want the API-local route", got, servedByProxy) + } + if got, servedByProxy := handler.identityStreamURLV3(&legacy, file, proxy); !servedByProxy || !strings.HasPrefix(got, proxy.URL) { + t.Fatalf("legacy identity URL = %q (proxy %v), want the signed proxy route", got, servedByProxy) + } + + card := playback.NewRecipeCard(secure.UserID, secure.ProfileID, file.ID, "", playback.TranscodeOpts{SessionID: secure.ID, InputPath: file.FilePath}) + if got := handler.buildProxyManifestURL(card, proxy, true); got != "/playback/transcode/session-secure/master.m3u8" { + t.Fatalf("secure manifest URL = %q, want the tokenless API-local manifest", got) + } + if got := handler.buildProxyManifestURL(card, proxy, false); !strings.HasPrefix(got, proxy.URL+"/stream/transcode/") { + t.Fatalf("legacy manifest URL = %q, want the signed proxy manifest", got) + } + if token := handler.signSessionToken(card, true); token != "" { + t.Fatalf("signer minted a token for a media-authorized session: %q", token) + } + if token := handler.signSessionToken(card, false); token == "" { + t.Fatal("signer refused a legacy session with a configured secret") + } +} + +// escalationFixtureV3 plans a progressive remux that must convert audio, which +// is exactly the route the header-authenticated transport cannot execute once +// local fallback is disabled. +func escalationFixtureV3(t *testing.T, hlsCapable bool) (*PlaybackHandler, playback.PlannerInputV3, playback.PlannerResultV3) { + t.Helper() + file := v3HandlerFixtureFile(t) + file.CodecAudio = "eac3" + file.AudioTracks = []models.AudioTrack{{Codec: "eac3", Channels: 6, Layout: "5.1", Default: true}} + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{ + "allow_4k_transcode": "true", + "playback.local_transcode_fallback": "false", + }} + registry := playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{ + {Name: playback.TransformationAudioToAACV3, RecipeVersion: "1", Available: true}, + {Name: playback.TransformationVideoToH264V3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3, Available: true}, + }) + presetLocalRegistryV3(handler, registry) + + request := v3HandlerStartRequest() + request.QualityPreference = "auto" + request.ClientPlaybackContext.Deliveries[playback.DeliveryClassProgressiveV3] = playback.DeliveryCapabilityV3{Enabled: true, SupportedOnDevice: true} + if hlsCapable { + request.ClientPlaybackContext.Deliveries[playback.DeliveryClassHLSV3] = playback.DeliveryCapabilityV3{Enabled: true, SupportedOnDevice: true} + } + input := playback.PlannerInputV3{ + Request: request, + RequestedFile: file, + EffectiveFile: file, + Settings: playback.PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: registry, + HLSRegistry: func() *playback.TransformationRegistryV3 { return registry }, + } + result := playback.PlanPlaybackV3(input) + if result.Terminal != nil || result.Plan == nil || result.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + t.Fatalf("fixture planned %#v, want a progressive remux", result) + } + if !planRequiresServerTransformationsV3(result.Plan) { + t.Fatalf("fixture plan carries no server transformation: %#v", result.Plan.Transformations) + } + return handler, input, result +} + +// A progressive remux the header-authenticated transport would refuse is +// escalated onto HLS up front, rather than handed to the client as a retryable +// capacity error it can only recover from with a replan round trip. +func TestEscalateRefusedProgressiveRemuxV3PlansHLSForCapableClients(t *testing.T) { + handler, input, result := escalationFixtureV3(t, true) + escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, func() playback.PlannerInputV3 { return input }, result) + if transportErr != nil { + t.Fatalf("escalation error = %#v", transportErr) + } + if escalated.Plan == nil || escalated.Plan.Delivery != playback.DeliveryRemuxHLSV3 { + t.Fatalf("escalated delivery = %#v, want %q", escalated.Plan, playback.DeliveryRemuxHLSV3) + } +} + +// A progressive-only client has no alternative delivery, so the refusal is +// final: a retryable error would make it retry a route no retry can satisfy. +func TestEscalateRefusedProgressiveRemuxV3IsTerminalForProgressiveOnlyClients(t *testing.T) { + handler, input, result := escalationFixtureV3(t, false) + _, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, func() playback.PlannerInputV3 { return input }, result) + if transportErr == nil || transportErr.reason != "local_transcode_disabled" || transportErr.retryable { + t.Fatalf("transport error = %#v, want a non-retryable local_transcode_disabled", transportErr) + } +} + +func TestEscalateRefusedProgressiveRemuxV3LeavesExecutableRoutesAlone(t *testing.T) { + handler, input, result := escalationFixtureV3(t, true) + planned := 0 + plannerInput := func() playback.PlannerInputV3 { + planned++ + return input + } + if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), false, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + t.Fatalf("legacy attempt was escalated: %#v %#v", escalated.Plan, transportErr) + } + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.local_transcode_fallback": "true"}} + if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + t.Fatalf("locally executable remux was escalated: %#v %#v", escalated.Plan, transportErr) + } + if planned != 0 { + t.Fatalf("planner input was rebuilt %d times on the non-escalating path", planned) + } +} + +// The capability fan-out costs a per-node HTTP round trip, so it must not run +// for a planner that cannot consume the eligibility predicate it produces. +func TestPlanNodeSessionV3SkipsCapabilityFanOutWithoutConsumer(t *testing.T) { + fetches := 0 + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetches++ + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}, + }}) + })) + defer node.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + // Enumerates pooled nodes, but implements neither PlanSessionWith nor the + // local-egress selector. + handler.NodePlanner = enumeratingNodePlannerV3{ + staticNodePlannerV3: staticNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}}}, + urls: []string{node.URL}, + } + plan := &playback.PlanV3{ + PlanID: "plan:no-consumer", + Delivery: playback.DeliveryRemuxHLSV3, + Transformations: []playback.TransformationV3{{Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}}, + } + for _, localEgress := range []bool{false, true} { + selected := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-no-consumer"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, localEgress) + if selected.TranscodeNode == nil || selected.TranscodeNode.URL != node.URL { + t.Fatalf("planner selection = %+v, want the static node", selected.TranscodeNode) + } + } + if fetches != 0 { + t.Fatalf("capability fan-out ran %d times for a planner that discards the predicate", fetches) + } +} + +// software_video_decode_v1 is attempt-sticky like header-authenticated media: a +// replan that sends an explicit feature list cannot drop it and silently +// convert a direct route into a transcode. +func TestHandleReplanPlaybackV3PinsAttemptStickyFeatures(t *testing.T) { + file := v3HandlerFixtureFile(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0), testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-stream-signing-secret" + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + start := v3HandlerStartRequest() + start.ClientFeatures = append(start.ClientFeatures, playback.FeatureSoftwareVideoDecodeV3) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, start))).WithContext(newAuthorizedPlaybackContext())) + var started playback.DecisionResponseV3 + if startRR.Code != http.StatusCreated || json.Unmarshal(startRR.Body.Bytes(), &started) != nil || started.PlaybackPlan == nil { + t.Fatalf("start status=%d body=%s", startRR.Code, startRR.Body.String()) + } + + nextContext := start.ClientPlaybackContext + nextContext.Output.OutputContextID = "route-2" + replanned := postPlaybackReplanV3(t, handler, started.SessionID, playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + ClientFeatures: []string{playback.FeaturePlaybackPlanV3}, // explicit list dropping the opt-in + Operation: playback.ReplanOperationOutputChangeV3, + PlaybackAttemptID: start.PlaybackAttemptID, + ReplanRequestID: "sticky-feature-replan-0001", + FailedPlanID: started.PlaybackPlan.PlanID, + PlanAttemptID: "sticky-feature-attempt-0001", + PlanAttemptKey: started.PlaybackPlan.PlanAttemptKey, + AttemptCount: 1, + PositionSeconds: 12, + SelectedTracks: started.PlaybackPlan.SelectedTracks, + Capabilities: start.Capabilities, + ClientPlaybackContext: nextContext, + }) + if replanned.PlaybackPlan == nil { + t.Fatalf("replan = %#v", replanned) + } + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if !playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureSoftwareVideoDecodeV3) { + t.Fatalf("durable client features = %v, the software-decode opt-in was dropped", record.NormalizedRequest.ClientFeatures) + } +} diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go index d8d7d16f4..d0c5c0f5d 100644 --- a/internal/api/handlers/playback_v3_union_test.go +++ b/internal/api/handlers/playback_v3_union_test.go @@ -177,7 +177,7 @@ func TestPrepareTransportV3LocalFallbackRejectsUnavailableTransformations(t *tes }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" || !transportErr.retryable { t.Fatalf("transport error = %#v", transportErr) } diff --git a/internal/auth/repository.go b/internal/auth/repository.go index a641a4a24..12aa91ac4 100644 --- a/internal/auth/repository.go +++ b/internal/auth/repository.go @@ -249,45 +249,66 @@ func (r *UserRepository) GetByEmail(ctx context.Context, email string) (*models. // access_policy_revision. Values are pre-computed, so every entry is safe to // build even when set is false. type userUpdateColumn struct { - column string - set bool - value any - // expr, when non-empty, is a SQL expression written in place of a bare - // bound value; it may reference the row's current columns. A "$?" inside - // it is replaced with the placeholder for value; without one, value is - // not bound. - expr string + column string + set bool + value any bumpsAccessPolicy bool } -// accessGroupUpdateColumn decides what the write does to access_group_id. -// Admin accounts are never grouped (see Create): granting the role clears the -// group no matter what the caller passed, and taking the role away without -// naming a group lands the account on the default group, as create does. A -// group written on its own is checked against the row's role inside the same -// statement, so a concurrent promotion cannot leave an admin grouped. -func accessGroupUpdateColumn(input models.UpdateUserInput) userUpdateColumn { +// accessGroupSetClause builds the SET clause and access-policy predicate for +// access_group_id given the next free placeholder index. access_group_id is +// handled outside the generic userUpdateColumn machinery because, unlike +// every other column, what gets written depends on the row's current role: +// +// - Granting admin (input.Role == "admin") clears the group unconditionally. +// - Changing role to anything else without naming a group lands the row on +// the default group, but only if it was an admin (accounts are never +// un-grouped by an unrelated role change). +// - Setting a group on its own (input.Role == nil) is guarded by a CASE so +// a write that races an admin promotion cannot leave the admin grouped. +// - Otherwise (explicit NULL, or a group set alongside a non-admin role +// change) the value is bound directly. +// +// Admin accounts are never grouped (see Create). Returns an empty setClause +// if access_group_id is not touched by this update. +// +// The default-group branch reads from a CTE (aliased in defaultGroupCTE) +// instead of inlining the subselect, because the same expression is spliced +// into both the SET clause and the access_policy_revision predicate — as a +// literal subselect it would run twice per UPDATE, but a CTE referenced more +// than once is materialized once by Postgres. +func accessGroupSetClause(input models.UpdateUserInput, argIndex int) (setClause, predicate, defaultGroupCTE string, args []any, nextArgIndex int) { const isAdmin = "role = '" + models.RoleAdmin + "'" - col := userUpdateColumn{column: "access_group_id", bumpsAccessPolicy: true} + nextArgIndex = argIndex switch { case input.Role != nil && *input.Role == models.RoleAdmin: - col.set = true - col.value = (*int64)(nil) + placeholder := fmt.Sprintf("$%d", argIndex) + setClause = "access_group_id = " + placeholder + args = []any{(*int64)(nil)} + nextArgIndex++ case input.Role != nil && !input.AccessGroupID.Set: - col.set = true - col.expr = "CASE WHEN " + isAdmin + - " THEN (SELECT id FROM access_groups WHERE is_default) ELSE access_group_id END" + defaultGroupCTE = "default_group AS (SELECT id FROM access_groups WHERE is_default)" + expr := "(CASE WHEN " + isAdmin + " THEN (SELECT id FROM default_group) ELSE access_group_id END)" + setClause = "access_group_id = " + expr case input.Role == nil && input.AccessGroupID.Set && input.AccessGroupID.Value != nil: - col.set = true - col.value = input.AccessGroupID.Value + placeholder := fmt.Sprintf("$%d", argIndex) // The cast pins the parameter type; inside a CASE the driver would // otherwise send it as text. - col.expr = "CASE WHEN " + isAdmin + " THEN NULL ELSE $?::bigint END" + expr := "(CASE WHEN " + isAdmin + " THEN NULL ELSE " + placeholder + "::bigint END)" + setClause = "access_group_id = " + expr + args = []any{input.AccessGroupID.Value} + nextArgIndex++ default: - col.set = input.AccessGroupID.Set - col.value = input.AccessGroupID.Value + if !input.AccessGroupID.Set { + return "", "", "", nil, argIndex + } + placeholder := fmt.Sprintf("$%d", argIndex) + setClause = "access_group_id = " + placeholder + args = []any{input.AccessGroupID.Value} + nextArgIndex++ } - return col + predicate = "access_group_id IS DISTINCT FROM " + strings.TrimPrefix(setClause, "access_group_id = ") + return setClause, predicate, defaultGroupCTE, args, nextArgIndex } // Update modifies a user's fields. Only non-nil fields in the input are updated. @@ -347,7 +368,6 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update {column: "download_allowed", set: input.DownloadAllowed.Set, value: input.DownloadAllowed.Value}, {column: "download_transcode_allowed", set: input.DownloadTranscodeAllowed.Set, value: input.DownloadTranscodeAllowed.Value}, {column: "requests_allowed", set: input.RequestsAllowed.Set, value: input.RequestsAllowed.Value}, - accessGroupUpdateColumn(input), } setClauses := []string{} @@ -359,26 +379,29 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update continue } placeholder := fmt.Sprintf("$%d", argIndex) - rhs := placeholder - binds := true - if col.expr != "" { - binds = strings.Contains(col.expr, "$?") - rhs = "(" + strings.ReplaceAll(col.expr, "$?", placeholder) + ")" - } - setClauses = append(setClauses, fmt.Sprintf("%s = %s", col.column, rhs)) + setClauses = append(setClauses, fmt.Sprintf("%s = %s", col.column, placeholder)) if col.bumpsAccessPolicy { accessPolicyPredicates = append( accessPolicyPredicates, - fmt.Sprintf("%s IS DISTINCT FROM %s", col.column, rhs), + fmt.Sprintf("%s IS DISTINCT FROM %s", col.column, placeholder), ) } - if !binds { - continue - } args = append(args, col.value) argIndex++ } + // access_group_id is not a plain userUpdateColumn: what gets written + // depends on the row's current role, so it is assembled directly rather + // than through the generic column loop above. + var defaultGroupCTE string + if setClause, predicate, cte, groupArgs, nextArgIndex := accessGroupSetClause(input, argIndex); setClause != "" { + setClauses = append(setClauses, setClause) + accessPolicyPredicates = append(accessPolicyPredicates, predicate) + defaultGroupCTE = cte + args = append(args, groupArgs...) + argIndex = nextArgIndex + } + if len(setClauses) == 0 { // Nothing to update; still verify the user exists. _, err := r.GetByID(ctx, id) @@ -395,8 +418,14 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update // Always bump updated_at. setClauses = append(setClauses, "updated_at = NOW()") - query := fmt.Sprintf("UPDATE users SET %s WHERE id = $%d", - strings.Join(setClauses, ", "), argIndex) + var query string + if defaultGroupCTE != "" { + query = fmt.Sprintf("WITH %s UPDATE users SET %s WHERE id = $%d", + defaultGroupCTE, strings.Join(setClauses, ", "), argIndex) + } else { + query = fmt.Sprintf("UPDATE users SET %s WHERE id = $%d", + strings.Join(setClauses, ", "), argIndex) + } args = append(args, id) tag, err := r.pool.Exec(ctx, query, args...) diff --git a/internal/auth/repository_access_group_clause_test.go b/internal/auth/repository_access_group_clause_test.go new file mode 100644 index 000000000..330b49437 --- /dev/null +++ b/internal/auth/repository_access_group_clause_test.go @@ -0,0 +1,121 @@ +package auth + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// TestAccessGroupSetClause pins the exact SQL/argument shape produced for +// access_group_id, independent of a live database. It's a regression guard +// for the expr/$? removal: the CASE guards, the NULL-on-promotion write, and +// the default-group CTE all have to keep generating the same effective SQL +// the old generic-expr mechanism did. +func TestAccessGroupSetClause(t *testing.T) { + admin := models.RoleAdmin + user := "user" + + t.Run("promoting to admin clears the group directly", func(t *testing.T) { + setClause, predicate, cte, args, nextArgIndex := accessGroupSetClause( + models.UpdateUserInput{Role: &admin}, 3, + ) + if setClause != "access_group_id = $3" { + t.Fatalf("setClause = %q", setClause) + } + if predicate != "access_group_id IS DISTINCT FROM $3" { + t.Fatalf("predicate = %q", predicate) + } + if cte != "" { + t.Fatalf("cte = %q, want none", cte) + } + if len(args) != 1 || args[0] != (*int64)(nil) { + t.Fatalf("args = %#v, want [nil]", args) + } + if nextArgIndex != 4 { + t.Fatalf("nextArgIndex = %d, want 4", nextArgIndex) + } + }) + + t.Run("demoting an admin without a group falls back to the default via a CTE, binding no placeholder", func(t *testing.T) { + setClause, predicate, cte, args, nextArgIndex := accessGroupSetClause( + models.UpdateUserInput{Role: &user}, 3, + ) + wantExpr := "(CASE WHEN role = '" + models.RoleAdmin + + "' THEN (SELECT id FROM default_group) ELSE access_group_id END)" + if setClause != "access_group_id = "+wantExpr { + t.Fatalf("setClause = %q", setClause) + } + if predicate != "access_group_id IS DISTINCT FROM "+wantExpr { + t.Fatalf("predicate = %q", predicate) + } + if cte != "default_group AS (SELECT id FROM access_groups WHERE is_default)" { + t.Fatalf("cte = %q", cte) + } + // The same alias appears in both setClause and predicate above, so + // the default-group subselect is only ever written once in the CTE + // text itself: Postgres materializes a multiply-referenced CTE once + // instead of re-running it per appearance. + if len(args) != 0 { + t.Fatalf("args = %#v, want none (no placeholder consumed)", args) + } + if nextArgIndex != 3 { + t.Fatalf("nextArgIndex = %d, want unchanged 3", nextArgIndex) + } + }) + + t.Run("setting a group alone is guarded against a concurrent admin promotion", func(t *testing.T) { + groupID := int64(42) + setClause, predicate, cte, args, nextArgIndex := accessGroupSetClause( + models.UpdateUserInput{AccessGroupID: models.SetValue(groupID)}, 5, + ) + wantExpr := "(CASE WHEN role = '" + models.RoleAdmin + "' THEN NULL ELSE $5::bigint END)" + if setClause != "access_group_id = "+wantExpr { + t.Fatalf("setClause = %q", setClause) + } + if predicate != "access_group_id IS DISTINCT FROM "+wantExpr { + t.Fatalf("predicate = %q", predicate) + } + if cte != "" { + t.Fatalf("cte = %q, want none", cte) + } + if len(args) != 1 || *(args[0].(*int64)) != groupID { + t.Fatalf("args = %#v, want [%d]", args, groupID) + } + if nextArgIndex != 6 { + t.Fatalf("nextArgIndex = %d, want 6", nextArgIndex) + } + }) + + t.Run("explicit null binds directly with no CASE", func(t *testing.T) { + setClause, predicate, cte, args, nextArgIndex := accessGroupSetClause( + models.UpdateUserInput{AccessGroupID: models.ClearValue[int64]()}, 2, + ) + if setClause != "access_group_id = $2" { + t.Fatalf("setClause = %q", setClause) + } + if predicate != "access_group_id IS DISTINCT FROM $2" { + t.Fatalf("predicate = %q", predicate) + } + if cte != "" { + t.Fatalf("cte = %q, want none", cte) + } + if len(args) != 1 || args[0] != (*int64)(nil) { + t.Fatalf("args = %#v, want [nil]", args) + } + if nextArgIndex != 3 { + t.Fatalf("nextArgIndex = %d, want 3", nextArgIndex) + } + }) + + t.Run("untouched leaves the column alone", func(t *testing.T) { + setClause, predicate, cte, args, nextArgIndex := accessGroupSetClause( + models.UpdateUserInput{}, 7, + ) + if setClause != "" || predicate != "" || cte != "" || args != nil { + t.Fatalf("got (%q, %q, %q, %#v), want all empty", setClause, predicate, cte, args) + } + if nextArgIndex != 7 { + t.Fatalf("nextArgIndex = %d, want unchanged 7", nextArgIndex) + } + }) +} diff --git a/internal/downloads/policy_test.go b/internal/downloads/policy_test.go index 96da5c40e..966113485 100644 --- a/internal/downloads/policy_test.go +++ b/internal/downloads/policy_test.go @@ -33,11 +33,44 @@ func TestDownloadQualityResolverResolve(t *testing.T) { Container: "mp4", Resolution: "1080p", } + // Sparse probe metadata: no video track, so bit depth, dimensions, frame + // rate and bitrate are unknown and the detailed decoder bounds cannot be + // evaluated against the source. + sparseFile := &models.MediaFile{ + ID: 4, + CodecVideo: "hevc", + CodecAudio: "aac", + Container: "mp4", + Resolution: "1080p", + } + boundedFile := &models.MediaFile{ + ID: 5, + CodecVideo: "hevc", + CodecAudio: "aac", + Container: "mp4", + Resolution: "2160p", + Bitrate: 55_000, + VideoTracks: []models.VideoTrack{{ + Codec: "hevc", Profile: "Main 10", Width: 3840, Height: 2160, + FrameRate: "60/1", Bitrate: 55_000, BitDepth: 10, + }}, + } caps := playback.ClientCapabilities{ CodecsVideo: []string{"h264"}, CodecsAudio: []string{"aac"}, Containers: []string{"mp4"}, } + detailedCaps := playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"hevc"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "hevc", BitDepths: []int{8, 10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: true, + }}, + } cases := []struct { name string @@ -97,6 +130,34 @@ func TestDownloadQualityResolverResolve(t *testing.T) { wantEffective: Quality20Mbps, wantBitrate: 20000, }, + { + // "Can't tell" must not cost the user an original download: with + // probe metadata too sparse to check the decoder bounds, the flat + // codec lists decide, exactly as they do without detailed caps. + name: "original with detailed caps stays direct when probe metadata is sparse", + requested: QualityOriginal, + file: sparseFile, + caps: detailedCaps, + transcodeEnabled: true, + userTranscode: true, + artifactsAvailable: true, + wantFormat: FormatOriginal, + wantQuality: QualityOriginal, + wantEffective: QualityOriginal, + }, + { + name: "original with detailed caps transcodes a source beyond the decoder bounds", + requested: QualityOriginal, + file: boundedFile, + caps: detailedCaps, + transcodeEnabled: true, + userTranscode: true, + artifactsAvailable: true, + wantFormat: FormatTranscode, + wantQuality: QualityOriginal, + wantEffective: Quality20Mbps, + wantBitrate: 20000, + }, { name: "remux is not a public quality", requested: FormatRemux, diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index fb272720f..2388e9d0c 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -376,13 +376,14 @@ func groupHealth(proxies, transcodes []*Node) map[string]bool { return health } -// pickTranscode returns the eligible transcode node with the fewest effective -// jobs, keeping the session on currentURL unless a candidate has at least two -// fewer jobs (the historical soft-affinity rule). -func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { +// pickNode returns the eligible node with the fewest effective jobs, keeping +// the session on currentURL unless a candidate has at least two fewer jobs +// (the historical soft-affinity rule). Shared by pickTranscode and +// pickLocalEgressTranscode, which differ only in their eligibility predicate. +func (p *Planner) pickNode(nodes []*Node, currentURL string, now time.Time, eligible func(*Node) bool) *Node { var best, current *Node - for _, n := range transcodes { - if !p.transcodeEligible(n, proxies, groupHealthy, estKbps, now) { + for _, n := range nodes { + if !eligible(n) { continue } if n.URL == currentURL { @@ -401,31 +402,25 @@ func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[st return current } +// pickTranscode returns the eligible transcode node with the fewest effective +// jobs, keeping the session on currentURL unless a candidate has at least two +// fewer jobs (the historical soft-affinity rule). +func (p *Planner) pickTranscode(transcodes, proxies []*Node, groupHealthy map[string]bool, currentURL string, estKbps int, now time.Time) *Node { + return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { + return p.transcodeEligible(n, proxies, groupHealthy, estKbps, now) + }) +} + // pickLocalEgressTranscode applies the transcode half of normal session // admission without requiring a healthy proxy partner. The API server is the // egress hop for this route, so unrelated proxy health and capacity must not -// suppress an otherwise healthy transcode executor. +// suppress an otherwise healthy transcode executor. Passing nil proxies to +// transcodeEligible reduces it to exactly that: healthy, enabled, under cap, +// and group-healthy, with no proxy partner required. func (p *Planner) pickLocalEgressTranscode(transcodes []*Node, groupHealthy map[string]bool, currentURL string, now time.Time) *Node { - var best, current *Node - for _, node := range transcodes { - if node == nil || !node.Healthy || !node.Enabled || !p.underCap(node, now) || - node.Group != nil && !groupHealthy[*node.Group] { - continue - } - if node.URL == currentURL { - current = node - } - if best == nil || p.effectiveJobs(node, now) < p.effectiveJobs(best, now) { - best = node - } - } - if current == nil || best == nil || current == best { - return best - } - if p.effectiveJobs(best, now)+2 <= p.effectiveJobs(current, now) { - return best - } - return current + return p.pickNode(transcodes, currentURL, now, func(n *Node) bool { + return p.transcodeEligible(n, nil, groupHealthy, 0, now) + }) } // transcodeEligible reports whether a transcode node may take a new session: diff --git a/internal/playback/attempt_sticky_features_v3_test.go b/internal/playback/attempt_sticky_features_v3_test.go new file mode 100644 index 000000000..ac596be07 --- /dev/null +++ b/internal/playback/attempt_sticky_features_v3_test.go @@ -0,0 +1,64 @@ +package playback + +import ( + "slices" + "testing" +) + +func TestPinAttemptStickyFeaturesV3(t *testing.T) { + tests := []struct { + name string + requested []string + negotiated []string + want []string + }{ + { + name: "a replan cannot drop a negotiated sticky feature", + requested: []string{FeaturePlaybackPlanV3}, + negotiated: []string{FeaturePlaybackPlanV3, FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + want: []string{FeaturePlaybackPlanV3, FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + }, + { + name: "a replan cannot add a sticky feature mid-attempt", + requested: []string{FeaturePlaybackPlanV3, FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + negotiated: []string{FeaturePlaybackPlanV3}, + want: []string{FeaturePlaybackPlanV3}, + }, + { + name: "an empty list still restores both sticky features", + requested: []string{}, + negotiated: []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + want: []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + }, + { + name: "case and padding do not smuggle a duplicate through", + requested: []string{" Header_Authenticated_Media_V1 ", FeatureDeviceQuirksV3}, + negotiated: []string{FeatureHeaderAuthenticatedMediaV3}, + want: []string{FeatureDeviceQuirksV3, FeatureHeaderAuthenticatedMediaV3}, + }, + { + name: "non-sticky features pass through in order", + requested: []string{FeatureDeviceQuirksV3, FeatureClientVideoTransforms}, + negotiated: []string{FeatureClientVideoTransforms}, + want: []string{FeatureDeviceQuirksV3, FeatureClientVideoTransforms}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := PinAttemptStickyFeaturesV3(test.requested, test.negotiated); !slices.Equal(got, test.want) { + t.Fatalf("pinned = %v, want %v", got, test.want) + } + }) + } +} + +// Every sticky feature must be one the server actually advertises, or a client +// could never negotiate it in the first place. +func TestAttemptStickyFeaturesV3AreAdvertised(t *testing.T) { + advertised := ServerFeaturesV3() + for _, feature := range AttemptStickyFeaturesV3() { + if !HasFeatureV3(advertised, feature) { + t.Fatalf("attempt-sticky feature %q is not advertised by the server", feature) + } + } +} diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index d644deebe..5f6c5bf56 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -666,8 +666,14 @@ type SubtitleArtifactV3 struct { } type SubtitleDecisionV3 struct { - Mode SubtitleModeV3 `json:"mode"` - TrackID string `json:"track_id,omitempty"` + Mode SubtitleModeV3 `json:"mode"` + TrackID string `json:"track_id,omitempty"` + // Artifact is the single track the client draws. It exists only under + // SubtitleRenderV3 and SubtitleConvertV3; SubtitleOffV3 and + // SubtitleBurnInV3 have no client-fetchable artifact and must publish none, + // including on a plan derived from an earlier plan of the same session. + // SubtitleOffV3 carries no TrackID either. Inventory URLs are independent + // of the selection and stay published in every mode. Artifact *SubtitleArtifactV3 `json:"artifact,omitempty"` // Inventory is the complete, gap-free combined-ordinal subtitle track list // for the effective source. It is authoritative: a client selects a track @@ -1114,6 +1120,46 @@ func HasFeatureV3(features []string, wanted string) bool { return slices.ContainsFunc(features, func(v string) bool { return strings.EqualFold(strings.TrimSpace(v), wanted) }) } +// AttemptStickyFeaturesV3 lists the client features that are negotiated once, +// at start, and are fixed for the lifetime of the playback attempt. Each of +// them selects a contract the durable plan and its live transport are built +// around rather than a per-plan preference: +// +// - header_authenticated_media_v1 picks the media security contract. A legacy +// signed URL from an earlier plan can outlive the plan that minted it, so +// switching mid-attempt would leave two contracts alive for one session. +// - software_video_decode_v1 widens the direct-play evidence tiers. Dropping +// it on a replan silently converts a direct route into a transcode and +// persists that downgrade into the durable normalized request. +// +// Stop/start is the explicit boundary for changing either. +func AttemptStickyFeaturesV3() []string { + return []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3} +} + +// PinAttemptStickyFeaturesV3 returns requested with every attempt-sticky +// feature forced back to the state the start negotiation established: a replan +// can neither add nor remove one, whatever its own client_features list says. +// Non-sticky features are passed through untouched and in order. +func PinAttemptStickyFeaturesV3(requested, negotiated []string) []string { + sticky := AttemptStickyFeaturesV3() + pinned := make([]string, 0, len(requested)+len(sticky)) + for _, feature := range requested { + if slices.ContainsFunc(sticky, func(candidate string) bool { + return strings.EqualFold(strings.TrimSpace(feature), candidate) + }) { + continue + } + pinned = append(pinned, feature) + } + for _, feature := range sticky { + if HasFeatureV3(negotiated, feature) { + pinned = append(pinned, feature) + } + } + return pinned +} + func NewTerminalResponseV3(reason, message string, retryable bool) DecisionResponseV3 { return DecisionResponseV3{ ProtocolVersion: ProtocolV3, diff --git a/internal/playback/resolver.go b/internal/playback/resolver.go index 7ff48a81a..0f69fad0b 100644 --- a/internal/playback/resolver.go +++ b/internal/playback/resolver.go @@ -38,16 +38,28 @@ type ClientCapabilities struct { VideoDecode []VideoDecodeCapabilityV3 `json:"video_decode,omitempty"` } +// hasDetailedVideoEvidence reports whether the payload carries a strict-tier +// detailed decoder description: per-decoder video_decode entries backed by an +// evidence tier that can validate them. It is the single predicate shared by +// the additive validator and Resolve so the two cannot drift. +func (c *ClientCapabilities) hasDetailedVideoEvidence() bool { + return (c.VideoEvidence == EvidenceExactV3 || c.VideoEvidence == EvidencePlatformAttestedV3) && + len(c.VideoDecode) > 0 +} + // NormalizeAndValidateVideoDecode applies the protocol-v3 detailed decoder -// limits to additive capability payloads such as download creation. Legacy -// flat-only payloads remain valid and unchanged. +// limits to additive capability payloads such as download creation. It mirrors +// the v3 playback start path: flat-list payloads at any evidence tier — and +// feature-token-only payloads — stay valid and unchanged, because those resolve +// from the flat codec lists exactly as playback does. Only a partial detailed +// opt-in is refused: video_decode entries whose evidence tier cannot validate +// them would otherwise be silently ignored. func (c *ClientCapabilities) NormalizeAndValidateVideoDecode() error { - softwareOptIn := HasFeatureV3(c.ClientFeatures, FeatureSoftwareVideoDecodeV3) - if c.VideoEvidence == "" && len(c.VideoDecode) == 0 && !softwareOptIn { + if len(c.VideoDecode) == 0 { return nil } - if (c.VideoEvidence != EvidenceExactV3 && c.VideoEvidence != EvidencePlatformAttestedV3) || len(c.VideoDecode) == 0 { - return errors.New("detailed download video evidence requires exact or platform_attested entries") + if !c.hasDetailedVideoEvidence() { + return errors.New("video_decode requires exact or platform_attested video_evidence") } detailed := ClientCodecCapabilitiesV3{ VideoEvidence: c.VideoEvidence, @@ -82,16 +94,26 @@ type PlayDecision struct { func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSettings) *PlayDecision { // Check if client supports the video codec. videoOK := containsStr(caps.CodecsVideo, file.CodecVideo) - detailedVideoEvidence := (caps.VideoEvidence == EvidenceExactV3 || caps.VideoEvidence == EvidencePlatformAttestedV3) && len(caps.VideoDecode) > 0 + detailedVideoEvidence := caps.hasDetailedVideoEvidence() if detailedVideoEvidence { - videoOK, _ = videoEligibleV3(SourceDescriptorFromFileV3(file, 0), StartRequestV3{ - ClientFeatures: caps.ClientFeatures, - Capabilities: ClientCodecCapabilitiesV3{ - VideoEvidence: caps.VideoEvidence, - CodecsVideo: caps.CodecsVideo, - VideoDecode: caps.VideoDecode, - }, - }) + source := SourceDescriptorFromFileV3(file, 0) + // Detailed validation needs complete probe facts (codec, bit depth, + // dimensions, frame rate, bitrate). A file whose probe metadata is + // sparse cannot be checked against the decoder bounds at all — that is + // "can't tell", not "incompatible", and forcing a transcode of an + // original-quality download over it would be a silent quality loss. Keep + // the flat-list answer in that case; a real mismatch (complete metadata + // whose entries do not cover the source) still fails closed below. + if routeVideoMetadataCompleteV3(source) { + videoOK, _ = videoEligibleV3(source, StartRequestV3{ + ClientFeatures: caps.ClientFeatures, + Capabilities: ClientCodecCapabilitiesV3{ + VideoEvidence: caps.VideoEvidence, + CodecsVideo: caps.CodecsVideo, + VideoDecode: caps.VideoDecode, + }, + }) + } } // Audio is considered OK if the client can decode the codec itself OR its // sink can passthrough it. Passthrough lets us stream-copy surround audio diff --git a/internal/playback/resolver_test.go b/internal/playback/resolver_test.go index dbe8cfcca..9ad0ce0fc 100644 --- a/internal/playback/resolver_test.go +++ b/internal/playback/resolver_test.go @@ -238,7 +238,10 @@ func TestResolver_DetailedHardwareEvidenceOverridesLegacyDownloadCeiling(t *test } } -func TestResolver_DetailedDownloadEvidenceFailsClosedWhenProbeFactsAreIncomplete(t *testing.T) { +func TestResolver_DetailedDownloadEvidenceFallsBackToFlatListsOnSparseProbeFacts(t *testing.T) { + // No video tracks: bit depth, dimensions, frame rate and bitrate are all + // unknown, so the decoder bounds cannot be evaluated. "Can't tell" must not + // force a transcode of an original-quality download. file := &models.MediaFile{ CodecVideo: "av1", CodecAudio: "aac", Container: "mp4", Resolution: "1080p", @@ -256,8 +259,147 @@ func TestResolver_DetailedDownloadEvidenceFailsClosedWhenProbeFactsAreIncomplete }}, } + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayDirect { + t.Fatalf("sparse-metadata source with detailed caps = %q, want direct", decision.Method) + } + + flatOnly := caps + flatOnly.VideoEvidence = "" + flatOnly.VideoDecode = nil + flat := playback.Resolve(file, flatOnly, defaultSettings()) + if detailed := playback.Resolve(file, caps, defaultSettings()); detailed.Method != flat.Method { + t.Fatalf("sparse-metadata detailed caps = %q, flat caps = %q; want identical", detailed.Method, flat.Method) + } +} + +func TestResolver_DetailedDownloadEvidenceFailsClosedOnCompleteMetadataMismatch(t *testing.T) { + // Complete probe facts whose decoder entry does not cover the source: a real + // mismatch, so the flat-list claim must not rescue it. + file := &models.MediaFile{ + CodecVideo: "av1", CodecAudio: "aac", Container: "mp4", + Resolution: "2160p", Bitrate: 55_000, + VideoTracks: []models.VideoTrack{{ + Codec: "av1", Profile: "Main", Width: 3840, Height: 2160, FrameRate: "60/1", + Bitrate: 55_000, BitDepth: 10, + }}, + } + caps := playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"av1"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + MaxResolution: "2160p", + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", BitDepths: []int{8, 10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: true, + }}, + } + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayTranscode { - t.Fatalf("incomplete strict evidence = %q, want transcode", decision.Method) + t.Fatalf("out-of-bounds source with complete metadata = %q, want transcode", decision.Method) + } +} + +func TestNormalizeAndValidateVideoDecode(t *testing.T) { + tests := []struct { + name string + caps playback.ClientCapabilities + wantErr bool + }{ + { + name: "declared evidence with flat lists only", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.EvidenceDeclaredV3, + CodecsVideo: []string{"h264"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + }, + }, + { + name: "feature token only", + caps: playback.ClientCapabilities{ + ClientFeatures: []string{playback.FeatureSoftwareVideoDecodeV3}, + CodecsVideo: []string{"h264"}, + }, + }, + { + name: "legacy flat payload", + caps: playback.ClientCapabilities{CodecsVideo: []string{"h264"}}, + }, + { + name: "platform attested entries", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"H264"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "H264", MaxWidth: 1920, MaxHeight: 1080, Hardware: true, + }}, + }, + }, + { + name: "entries with declared evidence", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.EvidenceDeclaredV3, + CodecsVideo: []string{"av1"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", Hardware: true, + }}, + }, + wantErr: true, + }, + { + name: "entries without evidence", + caps: playback.ClientCapabilities{ + CodecsVideo: []string{"av1"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", Hardware: true, + }}, + }, + wantErr: true, + }, + { + name: "malformed entry", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"av1"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", MaxWidth: -1, Hardware: true, + }}, + }, + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + caps := tc.caps + err := caps.NormalizeAndValidateVideoDecode() + if tc.wantErr { + if err == nil { + t.Fatal("err = nil, want a validation error") + } + return + } + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + }) + } +} + +func TestNormalizeAndValidateVideoDecodeLowercasesDetailedEntries(t *testing.T) { + caps := playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{" HEVC "}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: " HEVC ", MaxWidth: 3840, MaxHeight: 2160, Hardware: true, + }}, + } + if err := caps.NormalizeAndValidateVideoDecode(); err != nil { + t.Fatalf("err = %v, want nil", err) + } + if caps.CodecsVideo[0] != "hevc" || caps.VideoDecode[0].Codec != "hevc" { + t.Fatalf("normalization did not apply: %+v", caps) } } diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go index ac7e80a36..c8840f408 100644 --- a/internal/playback/transcode_manager.go +++ b/internal/playback/transcode_manager.go @@ -337,8 +337,10 @@ const ( // re-binding ownership to the live caller. The two-factor ownership rule is // preserved exactly — a live session with a non-zero, mismatched caller is // refused; reconstruct itself refuses a zero/mismatched caller — so this widens -// no access. getSession is supplied by the caller (its SessionManager.GetSession) -// so the manager needs no direct handle on the manager type. +// no access. A live session that requires media authorization additionally +// refuses a caller with no identity at all. getSession is supplied by the caller +// (its SessionManager.GetSession) so the manager needs no direct handle on the +// manager type. // // card is the reconstruction recipe the caller decoded from the verified stream // token the client presented (nil when the request carried no usable token). @@ -369,6 +371,16 @@ func (m *TranscodeManager) LoadOrReconstructSession(ctx context.Context, getSess // Live session: enforce the existing ownership check. A zero caller is // allowed (these routes treat the session UUID as a bearer when auth is // optional); a non-zero mismatch is refused. + // + // A session that negotiated header-authenticated media is the exception: its + // UUID is a route identifier and never a credential, so an unauthenticated + // caller is refused here — at the front door every serve handler shares — + // rather than at each handler's own fast path. Sessions that never + // negotiated the mode (legacy v3 and jellycompat alike) keep the bearer + // behavior unchanged. + if requestUserID == 0 && session.RequireMediaAuthorization { + return nil, SessionUnauthorized + } if requestUserID != 0 && session.UserID != requestUserID { return nil, SessionForbidden } diff --git a/internal/playback/transcode_manager_test.go b/internal/playback/transcode_manager_test.go index 825ce4e8d..9323cc6bf 100644 --- a/internal/playback/transcode_manager_test.go +++ b/internal/playback/transcode_manager_test.go @@ -117,6 +117,25 @@ func TestLoadOrReconstructSession(t *testing.T) { } }) + // A v3 transport that negotiated header-authenticated media never treats its + // session UUID as a credential, so the front door refuses an anonymous + // caller instead of leaving that rule to each serve handler. + t.Run("live media-authorized session, zero caller -> unauthorized", func(t *testing.T) { + reg := &fakeSessionRegistry{sessions: map[string]*Session{"s": {ID: "s", UserID: 5, RequireMediaAuthorization: true}}} + m := newMgr(reg) + if got, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 0, nil); status != SessionUnauthorized || got != nil { + t.Fatalf("status = %v session = %+v, want unauthorized", status, got) + } + }) + + t.Run("live media-authorized session, owner -> loaded", func(t *testing.T) { + reg := &fakeSessionRegistry{sessions: map[string]*Session{"s": {ID: "s", UserID: 5, RequireMediaAuthorization: true}}} + m := newMgr(reg) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 5, nil); status != SessionLoaded { + t.Fatalf("status = %v, want loaded", status) + } + }) + t.Run("miss + remux token + matching owner -> reconstructed with method", func(t *testing.T) { reg := &fakeSessionRegistry{} m := newMgr(reg) diff --git a/internal/worker/reconciler.go b/internal/worker/reconciler.go index 7854a2f78..63911123c 100644 --- a/internal/worker/reconciler.go +++ b/internal/worker/reconciler.go @@ -39,14 +39,18 @@ type SessionSync struct { TargetResolution string TargetVideoCodec string TargetAudioCodec string - TargetBitrateKbps int - TranscodeHWAccel string - StartedAt time.Time - UpdatedAt time.Time - PositionSeconds float64 - IsPaused bool - HasWebSocket bool - IsJellyfinCompat bool + // TargetAudioChannels is the encoded output channel count when audio is + // re-encoded; 0 means the node did not report one. Admin views must not + // substitute the source count for it. + TargetAudioChannels int + TargetBitrateKbps int + TranscodeHWAccel string + StartedAt time.Time + UpdatedAt time.Time + PositionSeconds float64 + IsPaused bool + HasWebSocket bool + IsJellyfinCompat bool } // AggregateData represents the aggregate counts for a single user that are @@ -155,9 +159,10 @@ func (r *Reconciler) ReconcileNodeSessions(ctx context.Context, reportingNode st reporting_node, started_at, updated_at, last_sync_at, client_ip, client_name, client_version, client_build, client_channel, client_user_agent, audio_track_index, transcode_audio, stream_bitrate_kbps, transcode_node_url, - target_resolution, target_video_codec, target_audio_codec, target_bitrate_kbps, + target_resolution, target_video_codec, target_audio_codec, target_audio_channels, + target_bitrate_kbps, transcode_hw_accel, position_seconds, is_paused, has_websocket, compat_origin) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW(), $10::inet, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW(), $10::inet, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29) ON CONFLICT (session_id) DO UPDATE SET user_id = EXCLUDED.user_id, profile_id = EXCLUDED.profile_id, @@ -180,6 +185,7 @@ func (r *Reconciler) ReconcileNodeSessions(ctx context.Context, reportingNode st target_resolution = EXCLUDED.target_resolution, target_video_codec = EXCLUDED.target_video_codec, target_audio_codec = EXCLUDED.target_audio_codec, + target_audio_channels = EXCLUDED.target_audio_channels, target_bitrate_kbps = EXCLUDED.target_bitrate_kbps, transcode_hw_accel = EXCLUDED.transcode_hw_accel, position_seconds = EXCLUDED.position_seconds, @@ -193,7 +199,8 @@ func (r *Reconciler) ReconcileNodeSessions(ctx context.Context, reportingNode st nullableString(s.ClientChannel), nullableString(s.ClientUserAgent), s.AudioTrackIndex, s.TranscodeAudio, nullableInt(s.StreamBitrateKbps), nullableString(s.TranscodeNodeURL), nullableString(s.TargetResolution), nullableString(s.TargetVideoCodec), - nullableString(s.TargetAudioCodec), nullableInt(s.TargetBitrateKbps), + nullableString(s.TargetAudioCodec), nullableInt(s.TargetAudioChannels), + nullableInt(s.TargetBitrateKbps), nullableString(s.TranscodeHWAccel), normalizePositionSeconds(s.PositionSeconds), s.IsPaused, s.HasWebSocket, s.IsJellyfinCompat) if err != nil { @@ -266,6 +273,7 @@ func loadNodeSessionsSnapshot(ctx context.Context, tx pgx.Tx, reportingNode stri COALESCE(target_resolution, ''), COALESCE(target_video_codec, ''), COALESCE(target_audio_codec, ''), + COALESCE(target_audio_channels, 0), COALESCE(target_bitrate_kbps, 0), COALESCE(transcode_hw_accel, ''), started_at, @@ -307,6 +315,7 @@ func loadNodeSessionsSnapshot(ctx context.Context, tx pgx.Tx, reportingNode stri &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, + &s.TargetAudioChannels, &s.TargetBitrateKbps, &s.TranscodeHWAccel, &s.StartedAt, @@ -366,6 +375,7 @@ func sessionSnapshotsEqual(left, right []SessionSync) bool { left[i].TargetResolution != right[i].TargetResolution || left[i].TargetVideoCodec != right[i].TargetVideoCodec || left[i].TargetAudioCodec != right[i].TargetAudioCodec || + left[i].TargetAudioChannels != right[i].TargetAudioChannels || left[i].TargetBitrateKbps != right[i].TargetBitrateKbps || left[i].TranscodeHWAccel != right[i].TranscodeHWAccel || !left[i].StartedAt.Equal(right[i].StartedAt) || diff --git a/migrations/sql/20260823101500_add_playback_session_target_audio_channels.sql b/migrations/sql/20260823101500_add_playback_session_target_audio_channels.sql new file mode 100644 index 000000000..03b0446b4 --- /dev/null +++ b/migrations/sql/20260823101500_add_playback_session_target_audio_channels.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- The admin activity views label a transcode's delivered audio, and without the +-- target channel count they had to borrow the source's — rendering a TrueHD 7.1 +-- source downmixed to AAC 5.1 as "AAC 7.1". The planner already resolves the +-- encoded channel count, so carry it alongside the target codec. NULL means the +-- reporting node did not know it; consumers must show no channel layout rather +-- than falling back to the source count. +ALTER TABLE public.playback_sessions_sync + ADD COLUMN IF NOT EXISTS target_audio_channels integer; + +-- +goose Down +ALTER TABLE public.playback_sessions_sync + DROP COLUMN IF EXISTS target_audio_channels; diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 93f7bee2f..d23b9e357 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -2514,6 +2514,10 @@ export interface AdminSession { target_resolution?: string; target_video_codec?: string; target_audio_codec?: string; + /** Channel count the transcode actually encodes. Absent when the reporting + * node did not know it — render the target codec with no channel layout + * rather than falling back to `source_audio_channels`. */ + target_audio_channels?: number | null; target_bitrate_kbps: number | null; transcode_hw_accel?: string; source_container?: string; diff --git a/web/src/components/UserPolicyFields.tsx b/web/src/components/UserPolicyFields.tsx index 1f2f4167b..3a247b944 100644 --- a/web/src/components/UserPolicyFields.tsx +++ b/web/src/components/UserPolicyFields.tsx @@ -122,6 +122,13 @@ export function policyInheritHints( }; } +// Admins are never grouped: the server clears access_group_id for the admin +// role (auth.Repository.CreateUser/UpdateUser), so every form that shows or +// submits a group for a user has to mirror that rule locally. +export function effectiveAccessGroupID(role: string, accessGroupID: number | null): number | null { + return role === "admin" ? null : accessGroupID; +} + interface PolicyContext { state: UserPolicyState; onChange: (state: UserPolicyState) => void; diff --git a/web/src/pages/AdminUserDetail.tsx b/web/src/pages/AdminUserDetail.tsx index 44b26612a..d1db0c621 100644 --- a/web/src/pages/AdminUserDetail.tsx +++ b/web/src/pages/AdminUserDetail.tsx @@ -28,6 +28,7 @@ import { Button } from "@/components/ui/button"; import { PolicyAccessFields, PolicyLimitFields, + effectiveAccessGroupID, policyInheritHints, policyStateFromUser, policyUpdateFields, @@ -1067,7 +1068,7 @@ function EditUserForm({ user, onClose }: { user: AdminUser; onClose: () => void // server sent — but only while the saved group is still the selected one. // An admin inherits from no group, so preview the no-group policy while the // picked group is kept for toggling the role back. - const hintGroupID = role === "admin" ? null : accessGroupID; + const hintGroupID = effectiveAccessGroupID(role, accessGroupID); const inheritHints = policyInheritHints(hintGroupID, accessGroups) ?? (hintGroupID === user.access_group_id ? user.effective_policy : undefined); @@ -1084,7 +1085,7 @@ function EditUserForm({ user, onClose }: { user: AdminUser; onClose: () => void enabled, // Admins are never grouped; derive it here so flipping the role back // before saving keeps the picked group. - access_group_id: role === "admin" ? null : accessGroupID, + access_group_id: effectiveAccessGroupID(role, accessGroupID), max_profiles: maxProfiles, ...policyUpdateFields(policy), }; diff --git a/web/src/pages/AdminUsers.tsx b/web/src/pages/AdminUsers.tsx index b3836f9b6..0211c9bd1 100644 --- a/web/src/pages/AdminUsers.tsx +++ b/web/src/pages/AdminUsers.tsx @@ -13,6 +13,7 @@ import { useAccessGroups } from "@/hooks/queries/admin/accessGroups"; import { PolicyAccessFields, PolicyLimitFields, + effectiveAccessGroupID, policyCreateFields, policyInheritHints, policyStateFromUser, @@ -543,7 +544,7 @@ function UserForm({ user, onClose }: { user: AdminUser | null; onClose: () => vo // new account lands on the default group — except an admin, which the server // deliberately leaves ungrouped (auth.Repository.CreateUser). const defaultGroupID = accessGroups.find((group) => group.is_default)?.id ?? null; - const inheritGroupID = role === "admin" ? null : user ? user.access_group_id : defaultGroupID; + const inheritGroupID = effectiveAccessGroupID(role, user ? user.access_group_id : defaultGroupID); const inheritHints = policyInheritHints(inheritGroupID, accessGroups) ?? (role === "admin" ? undefined : user?.effective_policy); @@ -561,7 +562,7 @@ function UserForm({ user, onClose }: { user: AdminUser | null; onClose: () => vo ...policyUpdateFields(policy), }; if (role === "admin") { - body.access_group_id = null; + body.access_group_id = effectiveAccessGroupID(role, user.access_group_id); } if (password) body.password = password; updateMutation.mutate({ id: user.id, body }, { onSuccess: onClose }); diff --git a/web/src/pages/admin-settings/InvitationsTab.tsx b/web/src/pages/admin-settings/InvitationsTab.tsx index d3e4fffb2..7cd9b6a3d 100644 --- a/web/src/pages/admin-settings/InvitationsTab.tsx +++ b/web/src/pages/admin-settings/InvitationsTab.tsx @@ -9,6 +9,7 @@ import { } from "@/hooks/queries/admin/invitations"; import { useAccessGroups } from "@/hooks/queries/admin/accessGroups"; import { useAdminLibraries } from "@/hooks/queries/admin/libraries"; +import { effectiveAccessGroupID } from "@/components/UserPolicyFields"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -295,7 +296,7 @@ function CreateInvitationForm({ { email, role, - access_group_id: role === "admin" ? null : accessGroupID, + access_group_id: effectiveAccessGroupID(role, accessGroupID), library_ids: libraryIDs, create_profile: createProfile, show_tour: showTour, diff --git a/web/src/pages/adminActivityPresentation.test.ts b/web/src/pages/adminActivityPresentation.test.ts index b51580d87..040c6f861 100644 --- a/web/src/pages/adminActivityPresentation.test.ts +++ b/web/src/pages/adminActivityPresentation.test.ts @@ -4,6 +4,8 @@ import { classifyActivityMethod, compareActivityMethods, isJellyfinSession, + formatAudioDetail, + formatAudioSummary, formatContainerDetail, formatDeliveredAudioSummary, formatDeliveredContainerSummary, @@ -61,6 +63,7 @@ function makeSession(overrides: Partial = {}): AdminSession { source_audio_channels: overrides.source_audio_channels ?? 2, audio_decision: overrides.audio_decision, target_audio_codec: overrides.target_audio_codec, + target_audio_channels: overrides.target_audio_channels, requested_video_codec: overrides.requested_video_codec ?? "hevc", requested_video_resolution: overrides.requested_video_resolution ?? "2160p", }; @@ -91,6 +94,7 @@ describe("adminActivityPresentation", () => { source_audio_codec: "eac3", source_audio_channels: 6, target_audio_codec: "aac", + target_audio_channels: 6, }); expect(formatPlaybackDecisionSummary(session)).toBe("transcode"); @@ -150,7 +154,35 @@ describe("adminActivityPresentation", () => { transcode_hw_accel: "qsv", }), ), - ).toBe("Audio SW"); + // Audio-only re-encodes have no video encoder, so they get named for the + // work rather than for an acceleration mode they never used. + ).toBe("Audio Transcode"); + }); + + it("labels a transcode's audio with the target channel count, never the source's", () => { + // TrueHD 7.1 downmixed to AAC 5.1 must not claim 7.1 output. + const downmixed = makeSession({ + audio_decision: "transcode", + source_audio_codec: "truehd", + source_audio_channels: 8, + target_audio_codec: "aac", + target_audio_channels: 6, + }); + expect(formatDeliveredAudioSummary(downmixed)).toBe("AAC 5.1"); + expect(formatAudioDetail(downmixed)).toBe("→ AAC 5.1"); + + // Server did not report a target count → codec alone, not the source's. + const unknownTarget = makeSession({ + audio_decision: "transcode", + source_audio_codec: "truehd", + source_audio_channels: 8, + target_audio_codec: "aac", + }); + expect(formatDeliveredAudioSummary(unknownTarget)).toBe("AAC"); + expect(formatAudioDetail(unknownTarget)).toBe("→ AAC"); + + // The source summary still describes the source in full. + expect(formatAudioSummary(unknownTarget)).toBe("TrueHD 7.1"); }); it("buckets activity sessions by the backend's per-stream decisions", () => { diff --git a/web/src/pages/adminActivityPresentation.ts b/web/src/pages/adminActivityPresentation.ts index 9492a5a77..eda394d9f 100644 --- a/web/src/pages/adminActivityPresentation.ts +++ b/web/src/pages/adminActivityPresentation.ts @@ -199,8 +199,12 @@ export function formatTranscodeModeSummary(session: AdminSession): string | null if (videoDecision !== "transcode" && audioDecision !== "transcode") { return null; } + // Every other label in this function names the video encoder's HW/SW mode, + // which does not exist when only audio is re-encoded: "Audio SW" read as a + // client-side software capability instead of "the audio stream is being + // re-encoded". Name the work, not an acceleration mode. if (videoDecision !== "transcode") { - return "Audio SW"; + return "Audio Transcode"; } const hwAccel = session.transcode_hw_accel?.trim().toLowerCase(); @@ -349,6 +353,22 @@ export function formatAudioSummary(session: AdminSession): string { return [lead, format].filter(Boolean).join(" · ") || "Unknown source"; } +/** + * The audio a transcode actually delivers: the target codec, plus the target + * channel layout only when the server reported one. The source channel count is + * deliberately not a fallback — a TrueHD 7.1 source downmixed to AAC 5.1 read as + * "AAC 7.1" while it did. Servers that do not send `target_audio_channels` get + * the bare codec instead of an invented layout. + */ +function formatTargetAudio(session: AdminSession): string { + return [ + formatCodec(session.target_audio_codec || "aac"), + formatChannelLayout(session.target_audio_channels), + ] + .filter(Boolean) + .join(" "); +} + export function formatDeliveredAudioSummary(session: AdminSession): string { const decision = session.audio_decision || (session.transcode_audio ? "transcode" : session.play_method); @@ -356,14 +376,7 @@ export function formatDeliveredAudioSummary(session: AdminSession): string { return formatAudioSummary(session); } - return ( - [ - formatCodec(session.target_audio_codec || "aac"), - formatChannelLayout(session.source_audio_channels), - ] - .filter(Boolean) - .join(" ") || "Audio transcode" - ); + return formatTargetAudio(session) || "Audio Transcode"; } export function formatAudioDetail(session: AdminSession): string { @@ -371,13 +384,8 @@ export function formatAudioDetail(session: AdminSession): string { session.audio_decision || (session.transcode_audio ? "transcode" : session.play_method), ); if (decision === "transcode") { - const target = [ - formatCodec(session.target_audio_codec || "aac"), - formatChannelLayout(session.source_audio_channels), - ] - .filter(Boolean) - .join(" "); - return target ? `→ ${target}` : "Audio transcode"; + const target = formatTargetAudio(session); + return target ? `→ ${target}` : "Audio Transcode"; } if (decision === "copy") { return "Audio stream copied"; From bbca34f396622ba1974900078495660e2b87aed1 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 02:01:59 -0400 Subject: [PATCH 26/44] fix(playback): regenerate conformance matrix for software_video_decode_v1 make verify-playback-fixtures failed on CI because one matrix entry was missing the new server feature string. Co-Authored-By: Claude Fable 5 --- internal/playback/testdata/protocol_v3/conformance_matrix.json | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index 8e251e89e..22bf6cf38 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -5409,6 +5409,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "software_video_decode_v1", "plan_source_duration_v1" ], "outcome": "adaptation_unavailable", From 8e3f03f003046569620ab0781d49daea8789d0ff Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 02:04:23 -0400 Subject: [PATCH 27/44] fix(downloads): apply the coarse resolution ceiling when the detailed bounds walk cannot run With detailed video_decode evidence and sparse probe metadata, Resolve skipped both the per-decoder bounds walk and the flat max_resolution ceiling, approving original-quality downloads beyond the device ceiling. Sparse metadata now fails closed to the flat contract, ceiling included; complete metadata keeps letting a validated detailed entry override the coarse ceiling. Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 6 ++++- internal/playback/resolver.go | 14 ++++++++-- internal/playback/resolver_test.go | 32 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 220d2867a..b30e5ff7b 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -346,7 +346,11 @@ ceilings: the flat `max_resolution` remains a coarse device ceiling, while the detailed entry decides whether a particular original file is safe. Apple keeps the legacy coarse ceiling at 1080p so older servers fail safely; a detailed hardware entry may independently preserve a 4K original on a new server. -Legacy flat-only download clients keep the previous resolver behavior. +Legacy flat-only download clients keep the previous resolver behavior. When a +file's probe metadata is too sparse for the detailed bounds walk to run at +all, the flat claims decide instead — including the coarse `max_resolution` +ceiling — so sparse metadata cannot widen eligibility beyond the flat +contract. **An omitted bound means "unconstrained", not "unknown".** Within a `video_decode[]` entry, an empty `profiles`, `levels`, or `bit_depths` list and a diff --git a/internal/playback/resolver.go b/internal/playback/resolver.go index 0f69fad0b..7de821ecf 100644 --- a/internal/playback/resolver.go +++ b/internal/playback/resolver.go @@ -95,6 +95,13 @@ func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSett // Check if client supports the video codec. videoOK := containsStr(caps.CodecsVideo, file.CodecVideo) detailedVideoEvidence := caps.hasDetailedVideoEvidence() + // detailedBoundsChecked is true only when the videoEligibleV3 bounds walk + // below actually ran. It gates the resolution ceiling check further down: + // when the detailed walk ran, its per-decoder max_width/max_height are + // authoritative and the coarse ceiling is redundant; when it could not run + // (sparse probe metadata), the coarse ceiling must still apply so sparse + // metadata cannot widen eligibility beyond the flat contract. + detailedBoundsChecked := false if detailedVideoEvidence { source := SourceDescriptorFromFileV3(file, 0) // Detailed validation needs complete probe facts (codec, bit depth, @@ -102,9 +109,12 @@ func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSett // sparse cannot be checked against the decoder bounds at all — that is // "can't tell", not "incompatible", and forcing a transcode of an // original-quality download over it would be a silent quality loss. Keep - // the flat-list answer in that case; a real mismatch (complete metadata + // the flat-list answer in that case — which includes the coarse + // max_resolution ceiling, so sparse metadata fails closed to that + // ceiling instead of failing open; a real mismatch (complete metadata // whose entries do not cover the source) still fails closed below. if routeVideoMetadataCompleteV3(source) { + detailedBoundsChecked = true videoOK, _ = videoEligibleV3(source, StartRequestV3{ ClientFeatures: caps.ClientFeatures, Capabilities: ClientCodecCapabilitiesV3{ @@ -124,7 +134,7 @@ func Resolve(file *models.MediaFile, caps ClientCapabilities, settings AdminSett containerOK := containsStr(caps.Containers, file.Container) // Check resolution constraint. - if !detailedVideoEvidence && !resolutionFits(file.Resolution, caps.MaxResolution) { + if !detailedBoundsChecked && !resolutionFits(file.Resolution, caps.MaxResolution) { if !settings.TranscodeEnabled { return &PlayDecision{ Method: PlayDirect, diff --git a/internal/playback/resolver_test.go b/internal/playback/resolver_test.go index 9ad0ce0fc..226f83a61 100644 --- a/internal/playback/resolver_test.go +++ b/internal/playback/resolver_test.go @@ -403,6 +403,38 @@ func TestNormalizeAndValidateVideoDecodeLowercasesDetailedEntries(t *testing.T) } } +func TestResolver_DetailedCapsWithSparseMetadataFailsClosedToCoarseCeiling(t *testing.T) { + // Detailed platform_attested caps with a hardware entry bounded to + // 1920x1080, but the source's probe metadata is incomplete (zero bitrate) + // so the detailed bounds walk cannot run at all. The coarse max_resolution + // ceiling must still apply — same outcome as a flat-only payload — rather + // than approving an original-quality 2160p download past the device ceiling. + file := &models.MediaFile{ + CodecVideo: "hevc", CodecAudio: "aac", Container: "mp4", + Resolution: "2160p", Bitrate: 0, + VideoTracks: []models.VideoTrack{{ + Codec: "hevc", Profile: "Main 10", Width: 3840, Height: 2160, + FrameRate: "60/1", Bitrate: 0, BitDepth: 10, + }}, + } + caps := playback.ClientCapabilities{ + VideoEvidence: playback.EvidencePlatformAttestedV3, + CodecsVideo: []string{"hevc"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + MaxResolution: "1080p", + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "hevc", BitDepths: []int{8, 10}, MaxWidth: 1920, + MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 40_000, + Hardware: true, + }}, + } + + if decision := playback.Resolve(file, caps, defaultSettings()); decision.Method != playback.PlayTranscode { + t.Fatalf("sparse-metadata source with coarse-ceiling-exceeding detailed caps = %q, want transcode", decision.Method) + } +} + func TestResolver_Transcode_ResolutionExceeds(t *testing.T) { file := &models.MediaFile{ CodecVideo: "h264", CodecAudio: "aac", Container: "mp4", From d6ba68b1ec3ebca7eb92082e271cc65ee4f789cb Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 02:44:57 -0400 Subject: [PATCH 28/44] feat(playback): restore proxy and transcode-node egress for header-authenticated media header_authenticated_media_v1 kept every media byte on the API server because proxies could only authenticate from the signed URL token that mode removes. A new attempt-sticky opt-in, authorized_media_origins_v1, restores distributed egress without putting a credential back in any URL: - Plans for an attempt that negotiated both features may return absolute, credential-free proxy URLs (/stream/v3/{session_id} family) for direct play, progressive remux, and node-executed HLS. - The proxy is told what to serve out of band: the API writes the session recipe to a Redis proxy-grant store (silo:proxygrant:, sibling of the noderecipe handoff), overwritten on replan and revoked on session stop, abort, and uncommitted-transport rollback. - The proxy authenticates the caller itself: bearer JWT against the live signing secret plus the same auth_sessions liveness check the API runs, then ownership against the grant. Revoking a login stops proxy playback immediately. Node-relay tokens are minted proxy-side and never reach the client. - RecipeCard now carries DVProfile/AudioOnly so a grant-served remux reproduces the exact bytes the token path would have. - The progressive-remux escalation to HLS now applies only when no proxy origin is available; grant-write failure falls back to the API origin under the same local_transcode_fallback gate as the no-origins mode. Header-auth-only clients and deployments without a proxy pool keep the current API-local behavior unchanged. Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 5 + docs/architecture/playback-protocol-v3.md | 53 ++- .../fixtures/valid/capability_response.json | 1 + .../v3/fixtures/valid/decision_response.json | 1 + docs/feature-changelog.md | 5 + internal/api/handlers/playback.go | 58 +-- internal/api/handlers/playback_v3.go | 288 ++++++++++++--- .../api/handlers/playback_v3_origins_test.go | 340 ++++++++++++++++++ internal/api/handlers/playback_v3_test.go | 70 ++-- .../handlers/playback_v3_tokenless_test.go | 28 +- .../api/handlers/playback_v3_union_test.go | 2 +- internal/api/router.go | 6 + internal/noderecipe/store.go | 61 +++- internal/noderecipe/store_test.go | 40 ++- .../attempt_sticky_features_v3_test.go | 21 +- internal/playback/protocol_v3.go | 37 +- internal/playback/protocol_v3_test.go | 1 + internal/playback/recipecard.go | 15 +- .../protocol_v3/capability_response.json | 1 + .../protocol_v3/conformance_matrix.json | 1 + .../protocol_v3/decision_response.json | 1 + internal/proxy/mediagrant.go | 188 ++++++++++ internal/proxy/mediagrant_test.go | 248 +++++++++++++ internal/proxy/server.go | 55 ++- 24 files changed, 1345 insertions(+), 181 deletions(-) create mode 100644 internal/api/handlers/playback_v3_origins_test.go create mode 100644 internal/proxy/mediagrant.go create mode 100644 internal/proxy/mediagrant_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index bc32ccaa9..fbec634c1 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -718,6 +718,11 @@ func main() { var handler http.Handler if mode == "proxy" { srv := proxy.NewServer(watcher, tracker) + // Serve header-authenticated sessions: the recipe comes from the + // shared grant store central wrote at plan time, and the caller's + // own access token is re-checked against the live login session in + // Postgres, so a revoked login stops streaming here immediately. + srv.SetMediaGrantAuthority(noderecipe.NewProxyGrantStore(redisClient, 0), auth.NewSessionRepository(pool)) srv.SetRemoteArtifactMissReporter(downloads.NewArtifactManager( downloads.NewArtifactRepository(pool), downloads.NewRepository(pool), diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index b30e5ff7b..36aaffdc1 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -98,13 +98,14 @@ the document is always the full one: "protocol_versions": [3], "features": ["playback_plan_v3", "neutral_playback_v3_contract_v1", "layout_aware_passthrough", "playback_route_diagnostics", "device_quirks_v1", "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", - "header_authenticated_media_v1", "software_video_decode_v1", "plan_source_duration_v1"], + "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_source_duration_v1"], "deliveries": ["original_http", "server_remux_progressive", "server_remux_hls", "server_transcode_hls"], "transformations": [{"name": "audio_to_aac", "executor": "server", "recipe_version": "1", "validated_claims": ["audio_decode"]}] } ``` -The eleven feature strings above are the full set this server version advertises: +The twelve feature strings above are the full set this server version advertises: | Feature | What it promises | | --- | --- | @@ -116,7 +117,8 @@ The eleven feature strings above are the full set this server version advertises | `seek_reanchor_v1` | The `seek_reanchor` replan operation is available (§6) | | `output_change_v1` | The `output_change` intent replan is available; clients must keep the active route when this feature is absent | | `direct_stream_resume_v1` | A direct route may resume mid-file rather than restarting | -| `header_authenticated_media_v1` | An opted-in client receives only API-local media URLs without signed credentials in their query or path, and authenticates every media request with its normal Authorization header (§4.1) | +| `header_authenticated_media_v1` | An opted-in client receives media URLs without signed credentials in their query or path, and authenticates every media request with its normal Authorization header (§4.1) | +| `authorized_media_origins_v1` | Meaningful only with the token above: the client also honors credential-free absolute media URLs on server-designated proxy origins, which restores distributed egress for a header-authenticated attempt (§4.1) | | `software_video_decode_v1` | Exact/platform-attested clients may qualify bounded `video_decode[]` entries with `hardware: false` for direct/original delivery; without the opt-in those evidence tiers remain hardware-only (§3) | | `plan_source_duration_v1` | `source.duration_seconds` is populated when known, so its absence means *unknown* rather than *unsupported* (§5) | @@ -474,8 +476,12 @@ re-request headers from `header_refresh_url` rather than restarting playback. `header_authenticated_media_v1` is an engine-neutral client opt-in. A client uses it only after the server advertises the same token, then includes it in the -top-level `client_features` on start and replan requests. For that attempt the -server returns only relative URLs on the authenticated API origin: +top-level `client_features` on start and replan requests. It negotiates *how* +media URLs authenticate; `authorized_media_origins_v1` (below) separately +negotiates *which origins* may serve them. + +With `header_authenticated_media_v1` alone, the server returns only relative +URLs on the authenticated API origin: - direct and progressive remux: `/stream/{session_id}` (an ordinary `seek` parameter may still be present); @@ -498,6 +504,31 @@ behind the API. A client that advertises no HLS delivery gets the non-retryable terminal `local_transcode_disabled` rather than a retryable capacity error it could only retry forever. +**Authorized media origins.** A client that also sends +`authorized_media_origins_v1` promises something further: it will fetch media +from absolute URLs the plan returns on origins the server designates, attaching +the same `Authorization` header it sends the API. For such an attempt a plan may +return a proxy origin instead of a relative path: + +- direct and progressive remux: `{proxy}/stream/v3/{session_id}` (again with an + ordinary `seek` parameter when non-zero); +- remux/transcode HLS: `{proxy}/stream/v3/{session_id}/master.m3u8`, whose + segment URIs stay relative and therefore resolve inside the same family. + +Those URLs still carry no credential of any kind — no `st`, no token path +segment, no query parameter. The proxy is told what to serve out of band, and +authenticates the caller itself: it validates the same access token against the +same live login session the API checks, so revoking a session stops proxy +playback immediately, exactly as it stops API playback. A server with no proxy +pool, or one that cannot record the handoff, simply keeps the attempt on the API +origin — the URLs above are an addition a plan may make, never one a client may +assume. The escalation described just above therefore applies only when no proxy +origin is available to run the remux. + +Only media moves. Start, replan, route events, progress and every other +control-plane call stay on the API origin, and the attempt's plan remains the +sole authority for which URL to fetch. + The client must attach its current `Authorization: Bearer ...` header to the manifest/file request and every derived request, including HLS segments, subtitle artifacts and font bundles. `stream.headers` deliberately does not @@ -517,14 +548,15 @@ credential-bearing URL. ### 4.2 Media and subtitle URL query parameters -Every URL a plan publishes belongs to one of two route families, and the query -parameters each family accepts are part of the contract. A client replays the +Every URL a plan publishes belongs to one of the route families below, and the +query parameters each family accepts are part of the contract. A client replays the URL it was handed byte-for-byte; it never composes one, never drops a parameter, and never carries a parameter across families. | Route family | Routes | Query parameters | | --- | --- | --- | | Media | `/stream/{session_id}`, `/playback/transcode/{session_id}/master.m3u8` and its segments | `seek` only — the progressive-remux start offset in seconds, present only when it is non-zero | +| Media on a designated origin | `{proxy}/stream/v3/{session_id}`, `{proxy}/stream/v3/{session_id}/master.m3u8` and its `segment/{name}` children (§4.1) | `seek` only, with the same meaning; these routes never accept a credential parameter of any kind | | Subtitle artifact | `/stream/{session_id}/subtitles/{combined_index}{.ext}`, `/stream/{session_id}/subtitles/{combined_index}/fonts` | `file_id`, always; plus `downloaded_subtitle_id` when the track is a downloaded or AI-generated one (§8) | A media route never carries `file_id` or `downloaded_subtitle_id` — the session @@ -674,18 +706,19 @@ seek is not an authority boundary for replacing the client's declared abilities mid-session. **Attempt-sticky features.** `client_features` is otherwise refreshed by any -replan that sends it, but two entries are fixed by the start negotiation and a +replan that sends it, but three entries are fixed by the start negotiation and a replan can neither add nor drop them: | Feature | Why it is fixed | | --- | --- | | `header_authenticated_media_v1` | It selects the media security contract. A signed URL from an earlier plan stays usable until its recipe expires, so a mid-attempt switch would leave two contracts alive for one session (§4.1) | +| `authorized_media_origins_v1` | It selects which origins may serve the attempt's media. A plan that already handed out a proxy origin outlives the replan that would revoke it, so the client would be left holding a URL it no longer trusts (§4.1) | | `software_video_decode_v1` | It widens the direct-play evidence tiers. Dropping it converts a direct route into a transcode and persists that downgrade into the durable request | -The server silently restores the negotiated state of both, whatever the replan +The server silently restores the negotiated state of each, whatever the replan sends — including an explicit list that omits one, which is otherwise a valid way to drop a feature. Seek replans never replace the feature list at all. -Changing either mode means stopping and starting a new attempt. +Changing any of these modes means stopping and starting a new attempt. --- diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json index ba8d3fdca..230cd0820 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json @@ -13,6 +13,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "authorized_media_origins_v1", "software_video_decode_v1", "plan_source_duration_v1" ], diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json index c70258f4c..b194163f6 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json @@ -10,6 +10,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "authorized_media_origins_v1", "software_video_decode_v1", "plan_source_duration_v1" ], diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index 71accbace..c3d45137f 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -1,5 +1,10 @@ # Feature Changelog +## 2026-08-23 + +### Serve tokenless playback from proxy nodes again +Playback protocol v3 now advertises the engine-neutral `authorized_media_origins_v1` opt-in, which a client sends together with `header_authenticated_media_v1`. Plans for such an attempt may return absolute, still credential-free media URLs on server-designated proxy origins (`/stream/v3/...`), so direct play, progressive remux, and HLS egress from the node pool instead of the API server. The proxy validates the caller's own access token against the same live login session the API checks, so revoking a session stops proxy playback immediately; replans and every other control-plane call stay on the API. A client that sends only `header_authenticated_media_v1` keeps today's API-local behavior unchanged, and so does a deployment with no proxy pool. + ## 2026-08-22 ### Qualify bounded software video decoders without weakening evidence tiers diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 26c7358ba..1a25903d5 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -157,28 +157,32 @@ type PlaybackHandler struct { SessionSyncer PlaybackSessionSyncer // optional; enables immediate session sync to shared admin view EventsHub *evt.Hub MissingMarker MissingFileMarker - NodePlanner nodepool.SessionPlanner // optional; enables proxy/transcode node selection - JWTSecret string // needed for signing stream tokens - ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks - EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series - ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item - OriginalLangLookup PlaybackOriginalLanguageLookup - SettingsRepo PlaybackSettingsReader // optional; reads server settings (e.g., allow_4k_transcode) - FileVersionFetcher PlaybackFileVersionFetcher // optional; queries sibling file versions for 4K guard - ProbeEnsurer PlaybackProbeEnsurer // optional; repairs missing probe metadata on demand - ChapterThumbnailQueuer PlaybackChapterThumbnailQueuer - IntroAnalyzer IntroEpisodeAnalyzer - IntroRepository PlaybackIntroEligibilityChecker - MarkerRegistry *markers.Registry - MarkerResolver markers.ExternalIDResolver - MarkerUpserter PlaybackMarkerUpserter - MarkerUpdateNotifier PlaybackMarkerUpdateNotifier - MarkerLazyContext context.Context - MarkerLazyInFlight sync.Map - SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles in playback - RealtimeHub *playback.RealtimeHub - CommandTracker *playback.CommandTracker - CommandDispatcher *playback.CommandDispatcher + NodePlanner nodepool.SessionPlanner // optional; enables proxy/transcode node selection + JWTSecret string // needed for signing stream tokens + // ProxyGrantStore hands a proxy the recipe it serves a header-authenticated + // session from. Optional: without it (or without Redis behind it) an attempt + // that negotiated authorized_media_origins_v1 simply stays on the API origin. + ProxyGrantStore proxyGrantStoreV3 + ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks + EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series + ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item + OriginalLangLookup PlaybackOriginalLanguageLookup + SettingsRepo PlaybackSettingsReader // optional; reads server settings (e.g., allow_4k_transcode) + FileVersionFetcher PlaybackFileVersionFetcher // optional; queries sibling file versions for 4K guard + ProbeEnsurer PlaybackProbeEnsurer // optional; repairs missing probe metadata on demand + ChapterThumbnailQueuer PlaybackChapterThumbnailQueuer + IntroAnalyzer IntroEpisodeAnalyzer + IntroRepository PlaybackIntroEligibilityChecker + MarkerRegistry *markers.Registry + MarkerResolver markers.ExternalIDResolver + MarkerUpserter PlaybackMarkerUpserter + MarkerUpdateNotifier PlaybackMarkerUpdateNotifier + MarkerLazyContext context.Context + MarkerLazyInFlight sync.Map + SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles in playback + RealtimeHub *playback.RealtimeHub + CommandTracker *playback.CommandTracker + CommandDispatcher *playback.CommandDispatcher // PlaybackConfig returns the current playback config (ffmpeg path, // hwaccel, transcode dir). Wired to the live config in integrated mode // so admin changes apply to newly started transcodes. Read it through @@ -935,6 +939,11 @@ func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *play } h.closeTranscodeForSession(session) + // A session that ends must stop egressing everywhere, not just here: the + // grant is a proxy's whole authority to serve these bytes, and unlike the + // recipe card it is never a reconstruction aid, so it is revoked on every + // stop and abort. + h.deleteProxyGrantV3(ctx, session.ID) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -969,6 +978,11 @@ func (h *PlaybackHandler) finalizeSessionAbort(ctx context.Context, session *pla // Abort is a connection drop / non-terminal teardown — keep the recipe card // so the client can reconstruct on reconnect. h.closeTranscodeForSession(session) + // A session that ends must stop egressing everywhere, not just here: the + // grant is a proxy's whole authority to serve these bytes, and unlike the + // recipe card it is never a reconstruction aid, so it is revoked on every + // stop and abort. + h.deleteProxyGrantV3(ctx, session.ID) if syncNow { h.syncSessionsNow(ctx, syncReason) } diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 8d28fce49..8c5c5e253 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -77,15 +77,48 @@ type preparedTimelineV3 struct { copySeekAnchorResolved bool } -// headerAuthenticatedMediaV3 reports whether a client's advertised feature set -// opted into the tokenless, header-authenticated media transport. +// mediaAuthModeV3 is the attempt's negotiated media transport mode: how a +// client-visible media URL authenticates, and therefore which origins may serve +// it. Both bits are resolved once from the attempt's (pinned) feature list and +// threaded down every branch rather than re-derived per URL builder. +type mediaAuthModeV3 struct { + // headerAuth is header_authenticated_media_v1: no client-visible URL + // carries a signed playback credential, and the client authenticates every + // media request with its own access token instead. + headerAuth bool + // proxyEgress is authorized_media_origins_v1 negotiated on top of + // headerAuth: the client also honors credential-free absolute URLs on + // server-designated proxy origins, so media bytes need not all egress from + // the API server. Never true without headerAuth — on a legacy attempt the + // signed proxy URL already carries its own authority. + proxyEgress bool +} + +// headerAuthenticatedMediaV3 resolves the negotiated media transport mode from +// a client's advertised feature set. // -// The negotiated mode is a bounded boolean threaded from the v3 request decoder -// down through transport preparation and into the session's stream state, the -// same way local egress is. It deliberately carries no credential, and the -// durable normalized request stays the source of truth for the attempt. -func headerAuthenticatedMediaV3(clientFeatures []string) bool { - return playback.HasFeatureV3(clientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) +// The mode is a bounded value threaded from the v3 request decoder down through +// transport preparation and into the session's stream state, the same way local +// egress is. It deliberately carries no credential, and the durable normalized +// request stays the source of truth for the attempt. +func headerAuthenticatedMediaV3(clientFeatures []string) mediaAuthModeV3 { + headerAuth := playback.HasFeatureV3(clientFeatures, playback.FeatureHeaderAuthenticatedMediaV3) + return mediaAuthModeV3{ + headerAuth: headerAuth, + proxyEgress: headerAuth && playback.HasFeatureV3(clientFeatures, playback.FeatureAuthorizedMediaOriginsV3), + } +} + +// proxyGrantStoreV3 hands a media-authorized session's recipe to the proxy that +// will serve it. The grant replaces the signed URL token as the proxy's +// instruction set; the proxy still authenticates the caller itself. +type proxyGrantStoreV3 interface { + // Enabled reports whether the store can actually carry a grant. A disabled + // store accepts Put silently, so a URL that only a stored grant can serve + // must not be published without checking it. + Enabled() bool + Put(ctx context.Context, sessionID string, card playback.RecipeCard) error + Delete(ctx context.Context, sessionID string) error } type transportErrorV3 struct { @@ -215,6 +248,19 @@ type proxyNodeEnumeratorV3 interface { ProxyNodeURLs() []string } +// proxyEgressOriginsAvailableV3 reports whether this deployment has any proxy +// origin an authorized-origins attempt could be sent to. A planner that cannot +// enumerate proxies counts as none: the escalation this gates exists precisely +// for the case where identity work has no executor, and assuming an origin the +// server cannot name would leave the attempt with nowhere to run. +func (h *PlaybackHandler) proxyEgressOriginsAvailableV3() bool { + if h == nil || h.NodePlanner == nil { + return false + } + enumerator, ok := h.NodePlanner.(proxyNodeEnumeratorV3) + return ok && len(enumerator.ProxyNodeURLs()) > 0 +} + // hlsPlanningRegistryV3 returns the registry HLS deliveries plan against: the // local probe plus every pooled transcode node's advertised transformations. // Only availability of locally-defined specs widens (name and recipe version @@ -674,7 +720,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr if result.Plan == nil { return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "The server produced no playback plan."} } - headerAuth := headerAuthenticatedMediaV3(req.ClientFeatures) + mode := headerAuthenticatedMediaV3(req.ClientFeatures) if checker, ok := h.sessionMgr.(transcodePermissionChecker); ok && (result.PlayMethod == playback.PlayTranscode || result.TranscodeAudio) { if err := checker.CheckTranscodingAllowed(r.Context(), userID, result.PlayMethod == playback.PlayTranscode); err != nil { reason := "transcoding_disabled" @@ -720,7 +766,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr abort() return playback.DecisionResponseV3{}, subtitleArtifactErrorV3("Failed to freeze the selected subtitle identity.", frozenErr) } - transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result, headerAuth) + transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result, mode) if transportErr != nil { abort() return playback.DecisionResponseV3{}, transportErr @@ -733,7 +779,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr } response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: playback.ServerFeaturesV3(), Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan} record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, FrozenRecipe: frozenRecipe, NormalizedRequest: req, StartResponse: response, RequestDigest: requestDigests.current, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)} - if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport, headerAuth); err != nil { + if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport, mode); err != nil { transport.rollback() abort() return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to commit the live playback session.", cause: err} @@ -798,26 +844,30 @@ func (h *PlaybackHandler) persistSeriesSelectionsV3(ctx context.Context, userID h.persistAudioPreference(ctx, userID, profileID, file, audioTrackIndex) } -// prepareTransportV3 resolves the plan into a live transport. headerAuth is the +// prepareTransportV3 resolves the plan into a live transport. mode is the // attempt's negotiated media-auth mode, resolved once by the caller and threaded // down every branch (like localEgress) rather than re-derived per URL builder. -func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, mode mediaAuthModeV3) (preparedTransportV3, *transportErrorV3) { timeline, timelineErr := h.prepareTransportTimelineV3(r.Context(), session, file, result) if timelineErr != nil { return preparedTransportV3{}, timelineErr } if result.Plan.Delivery != playback.DeliveryTranscodeHLSV3 && result.Plan.Delivery != playback.DeliveryRemuxHLSV3 { - return h.prepareIdentityTransportV3(r, session, file, result, timeline, headerAuth) + return h.prepareIdentityTransportV3(r, session, file, result, timeline, mode) } if h.NodePlanner != nil { - plan := h.planNodeSessionV3(r.Context(), session, result, headerAuth) + // Local egress is the header-authenticated mode WITHOUT authorized + // origins: the API is then the only client-facing media origin, so a + // proxy must not be selected at all. With authorized origins the normal + // proxy+transcode pairing applies again. + plan := h.planNodeSessionV3(r.Context(), session, result, mode.headerAuth && !mode.proxyEgress) if plan.TranscodeNode != nil { transformations, err := h.remoteTransformationsV3(r.Context(), plan.TranscodeNode.URL) if err == nil { err = validateAdvertisedTransformationsV3(result.Plan, transformations) } if err == nil { - transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan, timeline, headerAuth) + transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan, timeline, mode) if transportErr != nil { if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { releaser.ReleaseSession(session.ID) @@ -847,7 +897,7 @@ func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback. return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_capability_unavailable", message: "No available transcode executor can run the selected playback recipe.", retryable: true, cause: err} } } - return h.prepareLocalTransportV3(r, session, file, result, timeline, headerAuth) + return h.prepareLocalTransportV3(r, session, file, result, timeline, mode) } func (h *PlaybackHandler) prepareTransportTimelineV3(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTimelineV3, *transportErrorV3) { @@ -912,12 +962,12 @@ func planRequiresServerTransformationsV3(plan *playback.PlanV3) bool { return false } -func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, mode mediaAuthModeV3) (preparedTransportV3, *transportErrorV3) { routeSession := *session // The URL builders below refuse to mint a stream token for a session that // requires media authorization. The live session only learns the mode when // its stream state is committed, so stamp the route copy the builders see. - routeSession.RequireMediaAuthorization = headerAuth + routeSession.RequireMediaAuthorization = mode.headerAuth routeSession.PlayMethod = result.PlayMethod routeSession.BasePlayMethod = result.PlayMethod routeSession.MediaFileID = result.Plan.EffectiveMediaFileID @@ -929,10 +979,11 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p routeSession.RemuxDVMode = remuxDVModeForPlanV3(result.Plan) var proxyNode *nodepool.Node - if headerAuth { - // Proxy identity routes authenticate with a signed token in the URL path. - // Keep this negotiated mode on the authenticated API origin instead, so - // no client-visible URL can carry or disclose that credential. + if mode.headerAuth && !mode.proxyEgress { + // The legacy proxy identity routes authenticate with a signed token in + // the URL path. Without authorized origins this mode keeps everything on + // the authenticated API origin instead, so no client-visible URL can + // carry or disclose that credential. // // A remux that must run ffmpeg here has already been escalated onto an // HLS delivery (or refused outright) before the session started; this @@ -941,16 +992,21 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p return preparedTransportV3{}, localErr } } else { + // Legacy and authorized-origin attempts plan a proxy identically; only + // the URL they publish for it differs (signed token path vs. grant). var proxyErr *transportErrorV3 - proxyNode, proxyErr = h.planIdentityProxyV3(r, session.ID, result) + proxyNode, proxyErr = h.planIdentityProxyV3(r, session.ID, result, mode) if proxyErr != nil { return preparedTransportV3{}, proxyErr } } streamURL := fmt.Sprintf("/stream/%s", routeSession.ID) servedByProxy := false - if !headerAuth { + switch { + case !mode.headerAuth: streamURL, servedByProxy = h.identityStreamURLV3(&routeSession, file, proxyNode) + case mode.proxyEgress: + streamURL, servedByProxy = h.identityGrantStreamURLV3(r.Context(), &routeSession, file, proxyNode) } releaseProxyReservation := func() { if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { @@ -959,8 +1015,15 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p } if proxyNode != nil && !servedByProxy { // A planned proxy that could not be addressed (no signable token, no - // file record) falls back to the local path, so its reservation must be - // dropped now rather than pinning that node's budget until it ages out. + // file record, no writable grant) falls back to the local path, so its + // reservation must be dropped now rather than pinning that node's budget + // until it ages out. + // + // The fallback is local execution, so it honors the same + // local-fallback gate the no-origins mode enforces: an authorized-origins + // remux whose grant could not be written must not quietly spawn the + // ffmpeg the operator disabled. Start-time escalation cannot cover this + // case — it was legitimately skipped because the pool does offer a proxy. releaseProxyReservation() if err := h.refuseLocalIdentityWorkV3(r, result); err != nil { return preparedTransportV3{}, err @@ -997,15 +1060,80 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p committed = true // The session never reached the client, so a proxy admitted for it // must not keep consuming that node's job/bandwidth budget until the - // reservation ages out. + // reservation ages out — nor keep an egress grant for a transport + // that was never committed. if servedByProxy { releaseProxyReservation() + h.deleteProxyGrantV3(r.Context(), session.ID) } unlock() }, }, nil } +// identityGrantStreamURLV3 builds the stream URL for a direct-play or +// progressive-remux session that negotiated authorized media origins: an +// absolute, credential-free proxy URL backed by a server-side grant, otherwise +// the API-local path. +// +// The proxy serves from the grant alone, so the grant has to carry everything +// the API-local path would have read from the session and the file record — the +// media path it opens, and the source facts (Dolby Vision profile, audio-only) +// its remux needs. Omitting either would not fail loudly: the proxy would serve +// a subtly different stream than the plan promised. +// +// The bool reports whether the returned URL is actually a proxy URL, so the +// caller can release the planner reservation when it is not. A grant that +// cannot be written is not fatal: this attempt simply stays on the API origin, +// which is exactly the behavior of a header-authenticated attempt that +// negotiated no origins at all. +func (h *PlaybackHandler) identityGrantStreamURLV3(ctx context.Context, s *playback.Session, file *models.MediaFile, proxyNode *nodepool.Node) (string, bool) { + if proxyNode == nil || file == nil || s == nil { + return h.playbackStreamURL(s), false + } + card := identityRecipeCard(s) + card.InputPath = file.FilePath + card.DVProfile = file.PrimaryDVProfile() + card.AudioOnly = file.IsAudioOnly() + if !h.putProxyGrantV3(ctx, s.ID, card) { + return h.playbackStreamURL(s), false + } + return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + s.ID, true +} + +// putProxyGrantV3 stores the recipe a designated proxy origin serves this +// session from, reporting whether the grant is actually retrievable. A replan +// overwrites the previous grant under the same session id. +// +// A disabled store is a negative answer rather than a silent success: it +// accepts writes it cannot retrieve (the Redis-less integrated box), and +// publishing a proxy URL against one would hand the client a route that 404s. +func (h *PlaybackHandler) putProxyGrantV3(ctx context.Context, sessionID string, card playback.RecipeCard) bool { + if h.ProxyGrantStore == nil || !h.ProxyGrantStore.Enabled() || sessionID == "" { + return false + } + if err := h.ProxyGrantStore.Put(ctx, sessionID, card); err != nil { + slog.WarnContext(ctx, "protocol v3 proxy egress grant write failed; serving from the API origin", + "component", "api", "playback_session_id", sessionID, "error", err) + return false + } + return true +} + +// deleteProxyGrantV3 revokes a session's proxy egress authority. It runs +// wherever the session ends or its transport fails to commit: a grant that +// outlived its session would let a proxy keep serving bytes for playback the +// server considers over. +func (h *PlaybackHandler) deleteProxyGrantV3(ctx context.Context, sessionID string) { + if h == nil || h.ProxyGrantStore == nil || sessionID == "" { + return + } + if err := h.ProxyGrantStore.Delete(context.WithoutCancel(ctx), sessionID); err != nil { + slog.WarnContext(ctx, "failed to revoke proxy egress grant", + "component", "api", "playback_session_id", sessionID, "error", err) + } +} + // planIdentityProxyV3 selects the proxy node that will serve a direct-play or // progressive-remux session. These deliveries need no transcode node — the // bytes are either the source file or a single remux pipe — so the planner is @@ -1017,8 +1145,12 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p // exception is a remux that must run ffmpeg: that is transcode work, so it // honors the same local-fallback gate as the HLS routes rather than quietly // spawning an encoder on an API-only node. -func (h *PlaybackHandler) planIdentityProxyV3(r *http.Request, sessionID string, result playback.PlannerResultV3) (*nodepool.Node, *transportErrorV3) { - if h.NodePlanner == nil || h.JWTSecret == "" { +func (h *PlaybackHandler) planIdentityProxyV3(r *http.Request, sessionID string, result playback.PlannerResultV3, mode mediaAuthModeV3) (*nodepool.Node, *transportErrorV3) { + // A legacy attempt addresses its proxy with a signed token, so an unset + // signing secret rules the whole pool out. An authorized-origin attempt + // addresses it by session id against a server-side grant and needs no + // signing secret of its own. + if h.NodePlanner == nil || (h.JWTSecret == "" && !mode.proxyEgress) { return nil, h.refuseLocalIdentityWorkV3(r, result) } // Reserve against the session id the rest of the transport uses, so a @@ -1122,10 +1254,10 @@ func (h *PlaybackHandler) plannerInputV3(ctx context.Context, req playback.Start // escalateRefusedProgressiveRemuxV3 replaces a progressive remux that the // header-authenticated transport is guaranteed to refuse. // -// That mode bypasses the proxy identity routes (a proxy authenticates from the -// signed URL token this mode exists to remove), so a remux carrying server -// transformations is ffmpeg work with nowhere to run once -// playback.local_transcode_fallback is off — refuseLocalIdentityWorkV3 turns it +// Without authorized media origins that mode bypasses the proxy identity routes +// (a proxy authenticates from the signed URL token this mode exists to remove), +// so a remux carrying server transformations is ffmpeg work with nowhere to run +// once playback.local_transcode_fallback is off — refuseLocalIdentityWorkV3 turns it // into a retryable capacity_unavailable that nothing will ever satisfy. HLS is // the same recipe on a delivery the API can relay from a pooled transcode node, // so plan it here rather than making the client discover the refusal and @@ -1135,11 +1267,18 @@ func (h *PlaybackHandler) plannerInputV3(ctx context.Context, req playback.Start // a non-retryable error naming the policy, because retrying is exactly what it // must not do. // +// An attempt that negotiated authorized media origins has an executor again — +// a proxy runs the remux from its grant, exactly as it does for a legacy +// attempt — so nothing is escalated while the pool actually offers a proxy. +// With origins negotiated but no proxy configured the refusal is back, and so +// is this escalation. +// // plannerInput is evaluated only on the escalation path: rebuilding it costs a // settings resolution and a downloaded-subtitle listing, which the overwhelming // majority of starts must not pay for a route they never take. -func (h *PlaybackHandler) escalateRefusedProgressiveRemuxV3(ctx context.Context, headerAuth bool, plannerInput func() playback.PlannerInputV3, result playback.PlannerResultV3) (playback.PlannerResultV3, *transportErrorV3) { - if !headerAuth || result.Terminal != nil || result.Plan == nil || +func (h *PlaybackHandler) escalateRefusedProgressiveRemuxV3(ctx context.Context, mode mediaAuthModeV3, plannerInput func() playback.PlannerInputV3, result playback.PlannerResultV3) (playback.PlannerResultV3, *transportErrorV3) { + if !mode.headerAuth || (mode.proxyEgress && h.proxyEgressOriginsAvailableV3()) || + result.Terminal != nil || result.Plan == nil || result.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 || !planRequiresServerTransformationsV3(result.Plan) || nodepool.LocalTranscodeFallbackAllowed(ctx, h.SettingsRepo) { @@ -1370,7 +1509,7 @@ func appendPlaybackQueryV3(rawURL, key, value string) string { return rawURL + separator + key + "=" + value } -func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3, mode mediaAuthModeV3) (preparedTransportV3, *transportErrorV3) { cfg := h.playbackConfig() if err := os.MkdirAll(cfg.TranscodeDir, 0o755); err != nil { return preparedTransportV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to prepare the transcode directory.", cause: err} @@ -1425,9 +1564,9 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play } } url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) - if !headerAuth { + if !mode.headerAuth { card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) - url = appendStreamToken(url, h.signSessionToken(card, headerAuth)) + url = appendStreamToken(url, h.signSessionToken(card, mode.headerAuth)) } committed := false previousNodeURL := session.TranscodeNodeURL @@ -1474,7 +1613,7 @@ func manifestStartupTransportErrorV3(running bool, cause error) *transportErrorV return &transportErrorV3{reason: transcodeStartFailedReasonV3, message: message, retryable: running, cause: cause} } -func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan, timeline preparedTimelineV3, headerAuth bool) (preparedTransportV3, *transportErrorV3) { +func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan, timeline preparedTimelineV3, mode mediaAuthModeV3) (preparedTransportV3, *transportErrorV3) { node := nodePlan.TranscodeNode transportID := transportGenerationV3(session.ID, result.Plan.PlanID) videoCodec := result.TargetVideoCodec @@ -1497,15 +1636,20 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "The selected transcode node rejected the playback transport.", retryable: true} } url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) - if !headerAuth { - hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) - card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) - url = h.buildProxyManifestURL(card, nodePlan.ProxyNode, headerAuth) - } - // buildProxyManifestURL only returns an absolute proxy URL when a proxy was - // planned and the token could be signed; otherwise the client fetches the - // manifest from this server and the local liveness path applies. - servedByProxy := !headerAuth && nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") + // Either URL builder only returns an absolute proxy URL when a proxy was + // planned and its authority (a signed token, or a stored grant) could + // actually be established; otherwise the client fetches the manifest from + // this server and the local liveness path applies. + servedByProxy := false + switch { + case !mode.headerAuth: + card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) + url = h.buildProxyManifestURL(card, nodePlan.ProxyNode, mode.headerAuth) + servedByProxy = nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") + case mode.proxyEgress: + card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) + url, servedByProxy = h.grantManifestURLV3(r.Context(), card, nodePlan.ProxyNode) + } committed := false previousNodeURL := session.TranscodeNodeURL previousTransportID := remoteTransportID(session) @@ -1533,10 +1677,40 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { releaser.ReleaseSession(session.ID) } + // An egress grant written for a transport that never committed would + // point a proxy at a transcode that no longer exists. + if servedByProxy { + h.deleteProxyGrantV3(r.Context(), session.ID) + } unlock() }}, nil } +// remoteTranscodeRecipeCardV3 captures the byte-affecting recipe of a started +// remote transcode. It is what a proxy relays from (grant) or a client carries +// (signed token), and what a restarted node reconstructs from, so it must +// reflect the parameters the node accepted rather than the ones requested: the +// node reports the hardware acceleration it actually used. +func remoteTranscodeRecipeCardV3(session *playback.Session, file *models.MediaFile, nodeURL, transportID string, req transcodenode.TranscodeStartRequest, nodeResp transcodenode.TranscodeStartResponse) playback.RecipeCard { + hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) + return playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, nodeURL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) +} + +// grantManifestURLV3 is buildProxyManifestURL's authorized-origins sibling: it +// stores the session's transcode recipe as a proxy grant and returns the +// credential-free manifest URL on that origin. Segment URIs stay relative to +// the manifest, so the same /stream/v3/{session_id}/... family serves both. +// +// Without a planned proxy — or when the grant cannot be stored — the client +// fetches the manifest from this server, which relays the same node. +func (h *PlaybackHandler) grantManifestURLV3(ctx context.Context, card playback.RecipeCard, proxyNode *nodepool.Node) (string, bool) { + localURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", card.SessionID) + if proxyNode == nil || !h.putProxyGrantV3(ctx, card.SessionID, card) { + return localURL, false + } + return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + card.SessionID + "/master.m3u8", true +} + func sourceExecutionMetadataV3(file *models.MediaFile, result playback.PlannerResultV3) playback.SourceExecutionMetadataV3 { if result.FrozenSourceMetadata != nil { return *result.FrozenSourceMetadata @@ -1560,7 +1734,7 @@ func sourceVideoTranscodeFactsV3(file *models.MediaFile, result playback.Planner return profile, bitDepth } -func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, headerAuth bool) playback.SessionStreamState { +func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, mode mediaAuthModeV3) playback.SessionStreamState { state := playback.SessionStreamState{ PlayMethod: result.PlayMethod, BasePlayMethod: result.PlayMethod, @@ -1570,7 +1744,7 @@ func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *pla TranscodeNodeURL: transport.nodeURL, TranscodeTransportID: transport.transportID, TranscodeRouteSet: true, - RequireMediaAuthorization: headerAuth, + RequireMediaAuthorization: mode.headerAuth, MediaAuthorizationSet: true, ClientIP: clientip.FromContext(ctx), ClientName: session.ClientName, @@ -1597,8 +1771,8 @@ func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *pla return state } -func (h *PlaybackHandler) updateV3SessionState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, headerAuth bool) error { - return h.sessionMgr.UpdateStreamState(session.ID, h.v3SessionStreamState(ctx, session, file, result, transport, headerAuth)) +func (h *PlaybackHandler) updateV3SessionState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3, mode mediaAuthModeV3) error { + return h.sessionMgr.UpdateStreamState(session.ID, h.v3SessionStreamState(ctx, session, file, result, transport, mode)) } func plannedAudioTrackIndexV3(result playback.PlannerResultV3, fallback int) int { @@ -2249,13 +2423,13 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte // Media authentication is attempt-sticky (pinned in HandleReplanPlaybackV3), // so this mode always equals the one the attempt started under: a reused // transport cannot change the session's media security contract. - headerAuth := headerAuthenticatedMediaV3(start.ClientFeatures) + mode := headerAuthenticatedMediaV3(start.ClientFeatures) if !seekReanchor { // A freshly planned replan can land on the same refused progressive // remux a start would have; escalate it identically. A seek reanchor // replays the frozen recipe verbatim and must not change route identity, // so it is excluded — its route was escalated when the attempt started. - escalated, escalateErr := h.escalateRefusedProgressiveRemuxV3(r.Context(), headerAuth, + escalated, escalateErr := h.escalateRefusedProgressiveRemuxV3(r.Context(), mode, func() playback.PlannerInputV3 { return h.plannerInputV3(r.Context(), start, plannerRequestedFile, effectiveFile, audioIndex, attemptedKeys) }, result) @@ -2321,7 +2495,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte ) } else { var transportErr *transportErrorV3 - transport, transportErr = h.prepareTransportV3(r, session, effectiveFile, result, headerAuth) + transport, transportErr = h.prepareTransportV3(r, session, effectiveFile, result, mode) if transportErr != nil { return playback.DecisionResponseV3{}, *record, nil, transportErr } @@ -2370,7 +2544,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte originalRollback := transport.rollback replacement := playback.SessionReplacement{ EffectiveMediaFileID: effectiveFile.ID, - StreamState: h.v3SessionStreamState(r.Context(), session, effectiveFile, result, transport, headerAuth), + StreamState: h.v3SessionStreamState(r.Context(), session, effectiveFile, result, transport, mode), } if seekScopedRecovery { replacement.PositionSeconds = &req.PositionSeconds diff --git a/internal/api/handlers/playback_v3_origins_test.go b/internal/api/handlers/playback_v3_origins_test.go new file mode 100644 index 000000000..fe04e0048 --- /dev/null +++ b/internal/api/handlers/playback_v3_origins_test.go @@ -0,0 +1,340 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// recordingProxyGrantStoreV3 stands in for the shared Redis grant store: it +// records what a proxy would be told to serve, so a test can assert on the +// authority the URL depends on rather than only on the URL's shape. +type recordingProxyGrantStoreV3 struct { + disabled bool + putErr error + cards map[string]playback.RecipeCard + deleted []string +} + +func (s *recordingProxyGrantStoreV3) Enabled() bool { return !s.disabled } + +func (s *recordingProxyGrantStoreV3) Put(_ context.Context, sessionID string, card playback.RecipeCard) error { + if s.putErr != nil { + return s.putErr + } + if s.cards == nil { + s.cards = map[string]playback.RecipeCard{} + } + s.cards[sessionID] = card + return nil +} + +func (s *recordingProxyGrantStoreV3) Delete(_ context.Context, sessionID string) error { + s.deleted = append(s.deleted, sessionID) + return nil +} + +func authorizedOriginsModeV3() mediaAuthModeV3 { + return headerAuthenticatedMediaV3([]string{playback.FeatureHeaderAuthenticatedMediaV3, playback.FeatureAuthorizedMediaOriginsV3}) +} + +// The point of the mode: a header-authenticated attempt egresses from the pool +// again. The URL must name the proxy and carry no credential of any kind, and +// the proxy must have been handed the recipe it will serve from. +func TestPrepareTransportV3AuthorizedOriginsRestoreDirectPlayProxyEgress(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + planner := &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.NodePlanner = planner + grants := &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = grants + file := v3HandlerFixtureFile(t) + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-direct", UserID: 7, ProfileID: "profile-1"}, + file, + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + + if transport.url != "http://proxy-1/stream/v3/session-origin-direct" { + t.Fatalf("stream url = %q, want the credential-free proxy route", transport.url) + } + assertNoPlaybackCredentialV3(t, transport.url) + + card, ok := grants.cards["session-origin-direct"] + if !ok { + t.Fatal("no grant was written; the proxy has nothing to serve this session from") + } + if card.InputPath != file.FilePath { + t.Fatalf("grant media path = %q, want %q", card.InputPath, file.FilePath) + } + if card.UserID != 7 || card.PlayMethod != playback.PlayDirect { + t.Fatalf("grant identity = %#v, want the session's owner and play method", card) + } + + // A transport that never reaches the client must not leave a live grant + // behind, or the proxy keeps serving playback the server considers over. + transport.rollback() + if len(grants.deleted) != 1 || grants.deleted[0] != "session-origin-direct" { + t.Fatalf("grants deleted on rollback = %v, want the session's grant revoked", grants.deleted) + } + if len(planner.released) != 1 { + t.Fatalf("planner releases = %v, want the proxy reservation released", planner.released) + } +} + +// A remux egresses from the proxy too, and the grant has to carry the source +// facts the proxy cannot look up: without them it would serve a subtly +// different stream than the plan promised. +func TestPrepareTransportV3AuthorizedOriginsCarryRemuxSourceFacts(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + stubCopySeekAnchorV3(handler) + proxy := capableProxyStubV3(t) + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: proxy.URL + "/"}}} + grants := &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = grants + + file := v3HandlerFixtureFile(t) + file.VideoTracks[0].DVProfile = 7 + plan := identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3, playback.TransformationV3{Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}) + plan.Timeline = playback.TimelineV3{SourceStartSeconds: 39.5} + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-remux", UserID: 7, ProfileID: "profile-1"}, + file, + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TranscodeAudio: true, TargetAudioCodec: "aac"}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + defer transport.rollback() + + want := proxy.URL + "/stream/v3/session-origin-remux?seek=39.5" + if transport.url != want { + t.Fatalf("stream url = %q, want %q", transport.url, want) + } + assertNoPlaybackCredentialV3(t, transport.url) + + card := grants.cards["session-origin-remux"] + if card.DVProfile != 7 { + t.Fatalf("grant DV profile = %d, want 7 so the proxy strips the dangling RPU", card.DVProfile) + } + if !card.TranscodeAudio { + t.Fatal("grant must tell the proxy to convert audio") + } +} + +// A grant that cannot be stored is not fatal: the attempt degrades to exactly +// what a header-authenticated attempt without origins would have gotten. +func TestPrepareTransportV3AuthorizedOriginsFallBackToTheAPIWhenTheGrantFails(t *testing.T) { + for _, test := range []struct { + name string + store *recordingProxyGrantStoreV3 + }{ + {name: "write error", store: &recordingProxyGrantStoreV3{putErr: errors.New("redis is down")}}, + {name: "store disabled", store: &recordingProxyGrantStoreV3{disabled: true}}, + } { + t.Run(test.name, func(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + planner := &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.NodePlanner = planner + handler.ProxyGrantStore = test.store + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-fallback", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + defer transport.rollback() + + if transport.url != "/stream/session-origin-fallback" { + t.Fatalf("stream url = %q, want the API-local route", transport.url) + } + if len(planner.released) != 1 { + t.Fatalf("planner releases = %v, want the unusable proxy reservation released", planner.released) + } + }) + } +} + +// The grant-failure fallback lands on the API server, which is ffmpeg work for +// a remux carrying a server transformation. An operator who disabled +// playback.local_transcode_fallback disabled exactly that, so the fallback has +// to honor the same gate the no-origins mode enforces rather than quietly +// spawning an encoder. Escalation cannot cover this: it was legitimately +// skipped at plan time because the pool does offer a proxy. +func TestPrepareTransportV3AuthorizedOriginsRefuseLocalRemuxWhenTheGrantFails(t *testing.T) { + handler, _, result := escalationFixtureV3(t, true) + proxy := capableProxyStubV3(t) + planner := &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: proxy.URL}}} + handler.NodePlanner = planner + handler.ProxyGrantStore = &recordingProxyGrantStoreV3{putErr: errors.New("redis is down")} + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-refused", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + result, + authorizedOriginsModeV3()) + if transportErr == nil { + transport.rollback() + t.Fatalf("grant failure produced an API-local remux at %q; local fallback is disabled", transport.url) + } + if transportErr.reason != "capacity_unavailable" || !transportErr.retryable { + t.Fatalf("transport error = %#v, want a retryable capacity_unavailable", transportErr) + } + if len(planner.released) != 1 { + t.Fatalf("planner releases = %v, want the unusable proxy reservation released", planner.released) + } +} + +// Without the origins opt-in the mode is unchanged from what PR #723 shipped: +// everything stays on the API, and no grant is written at all. +func TestPrepareTransportV3HeaderAuthOnlyStaysOnTheAPIOrigin(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + grants := &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = grants + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-header-only", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + headerAuthenticatedMediaV3([]string{playback.FeatureHeaderAuthenticatedMediaV3})) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + defer transport.rollback() + + if transport.url != "/stream/session-header-only" { + t.Fatalf("stream url = %q, want the API-local route", transport.url) + } + if len(grants.cards) != 0 { + t.Fatalf("grants written = %v, want none for an attempt that negotiated no origins", grants.cards) + } +} + +// HLS keeps its pooled transcode node and gets its proxy back: the manifest is +// fetched from the proxy, whose relative segment URIs stay inside the same +// credential-free /stream/v3 family. +func TestPrepareTransportV3AuthorizedOriginsPublishGrantBackedHLSManifest(t *testing.T) { + var startRequest transcodenode.TranscodeStartRequest + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/hw-capabilities": + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + if err := json.NewDecoder(r.Body).Decode(&startRequest); err != nil { + t.Errorf("decode remote start: %v", err) + } + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{SessionID: startRequest.SessionID, Status: "started"}) + default: + w.WriteHeader(http.StatusNoContent) + } + })) + defer node.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + planner := &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}, ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.NodePlanner = planner + grants := &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = grants + + plan := &playback.PlanV3{ + PlanID: "plan:origin-hls", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{{Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}}, + } + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-hls", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + + if transport.url != "http://proxy-1/stream/v3/session-origin-hls/master.m3u8" { + t.Fatalf("manifest url = %q, want the credential-free proxy manifest", transport.url) + } + assertNoPlaybackCredentialV3(t, transport.url) + + card, ok := grants.cards["session-origin-hls"] + if !ok { + t.Fatal("no grant was written; the proxy cannot relay this transcode") + } + if card.TranscodeNodeURL != node.URL { + t.Fatalf("grant transcode node = %q, want %q", card.TranscodeNodeURL, node.URL) + } + if card.TranscodeTransportID != transport.transportID { + t.Fatalf("grant transport id = %q, want the plan-scoped transport %q", card.TranscodeTransportID, transport.transportID) + } +} + +// The escalation exists because header-authenticated identity work had no +// executor. Authorized origins give it one, so an attempt with a proxy +// available must keep the route the planner chose. +func TestEscalateRefusedProgressiveRemuxV3SkipsEscalationWhenOriginsHaveAProxy(t *testing.T) { + handler, input, result := escalationFixtureV3(t, true) + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + + escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) + if transportErr != nil { + t.Fatalf("escalation error = %#v", transportErr) + } + if escalated.Plan == nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + t.Fatalf("escalated delivery = %#v, want the planned progressive remux left alone", escalated.Plan) + } +} + +// With origins negotiated but no proxy in the pool the refusal is back, so the +// escalation must be too — otherwise the attempt plans a route nothing can run. +func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAnyProxyOrigin(t *testing.T) { + handler, input, result := escalationFixtureV3(t, true) + handler.NodePlanner = &recordingNodePlannerV3{} + + escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) + if transportErr != nil { + t.Fatalf("escalation error = %#v", transportErr) + } + if escalated.Plan == nil || escalated.Plan.Delivery != playback.DeliveryRemuxHLSV3 { + t.Fatalf("escalated delivery = %#v, want %q", escalated.Plan, playback.DeliveryRemuxHLSV3) + } +} + +// assertNoPlaybackCredentialV3 fails when a published URL carries any playback +// credential — the whole promise of the mode, on the proxy origin as much as on +// the API one. +func assertNoPlaybackCredentialV3(t *testing.T, rawURL string) { + t.Helper() + if strings.Contains(rawURL, streamTokenParam+"=") || strings.Contains(rawURL, "/stream/direct/") || + strings.Contains(rawURL, "/stream/remux/") || strings.Contains(rawURL, "/stream/transcode/") { + t.Fatalf("URL %q carries a playback credential", rawURL) + } +} diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 178777647..29eff8703 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -2420,7 +2420,7 @@ func TestPrepareTransportV3ProgressiveRemuxUsesResolvedCopyAnchor(t *testing.T) EffectiveMediaFileID: 42, Timeline: playback.TimelineV3{SourceStartSeconds: requested, PlayerStartSeconds: requested, CanSeekAnywhere: true, SeekRestoration: "player_position"}, } - transport, transportErr := handler.prepareTransportV3(httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, false) + transport, transportErr := handler.prepareTransportV3(httptest.NewRequest(http.MethodPost, "/", nil), session, file, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare progressive transport: %v", transportErr) } @@ -2476,9 +2476,7 @@ func TestPrepareTransportV3AudioOnlyRemuxSkipsVideoCopyAnchor(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-audio-only", MediaFileID: 42}, file, - playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, - false, - ) + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare audio-only transport: %v", transportErr) } @@ -2511,9 +2509,7 @@ func TestPrepareTransportV3CopyAnchorFailureIsRetryable(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-copy-failure"}, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}, - playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, - false, - ) + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}, mediaAuthModeV3{}) if transportErr == nil || transportErr.reason != "transcode_start_failed" || !transportErr.retryable || transportErr.cause == nil || transportErr.cause.Error() != "probe failed" { t.Fatalf("transport error = %#v, want retryable copy anchor failure", transportErr) } @@ -2546,7 +2542,7 @@ func TestPrepareTransportV3RejectsNodeMissingRequiredTransformation(t *testing.T }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" { t.Fatalf("transport error = %#v", transportErr) } @@ -2589,7 +2585,7 @@ func TestPrepareTransportV3RequiresRemoteManifestReadiness(t *testing.T) { }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-ready", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-ready", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare remote transport: %v", transportErr) } @@ -2695,9 +2691,7 @@ func TestPrepareTransportV3SendsResolvedCopyAnchorToRemoteExecutor(t *testing.T) httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-remote-copy-anchor", UserID: 7, ProfileID: "profile-1"}, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv", CodecVideo: "h264"}, - playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, - false, - ) + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare remote copy transport: %v", transportErr) } @@ -2754,7 +2748,7 @@ func TestPrepareTransportV3UsesFrozenSourceMetadataAfterProbeDrift(t *testing.T) file.Duration = 99 result := recipe.PlannerResult(plan) request := httptest.NewRequest(http.MethodPost, "/", nil) - transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-frozen-source", UserID: 7, ProfileID: "profile-1"}, file, result, false) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-frozen-source", UserID: 7, ProfileID: "profile-1"}, file, result, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare remote transport: %v", transportErr) } @@ -2810,7 +2804,7 @@ func TestPrepareLocalTransportV3ReturnsStableTerminalWhenFFmpegExitsBeforeReady( if timelineErr != nil { t.Fatalf("prepare timeline: %v", timelineErr) } - transport, transportErr := handler.prepareLocalTransportV3(request, &playback.Session{ID: "session-startup-failure", UserID: 7, ProfileID: "profile-1"}, file, result, timeline, false) + transport, transportErr := handler.prepareLocalTransportV3(request, &playback.Session{ID: "session-startup-failure", UserID: 7, ProfileID: "profile-1"}, file, result, timeline, mediaAuthModeV3{}) if transportErr == nil { transport.rollback() t.Fatal("failed ffmpeg startup returned a playable transport") @@ -3978,9 +3972,7 @@ func TestPrepareTransportV3RoutesDirectPlayThroughProxyNode(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-direct-proxy", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4029,9 +4021,7 @@ func TestPrepareTransportV3RoutesProgressiveRemuxThroughProxyNodeWithSeekAndDV(t httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-remux-proxy", UserID: 7, ProfileID: "profile-1"}, file, - playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TranscodeAudio: true, TargetAudioCodec: "aac"}, - false, - ) + playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux, TranscodeAudio: true, TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4110,9 +4100,7 @@ func TestPrepareTransportV3FallsBackLocallyWithoutEligibleProxy(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-direct-local", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4138,9 +4126,7 @@ func TestPrepareTransportV3RefusesLocalRemuxWhenFallbackDisabled(t *testing.T) { Plan: identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3, playback.TransformationV3{Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}), PlayMethod: playback.PlayRemux, TranscodeAudio: true, - }, - false, - ) + }, mediaAuthModeV3{}) if transportErr == nil || transportErr.reason != "capacity_unavailable" { t.Fatalf("transport error = %#v, want capacity_unavailable when local remux work is disabled", transportErr) } @@ -4158,9 +4144,7 @@ func TestPrepareTransportV3AllowsLocalDirectPlayWhenFallbackDisabled(t *testing. httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-direct-allowed", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("direct play refused with local fallback disabled: %#v", transportErr) } @@ -4177,9 +4161,7 @@ func TestPrepareTransportV3ReleasesProxyReservationOnRollback(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-rollback", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4232,9 +4214,7 @@ func TestPrepareTransportV3KeepsRemuxLocalWhenProxyLacksTheRecipe(t *testing.T) Plan: identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3, playback.TransformationV3{Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}), PlayMethod: playback.PlayRemux, TranscodeAudio: true, - }, - false, - ) + }, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4268,9 +4248,7 @@ func TestPrepareTransportV3DirectPlaySkipsProxyCapabilityProbe(t *testing.T) { httptest.NewRequest(http.MethodPost, "/", nil), &playback.Session{ID: "session-direct-noprobe", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4316,9 +4294,7 @@ func TestPrepareTransportV3MarksProxySessionsAsRemotelyTransported(t *testing.T) httptest.NewRequest(http.MethodPost, "/", nil), session, v3HandlerFixtureFile(t), - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %v", transportErr) } @@ -4401,9 +4377,7 @@ func TestPrepareTransportV3PrefersACapableSiblingProxy(t *testing.T) { Plan: identityProxyPlanV3(playback.DeliveryRemuxProgressiveV3, playback.TransformationV3{Name: playback.TransformationAudioToAACV3, Executor: playback.ExecutorServerV3, RecipeVersion: "1"}), PlayMethod: playback.PlayRemux, TranscodeAudio: true, - }, - false, - ) + }, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare identity transport: %#v", transportErr) } @@ -4431,9 +4405,7 @@ func TestPrepareTransportV3ClearsRemoteTransportMarkWhenServingLocally(t *testin proxied, transportErr := handler.prepareTransportV3( httptest.NewRequest(http.MethodPost, "/", nil), session, file, - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare proxy transport: %v", transportErr) } @@ -4443,9 +4415,7 @@ func TestPrepareTransportV3ClearsRemoteTransportMarkWhenServingLocally(t *testin handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{}} local, transportErr := handler.prepareTransportV3( httptest.NewRequest(http.MethodPost, "/", nil), session, file, - playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, - false, - ) + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, mediaAuthModeV3{}) if transportErr != nil { t.Fatalf("prepare local transport: %v", transportErr) } diff --git a/internal/api/handlers/playback_v3_tokenless_test.go b/internal/api/handlers/playback_v3_tokenless_test.go index 9369b0c3e..362dce880 100644 --- a/internal/api/handlers/playback_v3_tokenless_test.go +++ b/internal/api/handlers/playback_v3_tokenless_test.go @@ -110,6 +110,26 @@ func TestPlaybackURLBuildersRefuseTokensForMediaAuthorizedSessions(t *testing.T) if token := handler.signSessionToken(card, false); token == "" { t.Fatal("signer refused a legacy session with a configured secret") } + + // The authorized-origins builders publish the same proxy origin the legacy + // ones do, but address it by session id against a stored grant — so they + // must never fall back to minting the credential the mode removed. + grants := &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = grants + got, servedByProxy := handler.identityGrantStreamURLV3(context.Background(), secure, file, proxy) + if !servedByProxy || got != proxy.URL+"/stream/v3/session-secure" { + t.Fatalf("origins identity URL = %q (proxy %v), want the credential-free proxy route", got, servedByProxy) + } + assertNoPlaybackCredentialV3(t, got) + if _, ok := grants.cards["session-secure"]; !ok { + t.Fatal("origins identity URL was published without a grant behind it") + } + + got, servedByProxy = handler.grantManifestURLV3(context.Background(), card, proxy) + if !servedByProxy || got != proxy.URL+"/stream/v3/session-secure/master.m3u8" { + t.Fatalf("origins manifest URL = %q (proxy %v), want the credential-free proxy manifest", got, servedByProxy) + } + assertNoPlaybackCredentialV3(t, got) } // escalationFixtureV3 plans a progressive remux that must convert audio, which @@ -161,7 +181,7 @@ func escalationFixtureV3(t *testing.T, hlsCapable bool) (*PlaybackHandler, playb // capacity error it can only recover from with a replan round trip. func TestEscalateRefusedProgressiveRemuxV3PlansHLSForCapableClients(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) - escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, func() playback.PlannerInputV3 { return input }, result) + escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), mediaAuthModeV3{headerAuth: true}, func() playback.PlannerInputV3 { return input }, result) if transportErr != nil { t.Fatalf("escalation error = %#v", transportErr) } @@ -174,7 +194,7 @@ func TestEscalateRefusedProgressiveRemuxV3PlansHLSForCapableClients(t *testing.T // final: a retryable error would make it retry a route no retry can satisfy. func TestEscalateRefusedProgressiveRemuxV3IsTerminalForProgressiveOnlyClients(t *testing.T) { handler, input, result := escalationFixtureV3(t, false) - _, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, func() playback.PlannerInputV3 { return input }, result) + _, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), mediaAuthModeV3{headerAuth: true}, func() playback.PlannerInputV3 { return input }, result) if transportErr == nil || transportErr.reason != "local_transcode_disabled" || transportErr.retryable { t.Fatalf("transport error = %#v, want a non-retryable local_transcode_disabled", transportErr) } @@ -187,11 +207,11 @@ func TestEscalateRefusedProgressiveRemuxV3LeavesExecutableRoutesAlone(t *testing planned++ return input } - if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), false, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), mediaAuthModeV3{}, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { t.Fatalf("legacy attempt was escalated: %#v %#v", escalated.Plan, transportErr) } handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.local_transcode_fallback": "true"}} - if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), true, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { + if escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), mediaAuthModeV3{headerAuth: true}, plannerInput, result); transportErr != nil || escalated.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 { t.Fatalf("locally executable remux was escalated: %#v %#v", escalated.Plan, transportErr) } if planned != 0 { diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go index d0c5c0f5d..05120c17a 100644 --- a/internal/api/handlers/playback_v3_union_test.go +++ b/internal/api/handlers/playback_v3_union_test.go @@ -177,7 +177,7 @@ func TestPrepareTransportV3LocalFallbackRejectsUnavailableTransformations(t *tes }, } request := httptest.NewRequest(http.MethodPost, "/", nil) - _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, false) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}, mediaAuthModeV3{}) if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" || !transportErr.retryable { t.Fatalf("transport error = %#v", transportErr) } diff --git a/internal/api/router.go b/internal/api/router.go index 3f8cd5fdc..8a2a24159 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -49,6 +49,7 @@ import ( metatrakt "github.com/Silo-Server/silo-server/internal/metadata/trakt" metadatatranslation "github.com/Silo-Server/silo-server/internal/metadata/translation" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/notifications" "github.com/Silo-Server/silo-server/internal/onboarding" "github.com/Silo-Server/silo-server/internal/opslog" @@ -1005,6 +1006,11 @@ func NewRouter(deps Dependencies) chi.Router { if deps.Config != nil && deps.Config.Auth.JWTSecret != "" { playbackHandler.JWTSecret = deps.Config.Auth.JWTSecret } + // Hand proxy nodes the recipes they serve header-authenticated sessions + // from, so an attempt that negotiated authorized media origins egresses + // from the pool instead of this server. Nil-safe: without Redis the + // store reports itself disabled and every such attempt stays API-local. + playbackHandler.ProxyGrantStore = noderecipe.NewProxyGrantStore(deps.RedisClient, 0) if deps.Config != nil { playbackHandler.PlaybackConfig = func() config.PlaybackConfig { return deps.CurrentConfig().Playback diff --git a/internal/noderecipe/store.go b/internal/noderecipe/store.go index 4210d4ca9..3ab12aa6a 100644 --- a/internal/noderecipe/store.go +++ b/internal/noderecipe/store.go @@ -15,6 +15,14 @@ // by the upstream session id when it starts a remote transcode, and the // transcode node reads it on a reconstruct miss. It is // off the hot path — written once at start, read only after a node restart. +// +// The same central→node recipe handoff serves a second, independent purpose +// under its own key space: a proxy grant. When an attempt negotiates +// authorized_media_origins_v1 the plan hands the client a credential-free proxy +// URL, so the proxy has no token to serve from — central writes the session's +// recipe here at plan time and the proxy reads it after authenticating the +// caller's own login session. NewStore and NewProxyGrantStore are the two key +// spaces; everything else about them is identical. package noderecipe import ( @@ -32,29 +40,54 @@ import ( // KeyPrefix namespaces per-session recipe keys: silo:noderecipe:. const KeyPrefix = "silo:noderecipe:" +// ProxyGrantKeyPrefix namespaces per-session proxy grants: +// silo:proxygrant:. It is deliberately a separate key space +// from KeyPrefix: the two are written by different flows, consumed by different +// node roles, and a lookup in one must never resolve the other's entry. +const ProxyGrantKeyPrefix = "silo:proxygrant:" + // DefaultTTL bounds how long a stored recipe survives. It matches the stream // token lifetime (playback.MaxTokenTTL, 24h): past it no surviving token could // still drive a reconstruct, so the recipe is safe to lapse. const DefaultTTL = playback.MaxTokenTTL -// Store is the Redis-backed recipe store shared by central (writer, at remote -// transcode start) and the transcode nodes (reader, on a reconstruct miss). +// Store is the Redis-backed recipe store shared by central (writer) and the +// nodes (readers). One instance owns exactly one key prefix; see NewStore and +// NewProxyGrantStore for the two uses. type Store struct { - rdb *redis.Client - ttl time.Duration + rdb *redis.Client + prefix string + ttl time.Duration } -// NewStore wraps a Redis client. A nil client yields a disabled store whose -// writes no-op and whose reads miss, so a single integrated box (no Redis, no -// remote node) needs no special-casing. +// NewStore wraps a Redis client for the transcode-node recipe handoff. A nil +// client yields a disabled store whose writes no-op and whose reads miss, so a +// single integrated box (no Redis, no remote node) needs no special-casing. func NewStore(rdb *redis.Client, ttl time.Duration) *Store { + return newStore(rdb, KeyPrefix, ttl) +} + +// NewProxyGrantStore wraps a Redis client for the proxy-grant key space: the +// recipe a proxy serves a header-authenticated session from once it has +// authenticated the caller. Same nil-safety and TTL as NewStore. +func NewProxyGrantStore(rdb *redis.Client, ttl time.Duration) *Store { + return newStore(rdb, ProxyGrantKeyPrefix, ttl) +} + +func newStore(rdb *redis.Client, prefix string, ttl time.Duration) *Store { if ttl <= 0 { ttl = DefaultTTL } - return &Store{rdb: rdb, ttl: ttl} + return &Store{rdb: rdb, prefix: prefix, ttl: ttl} } -func key(sessionID string) string { return KeyPrefix + sessionID } +// Enabled reports whether this store can actually carry a recipe. A caller that +// hands out a URL only the stored recipe can serve must check it: a disabled +// store accepts Put silently (by design, for the Redis-less integrated box), so +// a successful write is not on its own evidence that the recipe exists. +func (s *Store) Enabled() bool { return s != nil && s.rdb != nil } + +func (s *Store) key(sessionID string) string { return s.prefix + sessionID } // Put stores the reconstruction recipe for a remote transcode session. Best // effort: a write error is returned for the caller to log, never fatal. @@ -66,7 +99,7 @@ func (s *Store) Put(ctx context.Context, sessionID string, card playback.RecipeC if err != nil { return err } - return s.rdb.Set(ctx, key(sessionID), data, s.ttl).Err() + return s.rdb.Set(ctx, s.key(sessionID), data, s.ttl).Err() } // Get returns the stored recipe for sessionID. It fails CLOSED — a miss or any @@ -76,16 +109,16 @@ func (s *Store) Get(ctx context.Context, sessionID string) (*playback.RecipeCard if s == nil || s.rdb == nil || sessionID == "" { return nil, false } - data, err := s.rdb.Get(ctx, key(sessionID)).Bytes() + data, err := s.rdb.Get(ctx, s.key(sessionID)).Bytes() if err != nil { if !errors.Is(err, redis.Nil) { - slog.WarnContext(ctx, "load node recipe failed", "component", "noderecipe", "error", err, "playback_session_id", sessionID) + slog.WarnContext(ctx, "load node recipe failed", "component", "noderecipe", "key_prefix", s.prefix, "error", err, "playback_session_id", sessionID) } return nil, false } var card playback.RecipeCard if err := json.Unmarshal(data, &card); err != nil { - slog.WarnContext(ctx, "decode node recipe failed", "component", "noderecipe", "error", err, "playback_session_id", sessionID) + slog.WarnContext(ctx, "decode node recipe failed", "component", "noderecipe", "key_prefix", s.prefix, "error", err, "playback_session_id", sessionID) return nil, false } return &card, true @@ -99,5 +132,5 @@ func (s *Store) Delete(ctx context.Context, sessionID string) error { if s == nil || s.rdb == nil || sessionID == "" { return nil } - return s.rdb.Del(ctx, key(sessionID)).Err() + return s.rdb.Del(ctx, s.key(sessionID)).Err() } diff --git a/internal/noderecipe/store_test.go b/internal/noderecipe/store_test.go index c85917df0..e18cbfc4d 100644 --- a/internal/noderecipe/store_test.go +++ b/internal/noderecipe/store_test.go @@ -51,11 +51,49 @@ func TestNilStore_DeleteNoop(t *testing.T) { } func TestKeyNamespacing(t *testing.T) { - if got := key("abc"); got != "silo:noderecipe:abc" { + if got := NewStore(nil, 0).key("abc"); got != "silo:noderecipe:abc" { t.Fatalf("key(abc) = %q, want silo:noderecipe:abc", got) } } +// The two key spaces share one implementation, so nothing but the prefix may +// distinguish them: a proxy grant must never resolve a node recipe, and the +// transcode node's reconstruct lookup must never resolve a grant. +func TestProxyGrantStoreIsolatesItsKeySpace(t *testing.T) { + grants := NewProxyGrantStore(nil, 0) + if got := grants.key("abc"); got != "silo:proxygrant:abc" { + t.Fatalf("proxy grant key(abc) = %q, want silo:proxygrant:abc", got) + } + if grants.key("abc") == NewStore(nil, 0).key("abc") { + t.Fatal("proxy grants and node recipes share a key") + } + if grants.ttl != DefaultTTL { + t.Fatalf("proxy grant ttl = %v, want %v", grants.ttl, DefaultTTL) + } +} + +// A disabled store accepts writes it cannot serve, so callers that publish a +// URL only the grant can satisfy need this distinction to stay on the API. +func TestProxyGrantStoreReportsWhetherItCanCarryAGrant(t *testing.T) { + var missing *Store + if missing.Enabled() { + t.Fatal("nil store reported itself enabled") + } + disabled := NewProxyGrantStore(nil, 0) + if disabled.Enabled() { + t.Fatal("Redis-less store reported itself enabled") + } + if err := disabled.Put(context.Background(), "sid", playback.RecipeCard{}); err != nil { + t.Fatalf("disabled proxy grant Put returned error: %v", err) + } + if _, ok := disabled.Get(context.Background(), "sid"); ok { + t.Fatal("disabled proxy grant Get returned a hit, want miss") + } + if err := disabled.Delete(context.Background(), "sid"); err != nil { + t.Fatalf("disabled proxy grant Delete returned error: %v", err) + } +} + func TestDefaultTTLMatchesTokenLifetime(t *testing.T) { if DefaultTTL != playback.MaxTokenTTL { t.Fatalf("DefaultTTL = %v, want playback.MaxTokenTTL %v", DefaultTTL, playback.MaxTokenTTL) diff --git a/internal/playback/attempt_sticky_features_v3_test.go b/internal/playback/attempt_sticky_features_v3_test.go index ac596be07..d11fc8ad5 100644 --- a/internal/playback/attempt_sticky_features_v3_test.go +++ b/internal/playback/attempt_sticky_features_v3_test.go @@ -25,10 +25,25 @@ func TestPinAttemptStickyFeaturesV3(t *testing.T) { want: []string{FeaturePlaybackPlanV3}, }, { - name: "an empty list still restores both sticky features", + name: "an empty list still restores every sticky feature", requested: []string{}, - negotiated: []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, - want: []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3}, + negotiated: []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3}, + want: []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3}, + }, + { + // The origin trust set is what the client enforces against the URLs + // it was handed; a replan that revoked it mid-attempt would leave a + // live plan pointing at an origin the client no longer accepts. + name: "a replan cannot drop the negotiated media origins", + requested: []string{FeaturePlaybackPlanV3, FeatureHeaderAuthenticatedMediaV3}, + negotiated: []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3}, + want: []string{FeaturePlaybackPlanV3, FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3}, + }, + { + name: "a replan cannot add media origins mid-attempt", + requested: []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3}, + negotiated: []string{FeatureHeaderAuthenticatedMediaV3}, + want: []string{FeatureHeaderAuthenticatedMediaV3}, }, { name: "case and padding do not smuggle a duplicate through", diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index 5f6c5bf56..b9c7bd456 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -34,14 +34,26 @@ const ( // access-token Authorization header to every media request, including HLS // manifests/segments and sidecar subtitle/font requests. FeatureHeaderAuthenticatedMediaV3 = "header_authenticated_media_v1" - PlanRecipeVersionV3 = "v3.4" - ClientDV7ToDV81V3 = "client_dv7_to_dv81" - ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" - ClientDVTransformVersionV3 = "1" - ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" - ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" - ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" - DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" + // FeatureAuthorizedMediaOriginsV3 is the client's promise to fetch media + // from the absolute URLs a plan returns on server-designated origins (proxy + // nodes), attaching its normal access-token Authorization header to those + // requests exactly as it does to the API origin. It is meaningful only + // together with header_authenticated_media_v1: on its own there is nothing + // to designate, because a legacy attempt already receives signed proxy URLs + // that authenticate themselves. + // + // Without it a header-authenticated attempt stays entirely on the API + // origin, so every byte egresses from the API server; with it the plan may + // hand out credential-free proxy origins and distributed egress is restored. + FeatureAuthorizedMediaOriginsV3 = "authorized_media_origins_v1" + PlanRecipeVersionV3 = "v3.4" + ClientDV7ToDV81V3 = "client_dv7_to_dv81" + ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" + ClientDVTransformVersionV3 = "1" + ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" + ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" + ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" + DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" ) // ServerFeaturesV3 returns the complete feature set advertised by protocol-v3 @@ -58,6 +70,7 @@ func ServerFeaturesV3() []string { FeatureOutputChangeV3, FeatureDirectStreamResumeV3, FeatureHeaderAuthenticatedMediaV3, + FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3, // Advertised so a client can tell "this server does not populate // source.duration_seconds" apart from "this server knows the runtime @@ -1128,13 +1141,17 @@ func HasFeatureV3(features []string, wanted string) bool { // - header_authenticated_media_v1 picks the media security contract. A legacy // signed URL from an earlier plan can outlive the plan that minted it, so // switching mid-attempt would leave two contracts alive for one session. +// - authorized_media_origins_v1 selects which origins may serve the attempt's +// media. The trust set a client honors must not change mid-attempt: a plan +// that already handed out a proxy origin outlives the replan that would +// revoke it, so the client would be left holding a URL it no longer trusts. // - software_video_decode_v1 widens the direct-play evidence tiers. Dropping // it on a replan silently converts a direct route into a transcode and // persists that downgrade into the durable normalized request. // -// Stop/start is the explicit boundary for changing either. +// Stop/start is the explicit boundary for changing any of them. func AttemptStickyFeaturesV3() []string { - return []string{FeatureHeaderAuthenticatedMediaV3, FeatureSoftwareVideoDecodeV3} + return []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3} } // PinAttemptStickyFeaturesV3 returns requested with every attempt-sticky diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index 2f6d7070c..8f2520697 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -31,6 +31,7 @@ func TestServerFeaturesV3ReturnsCompleteIndependentSlices(t *testing.T) { FeatureOutputChangeV3: {}, FeatureDirectStreamResumeV3: {}, FeatureHeaderAuthenticatedMediaV3: {}, + FeatureAuthorizedMediaOriginsV3: {}, FeatureSoftwareVideoDecodeV3: {}, FeaturePlanSourceDurationV3: {}, } diff --git a/internal/playback/recipecard.go b/internal/playback/recipecard.go index b6a576aae..858e36e88 100644 --- a/internal/playback/recipecard.go +++ b/internal/playback/recipecard.go @@ -46,8 +46,15 @@ type RecipeCard struct { // Encode parameters — mirror of the byte-affecting TranscodeOpts fields. // Direct cards leave them zero; remux cards use the audio targets when the // selected stream must be converted. - InputPath string `json:"input_path"` - OutputSubdir string `json:"output_subdir,omitempty"` + InputPath string `json:"input_path"` + OutputSubdir string `json:"output_subdir,omitempty"` + // DVProfile and AudioOnly are source facts the catalog owns and a remote + // executor cannot look up for itself: the remux needs the Dolby Vision + // profile to strip a dangling Profile 7 RPU, and the audio-only flag to keep + // the content type the plan promised. They ride the card so a proxy serving + // this session from a grant produces the same bytes the API would have. + DVProfile int `json:"dv_profile,omitempty"` + AudioOnly bool `json:"audio_only,omitempty"` SourceVideoCodec string `json:"source_video_codec,omitempty"` SourceVideoProfile string `json:"source_video_profile,omitempty"` SourceVideoBitDepth int `json:"source_video_bit_depth,omitempty"` @@ -212,6 +219,8 @@ func (c RecipeCard) ToClaims() streamtoken.Claims { SessionID: c.SessionID, MediaPath: c.InputPath, OutputSubdir: c.OutputSubdir, + DVProfile: c.DVProfile, + AudioOnly: c.AudioOnly, PlayMethod: string(c.PlayMethod), TranscodeAudio: c.TranscodeAudio, RemuxDVMode: string(c.RemuxDVMode), @@ -269,6 +278,8 @@ func RecipeCardFromClaims(c *streamtoken.Claims) RecipeCard { RemuxDVMode: RemuxDVMode(c.RemuxDVMode), InputPath: c.MediaPath, OutputSubdir: c.OutputSubdir, + DVProfile: c.DVProfile, + AudioOnly: c.AudioOnly, SourceVideoCodec: c.SourceVideoCodec, SourceVideoProfile: c.SourceVideoProfile, SourceVideoBitDepth: c.SourceVideoBitDepth, diff --git a/internal/playback/testdata/protocol_v3/capability_response.json b/internal/playback/testdata/protocol_v3/capability_response.json index ba8d3fdca..230cd0820 100644 --- a/internal/playback/testdata/protocol_v3/capability_response.json +++ b/internal/playback/testdata/protocol_v3/capability_response.json @@ -13,6 +13,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "authorized_media_origins_v1", "software_video_decode_v1", "plan_source_duration_v1" ], diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index 22bf6cf38..e3b90a10f 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -5409,6 +5409,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "authorized_media_origins_v1", "software_video_decode_v1", "plan_source_duration_v1" ], diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json index c70258f4c..b194163f6 100644 --- a/internal/playback/testdata/protocol_v3/decision_response.json +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -10,6 +10,7 @@ "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", + "authorized_media_origins_v1", "software_video_decode_v1", "plan_source_duration_v1" ], diff --git a/internal/proxy/mediagrant.go b/internal/proxy/mediagrant.go new file mode 100644 index 000000000..2997c4772 --- /dev/null +++ b/internal/proxy/mediagrant.go @@ -0,0 +1,188 @@ +package proxy + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/auth" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// The /stream/v3 family is the proxy side of authorized_media_origins_v1. A +// client that negotiated the mode receives absolute proxy URLs carrying no +// playback credential at all, and attaches the same Authorization header it +// sends the API. This node therefore has to answer two questions the token +// routes answered from the URL alone: +// +// - what to serve: the API wrote the session's recipe to the shared grant +// store when it planned the route, keyed by playback session id; +// - who is asking: the bearer token is the user's own access token, so it is +// validated against the live login session in Postgres on every request, +// which keeps revocation immediate here as well as on the API. +// +// Both answers are required. A grant alone would let any authenticated user +// stream any session; a valid login alone says nothing about which session's +// bytes the caller is entitled to. + +// proxyGrantLookup reads the recipe central authorized for a session. It fails +// closed: a miss is a 404, never a guess. +type proxyGrantLookup interface { + Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) +} + +// loginSessionValidator reports whether a login session is still active +// (not revoked, not expired). Implemented by *auth.SessionRepository. +type loginSessionValidator interface { + IsValid(ctx context.Context, sessionID string) (bool, error) +} + +// grantErrorResponse mirrors the API's error body so a client sees one error +// shape whichever origin served it. +type grantErrorResponse struct { + Error string `json:"error"` + Message string `json:"message"` +} + +func writeGrantError(w http.ResponseWriter, status int, code, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(grantErrorResponse{Error: code, Message: message}) +} + +// authorizeGrant resolves a /stream/v3 request to the recipe it may serve. +// +// It deliberately accepts only a bearer access token in the Authorization +// header: no ?token= fallback (a query credential is exactly what this mode +// removes) and no sa_ API keys (a machine key is not a viewer, and scope +// enforcement lives on the API). The JWT is verified against the watcher's +// CURRENT secret, so a rotated secret invalidates in-flight streams here the +// same way it does on the token routes. +func (s *Server) authorizeGrant(w http.ResponseWriter, r *http.Request) (*playback.RecipeCard, bool) { + if s.grants == nil || s.loginSessions == nil { + writeGrantError(w, http.StatusServiceUnavailable, "service_unavailable", "This node cannot serve header-authenticated media") + return nil, false + } + sessionID := chi.URLParam(r, "session_id") + if sessionID == "" { + writeGrantError(w, http.StatusBadRequest, "bad_request", "Session ID is required") + return nil, false + } + + cfg := s.watcher.Config() + secret := "" + if cfg != nil { + secret = cfg.Auth.JWTSecret + } + token := grantBearerToken(r) + if token == "" || secret == "" { + writeGrantError(w, http.StatusUnauthorized, "unauthorized", "Missing or malformed authorization header") + return nil, false + } + claims, err := auth.NewJWTService(secret, 0, 0).ValidateToken(token) + if err != nil || claims.TokenType != auth.TokenTypeAccess { + writeGrantError(w, http.StatusUnauthorized, "unauthorized", "Invalid or expired token") + return nil, false + } + valid, err := s.loginSessions.IsValid(r.Context(), claims.SessionID) + if err != nil || !valid { + if err != nil { + slog.WarnContext(r.Context(), "login session check failed", "component", "proxy", "error", err, "playback_session_id", sessionID) + } + writeGrantError(w, http.StatusUnauthorized, "unauthorized", "Session is no longer valid") + return nil, false + } + + card, ok := s.grants.Get(r.Context(), sessionID) + if !ok || card == nil { + writeGrantError(w, http.StatusNotFound, "playback_session_not_found", "Playback session not found") + return nil, false + } + if card.UserID != claims.UserID { + writeGrantError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") + return nil, false + } + return card, true +} + +// grantBearerToken extracts the access token from the Authorization header +// only. An sa_ API key is rejected outright rather than passed on to JWT +// validation, so the refusal is a deliberate policy rather than a parse error. +func grantBearerToken(r *http.Request) string { + header := r.Header.Get("Authorization") + parts := strings.SplitN(header, " ", 2) + if len(parts) != 2 || !strings.EqualFold(parts[0], "bearer") { + return "" + } + token := strings.TrimSpace(parts[1]) + if strings.HasPrefix(token, "sa_") { + return "" + } + return token +} + +// handleGrantIdentity serves a direct-play or progressive-remux session. It +// runs the same serve paths as the token routes, from claims projected out of +// the grant, so seek handling, range/ETag behavior and session tracking are the +// ones the legacy routes already have. +func (s *Server) handleGrantIdentity(w http.ResponseWriter, r *http.Request) { + card, ok := s.authorizeGrant(w, r) + if !ok { + return + } + claims := card.ToClaims() + switch card.PlayMethod { + case playback.PlayRemux: + s.serveRemuxClaims(w, r, &claims) + case playback.PlayTranscode: + writeGrantError(w, http.StatusBadRequest, "bad_request", "Transcode streams use manifest/segment endpoints") + default: + s.serveDirectPlayClaims(w, r, &claims) + } +} + +func (s *Server) handleGrantTranscodeManifest(w http.ResponseWriter, r *http.Request) { + card, ok := s.authorizeGrant(w, r) + if !ok { + return + } + claims := card.ToClaims() + s.touchTranscodeSession(r, &claims) + s.relayGrantToTranscodeNode(w, r, &claims, "/transcode/"+transcodeTransportIDFromClaims(&claims)+"/master.m3u8") +} + +func (s *Server) handleGrantTranscodeSegment(w http.ResponseWriter, r *http.Request) { + card, ok := s.authorizeGrant(w, r) + if !ok { + return + } + claims := card.ToClaims() + s.touchTranscodeSession(r, &claims) + s.relayGrantToTranscodeNode(w, r, &claims, "/transcode/"+transcodeTransportIDFromClaims(&claims)+"/segment/"+chi.URLParam(r, "name")) +} + +// relayGrantToTranscodeNode forwards to the transcode node exactly as the token +// routes do, minting the node-facing stream token here from the grant. +// +// That token never reaches the client: it is the node's own reconstruction +// descriptor (it re-verifies it independently and can re-spawn ffmpeg from it +// after its own restart), so the credential the client was promised it would +// never see stays strictly on the proxy→node hop. +func (s *Server) relayGrantToTranscodeNode(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims, path string) { + cfg := s.watcher.Config() + forwardToken := "" + if cfg != nil && cfg.Auth.JWTSecret != "" { + token, err := streamtoken.Sign(*claims, cfg.Auth.JWTSecret, playback.MaxTokenTTL) + if err != nil { + slog.WarnContext(r.Context(), "sign node relay stream token failed", "component", "proxy", "error", err, "playback_session_id", claims.SessionID) + } else { + forwardToken = token + } + } + s.proxyToTranscodeNode(w, r, claims, path, forwardToken) +} diff --git a/internal/proxy/mediagrant_test.go b/internal/proxy/mediagrant_test.go new file mode 100644 index 000000000..666f9302f --- /dev/null +++ b/internal/proxy/mediagrant_test.go @@ -0,0 +1,248 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/auth" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +const grantTestSecret = "media-grant-proxy-secret" + +type stubGrantStore struct { + cards map[string]playback.RecipeCard +} + +func (s stubGrantStore) Get(_ context.Context, sessionID string) (*playback.RecipeCard, bool) { + card, ok := s.cards[sessionID] + if !ok { + return nil, false + } + return &card, true +} + +type stubLoginSessions struct { + valid map[string]bool +} + +func (s stubLoginSessions) IsValid(_ context.Context, sessionID string) (bool, error) { + return s.valid[sessionID], nil +} + +func newGrantProxyServer(t *testing.T, cards map[string]playback.RecipeCard) *Server { + t.Helper() + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = grantTestSecret + w.SetConfigForTest(cfg) + srv := NewServer(w, nodesessions.NewTracker(nil, "http://proxy-1", "proxy-1", "proxy")) + srv.SetMediaGrantAuthority(stubGrantStore{cards: cards}, stubLoginSessions{valid: map[string]bool{"login-1": true}}) + return srv +} + +func grantAccessToken(t *testing.T, userID int, loginSessionID string) string { + t.Helper() + token, err := auth.NewJWTService(grantTestSecret, time.Hour, time.Hour).GenerateAccessToken(userID, "user", loginSessionID) + if err != nil { + t.Fatal(err) + } + return token +} + +func grantRequest(t *testing.T, srv *Server, method, path, bearer string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, nil) + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + return rr +} + +func writeGrantMedia(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "movie.mp4") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// The grant says what to serve; the caller's own access token says whether they +// may have it. Every way of failing the second question must refuse before any +// media byte is written. +func TestProxyGrantRoutesRefuseUnauthenticatedAndUnauthorizedCallers(t *testing.T) { + path := writeGrantMedia(t, "0123456789") + cards := map[string]playback.RecipeCard{ + "session-1": {SessionID: "session-1", UserID: 7, ProfileID: "profile-1", MediaFileID: 42, PlayMethod: playback.PlayDirect, InputPath: path}, + } + srv := newGrantProxyServer(t, cards) + + for _, test := range []struct { + name string + path string + bearer string + wantStatus int + wantError string + }{ + {name: "no bearer", path: "/stream/v3/session-1", wantStatus: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "token signed by another secret", path: "/stream/v3/session-1", bearer: foreignAccessToken(t), wantStatus: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "revoked login session", path: "/stream/v3/session-1", bearer: grantAccessToken(t, 7, "login-revoked"), wantStatus: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "another user's session", path: "/stream/v3/session-1", bearer: grantAccessToken(t, 8, "login-1"), wantStatus: http.StatusForbidden, wantError: "forbidden"}, + {name: "no grant", path: "/stream/v3/session-missing", bearer: grantAccessToken(t, 7, "login-1"), wantStatus: http.StatusNotFound, wantError: "playback_session_not_found"}, + } { + t.Run(test.name, func(t *testing.T) { + rr := grantRequest(t, srv, http.MethodGet, test.path, test.bearer) + if rr.Code != test.wantStatus { + t.Fatalf("status = %d, want %d (body %s)", rr.Code, test.wantStatus, rr.Body.String()) + } + var body grantErrorResponse + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("error body %q is not the API's error shape: %v", rr.Body.String(), err) + } + if body.Error != test.wantError { + t.Fatalf("error code = %q, want %q", body.Error, test.wantError) + } + if rr.Body.Len() > 0 && rr.Body.String()[0] == '0' { + t.Fatal("refused request received media bytes") + } + }) + } +} + +// An API key is a machine credential, not a viewer: it is refused here rather +// than validated, so scope enforcement stays on the API. +func TestProxyGrantRoutesRefuseAPIKeys(t *testing.T) { + srv := newGrantProxyServer(t, map[string]playback.RecipeCard{}) + rr := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-1", "sa_"+grantAccessToken(t, 7, "login-1")) + if rr.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 (body %s)", rr.Code, rr.Body.String()) + } +} + +// A node that predates the mode (no grant store, no database) must not pretend +// it can serve these routes, and must keep working otherwise. +func TestProxyGrantRoutesReportUnavailableWithoutTheirDependencies(t *testing.T) { + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = grantTestSecret + w.SetConfigForTest(cfg) + srv := NewServer(w, nodesessions.NewTracker(nil, "http://proxy-1", "proxy-1", "proxy")) + + rr := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-1", grantAccessToken(t, 7, "login-1")) + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 (body %s)", rr.Code, rr.Body.String()) + } +} + +func TestProxyGrantDirectPlayServesTheGrantedFile(t *testing.T) { + path := writeGrantMedia(t, "0123456789") + srv := newGrantProxyServer(t, map[string]playback.RecipeCard{ + "session-1": {SessionID: "session-1", UserID: 7, ProfileID: "profile-1", MediaFileID: 42, PlayMethod: playback.PlayDirect, InputPath: path}, + }) + + rr := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-1", grantAccessToken(t, 7, "login-1")) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + if rr.Body.String() != "0123456789" { + t.Fatalf("body = %q, want the granted file's bytes", rr.Body.String()) + } + // direct_stream_resume_v1 needs the strong validator the shared serve path + // sets; the grant route must not lose it by serving the file some other way. + if rr.Header().Get("ETag") == "" { + t.Fatal("direct play served without an ETag; a resumed range could not validate") + } +} + +// A transcode grant is relayed to the node exactly like the token route, with +// the node-facing token minted here — never handed to the client. +func TestProxyGrantTranscodeRelaysToTheNodeWithAProxyMintedToken(t *testing.T) { + var forwarded, authorization, relayPath string + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + forwarded = r.Header.Get("X-Silo-Stream-Token") + authorization = r.Header.Get("Authorization") + relayPath = r.URL.Path + w.Header().Set("Content-Type", "application/vnd.apple.mpegurl") + _, _ = w.Write([]byte("#EXTM3U\nsegment/seg_00001.m4s\n")) + })) + defer node.Close() + + srv := newGrantProxyServer(t, map[string]playback.RecipeCard{ + "session-hls": { + SessionID: "session-hls", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + PlayMethod: playback.PlayTranscode, + TranscodeNodeURL: node.URL, + TranscodeTransportID: "session-hls-plan-a", + InputPath: "/media/movie.mkv", + TargetCodecVideo: "h264", + }, + }) + + rr := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-hls/master.m3u8", grantAccessToken(t, 7, "login-1")) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + if relayPath != "/transcode/session-hls-plan-a/master.m3u8" { + t.Fatalf("relay path = %q, want the plan-scoped transport manifest", relayPath) + } + if authorization != "Bearer "+grantTestSecret { + t.Fatalf("node authorization = %q", authorization) + } + claims, err := streamtoken.Verify(forwarded, grantTestSecret) + if err != nil { + t.Fatalf("verify forwarded stream token: %v", err) + } + if claims.SessionID != "session-hls" || claims.MediaPath != "/media/movie.mkv" { + t.Fatalf("forwarded claims = %#v, want the granted recipe", claims) + } + // The manifest's segment URIs stay relative, so they resolve back into this + // same credential-free family rather than a token route. + if body := rr.Body.String(); body != "#EXTM3U\nsegment/seg_00001.m4s\n" { + t.Fatalf("manifest body = %q, want the node's relative segment URIs", body) + } + + segment := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-hls/segment/seg_00001.m4s", grantAccessToken(t, 7, "login-1")) + if segment.Code != http.StatusOK { + t.Fatalf("segment status = %d, body = %s", segment.Code, segment.Body.String()) + } + if relayPath != "/transcode/session-hls-plan-a/segment/seg_00001.m4s" { + t.Fatalf("segment relay path = %q", relayPath) + } +} + +// A transcode session has no progressive body to serve, and answering one from +// the identity route would hand the client a stream the plan never described. +func TestProxyGrantIdentityRefusesATranscodeGrant(t *testing.T) { + srv := newGrantProxyServer(t, map[string]playback.RecipeCard{ + "session-hls": {SessionID: "session-hls", UserID: 7, PlayMethod: playback.PlayTranscode, TranscodeNodeURL: "http://node-1"}, + }) + rr := grantRequest(t, srv, http.MethodGet, "/stream/v3/session-hls", grantAccessToken(t, 7, "login-1")) + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body %s)", rr.Code, rr.Body.String()) + } +} + +func foreignAccessToken(t *testing.T) string { + t.Helper() + token, err := auth.NewJWTService("a-different-secret", time.Hour, time.Hour).GenerateAccessToken(7, "user", "login-1") + if err != nil { + t.Fatal(err) + } + return token +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 32146eefc..d59a5ca54 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -32,7 +32,13 @@ type Server struct { tracker *nodesessions.Tracker httpClient *http.Client artifactMissReporter remoteArtifactMissReporter - egress *egressMeter + // grants and loginSessions back the credential-free /stream/v3 routes: the + // grant says what to serve, the login-session validator says whether the + // caller may still have it. Both nil in a deployment that predates the + // mode, which is why those routes answer 503 rather than assuming either. + grants proxyGrantLookup + loginSessions loginSessionValidator + egress *egressMeter // subCache stores full-track PGS (.sup) extracts under the transcode dir // so repeat selections skip the whole-file ffmpeg demux. subCache *playback.SubtitleCache @@ -70,6 +76,17 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv } } +// SetMediaGrantAuthority wires the two dependencies the credential-free +// /stream/v3 routes need: the store central writes a session's recipe to, and +// the live login-session validator this proxy re-checks every request against. +// It must be called during construction, before the server begins handling +// requests. Either argument may be nil, which leaves those routes unavailable +// (503) while the token routes keep working unchanged. +func (s *Server) SetMediaGrantAuthority(grants proxyGrantLookup, sessions loginSessionValidator) { + s.grants = grants + s.loginSessions = sessions +} + // SetRemoteArtifactMissReporter wires the authoritative database transition // used when an origin returns 404 after the API's proxy preflight. It must be // called during construction, before the server begins handling requests. @@ -127,6 +144,14 @@ func (s *Server) Handler() http.Handler { r.Head("/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest) r.Get("/stream/transcode/{token}/master.m3u8", s.handleTranscodeManifest) r.Get("/stream/transcode/{token}/segment/{name}", s.handleTranscodeSegment) + // Credential-free grant routes (authorized_media_origins_v1). Same media + // bytes as the token routes above, addressed by session id and + // authorized by the caller's own Authorization header. + r.Head("/stream/v3/{session_id}", s.handleGrantIdentity) + r.Get("/stream/v3/{session_id}", s.handleGrantIdentity) + r.Head("/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest) + r.Get("/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest) + r.Get("/stream/v3/{session_id}/segment/{name}", s.handleGrantTranscodeSegment) r.Get("/stream/subtitles/{token}/{track}/fonts", s.handleSubtitleFonts) r.Get("/stream/subtitles/{token}/{track}", s.handleSubtitle) r.Head("/downloads/file/{token}", s.handleDownloadFile) @@ -213,7 +238,14 @@ func (s *Server) handleDirectPlay(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + s.serveDirectPlayClaims(w, r, claims) +} +// serveDirectPlayClaims serves a direct-play session from an already-authorized +// recipe. The token routes reach it with claims they verified; the grant routes +// reach it with the same claims projected from a grant they authorized against +// the caller's login session — the serving behavior must not differ. +func (s *Server) serveDirectPlayClaims(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims) { info := sessionInfo(s.tracker, claims, "direct_play") s.tracker.Track(r.Context(), info) defer s.tracker.Remove(r.Context(), claims.SessionID) @@ -351,7 +383,13 @@ func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) { if claims == nil { return } + s.serveRemuxClaims(w, r, claims) +} +// serveRemuxClaims serves a progressive remux from an already-authorized +// recipe, shared by the token routes and the grant routes for the same reason +// serveDirectPlayClaims is. +func (s *Server) serveRemuxClaims(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims) { info := sessionInfo(s.tracker, claims, "remux") s.tracker.Track(r.Context(), info) defer s.tracker.Remove(r.Context(), claims.SessionID) @@ -381,7 +419,7 @@ func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request) return } s.touchTranscodeSession(r, claims) - s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/master.m3u8") + s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/master.m3u8", chi.URLParam(r, "token")) } func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) { @@ -391,7 +429,7 @@ func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) } s.touchTranscodeSession(r, claims) name := chi.URLParam(r, "name") - s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/segment/"+name) + s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/segment/"+name, chi.URLParam(r, "token")) } func transcodeTransportIDFromClaims(claims *streamtoken.Claims) string { @@ -527,8 +565,11 @@ func (s *Server) handleSubtitleFonts(w http.ResponseWriter, r *http.Request) { } } -// proxyToTranscodeNode forwards the request to the transcode node specified in the claims. -func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims, path string) { +// proxyToTranscodeNode forwards the request to the transcode node specified in +// the claims. forwardToken is the stream token handed to the node out of band +// (never to the client): the client's own token on a token route, a +// proxy-minted one on a grant route. +func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims, path, forwardToken string) { cfg := s.watcher.Config() if claims.TranscodeNode == "" { http.Error(w, "no transcode node in token", http.StatusBadRequest) @@ -551,8 +592,8 @@ func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, cl // recipe, so the node can re-spawn ffmpeg seeked to the requested segment instead // of 404ing (the integrated server already does this from the same token). The // node re-verifies the token independently before trusting it. - if token := chi.URLParam(r, "token"); token != "" { - req.Header.Set("X-Silo-Stream-Token", token) + if forwardToken != "" { + req.Header.Set("X-Silo-Stream-Token", forwardToken) } resp, err := s.httpClient.Do(req) From 5875949c2859731bb8ed29151e4f6459c070f4e3 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:16:33 -0400 Subject: [PATCH 29/44] fix(playback): address automated review findings on tokenless proxy egress - Preserve the displaced proxy grant across a replan and restore it on rollback, so a failed replacement no longer 404s the restored plan's proxy URL; revoke the grant when a proxy-egress attempt commits onto a transport the API serves itself (identity, relay, or local transcode). - Gate the progressive-remux escalation on a usable grant store as well as configured proxies: a process that can never authorize proxy egress escalates to HLS instead of refusing forever, while transient proxy ineligibility keeps the legacy retryable refusal. - Advertise target_audio_channels in the admin sessions capability endpoint so independently deployed clients can feature-detect it. - Reject an unrecognized video_evidence value on flat download payloads instead of silently resolving from flat claims. - Handle SessionUnauthorized defensively in the stream and jellycompat serve switches (unreachable today; prevents a nil dereference if the caller invariants ever drift). - Document the tokenless replica-affinity constraint in the protocol spec. Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 12 ++ internal/api/handlers/downloads_test.go | 27 ++++ internal/api/handlers/playback_sessions.go | 10 +- .../api/handlers/playback_sessions_test.go | 3 +- internal/api/handlers/playback_v3.go | 128 ++++++++++++++---- .../api/handlers/playback_v3_origins_test.go | 127 +++++++++++++++++ .../handlers/playback_v3_tokenless_test.go | 4 +- internal/api/handlers/stream.go | 6 + internal/jellycompat/streams.go | 6 + internal/playback/resolver.go | 8 ++ internal/playback/resolver_test.go | 24 ++++ 11 files changed, 327 insertions(+), 28 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 36aaffdc1..def7a96ec 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -546,6 +546,18 @@ sticky for the lifetime of the attempt; a client that can no longer honor it must stop and start a new attempt rather than downgrade a replan to a credential-bearing URL. +**Replica affinity.** Because there is no reconstruction recipe, a +header-authenticated attempt's session exists only in the memory of the API +process that started it. A media request routed to any other replica finds no +session and returns the expired/missing response, so a deployment serving +tokenless attempts currently needs either a single API replica or session +affinity on the media routes; legacy token-bearing attempts are unaffected, +since they reconstruct anywhere. Proxy-origin URLs +(`authorized_media_origins_v1`) are also unaffected: the proxy serves from the +shared grant store rather than from an API process's memory. Moving session +state into shared storage is the fix, and until it lands this constraint is +part of the deployment contract. + ### 4.2 Media and subtitle URL query parameters Every URL a plan publishes belongs to one of the route families below, and the diff --git a/internal/api/handlers/downloads_test.go b/internal/api/handlers/downloads_test.go index 558a569a9..7cd6dddb4 100644 --- a/internal/api/handlers/downloads_test.go +++ b/internal/api/handlers/downloads_test.go @@ -502,6 +502,33 @@ func TestHandleCreateDownloadAcceptsFlatCapabilityPayloads(t *testing.T) { } } +// A misspelled evidence tier on an otherwise flat payload must 400 exactly as +// it does on the v3 playback start path. Accepting it would degrade the client +// to flat resolution with no signal that its tier was never read. +func TestHandleCreateDownloadRejectsUnknownVideoEvidenceOnFlatPayloads(t *testing.T) { + svc := &fakeDownloadService{} + h := NewDownloadHandler(svc) + body := []byte(`{ + "content_id":"c1", + "caps":{ + "video_evidence":"exat", + "codecs_video":["h264"], + "codecs_audio":["aac"], + "containers":["mp4"], + "max_resolution":"1080p" + } + }`) + rec := httptest.NewRecorder() + h.HandleCreateDownload(rec, downloadTestRequest(http.MethodPost, "/downloads", body, 7, "", "")) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body: %s)", rec.Code, rec.Body.String()) + } + if svc.gotCreateReq.ContentID != "" { + t.Fatal("an unrecognized video_evidence tier reached the download service") + } +} + func TestHandleCreateDownloadRejectsDetailedEntriesWithoutStrictEvidence(t *testing.T) { svc := &fakeDownloadService{} h := NewDownloadHandler(svc) diff --git a/internal/api/handlers/playback_sessions.go b/internal/api/handlers/playback_sessions.go index bddbd7429..0b99a2e1b 100644 --- a/internal/api/handlers/playback_sessions.go +++ b/internal/api/handlers/playback_sessions.go @@ -88,8 +88,9 @@ type playbackSessionRow struct { // playbackSessionsCapabilitiesResponse advertises the additive fields of the // live admin session payload so independently deployed clients (Android, -// Apple) can feature-detect them. Both fields are omitempty on the wire, so -// absence on a row is otherwise indistinguishable from an older server. +// Apple) can feature-detect them. The advertised fields are omitempty on the +// wire, so absence on a row is otherwise indistinguishable from an older +// server. type playbackSessionsCapabilitiesResponse struct { // EffectivePlayMethod reports that rows carry effective_play_method. EffectivePlayMethod bool `json:"effective_play_method"` @@ -103,6 +104,10 @@ type playbackSessionsCapabilitiesResponse struct { ClientBuild bool `json:"client_build"` // ClientChannel reports that rows carry client_channel. ClientChannel bool `json:"client_channel"` + // TargetAudioChannels reports that rows carry target_audio_channels; + // absent on a row then means the reporting node did not know the encoded + // layout. + TargetAudioChannels bool `json:"target_audio_channels"` } // HandleGetSessionsCapabilities exposes additive feature support for the live @@ -114,6 +119,7 @@ func (h *AdminHandler) HandleGetSessionsCapabilities(w http.ResponseWriter, _ *h IsJellyfinClient: true, ClientBuild: true, ClientChannel: true, + TargetAudioChannels: true, }) } diff --git a/internal/api/handlers/playback_sessions_test.go b/internal/api/handlers/playback_sessions_test.go index 07cc459c9..f281110c7 100644 --- a/internal/api/handlers/playback_sessions_test.go +++ b/internal/api/handlers/playback_sessions_test.go @@ -70,7 +70,8 @@ func TestSessionsCapabilitiesAdvertisesActivityFields(t *testing.T) { if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { t.Fatalf("decode capabilities: %v", err) } - if !resp.EffectivePlayMethod || !resp.IsJellyfinClient || !resp.ClientBuild || !resp.ClientChannel { + if !resp.EffectivePlayMethod || !resp.IsJellyfinClient || !resp.ClientBuild || !resp.ClientChannel || + !resp.TargetAudioChannels { t.Fatalf("capabilities must advertise every additive field: %+v", resp) } want := []string{"direct", "remux", "transcode", "audio"} diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 8c5c5e253..42d0f7d51 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -117,6 +117,10 @@ type proxyGrantStoreV3 interface { // store accepts Put silently, so a URL that only a stored grant can serve // must not be published without checking it. Enabled() bool + // Get reads the grant a session currently egresses from. A replan uses it + // to remember the recipe it is about to overwrite, so a failed replacement + // can hand the restored plan its authority back. + Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) Put(ctx context.Context, sessionID string, card playback.RecipeCard) error Delete(ctx context.Context, sessionID string) error } @@ -248,15 +252,29 @@ type proxyNodeEnumeratorV3 interface { ProxyNodeURLs() []string } -// proxyEgressOriginsAvailableV3 reports whether this deployment has any proxy -// origin an authorized-origins attempt could be sent to. A planner that cannot +// proxyEgressOriginsAvailableV3 reports whether this deployment can actually +// send an authorized-origins attempt to a proxy. A planner that cannot // enumerate proxies counts as none: the escalation this gates exists precisely // for the case where identity work has no executor, and assuming an origin the -// server cannot name would leave the attempt with nowhere to run. +// server cannot name would leave the attempt with nowhere to run. An unusable +// grant store counts as none for the same reason — a proxy origin serves only +// from a stored grant, so without one no proxy URL is publishable by this +// process, ever. +// +// The distinction this preserves is between "not right now" and "not ever". A +// configured-but-currently-ineligible proxy (saturated, unhealthy, cannot run +// the recipe) deliberately still suppresses escalation: that attempt gets the +// same retryable capacity_unavailable a legacy attempt gets, and retrying is +// the correct response. Only a deployment that cannot do proxy egress at all — +// no proxies in the pool, or no grant store to authorize one with — escalates +// to HLS. func (h *PlaybackHandler) proxyEgressOriginsAvailableV3() bool { if h == nil || h.NodePlanner == nil { return false } + if h.ProxyGrantStore == nil || !h.ProxyGrantStore.Enabled() { + return false + } enumerator, ok := h.NodePlanner.(proxyNodeEnumeratorV3) return ok && len(enumerator.ProxyNodeURLs()) > 0 } @@ -1002,11 +1020,16 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p } streamURL := fmt.Sprintf("/stream/%s", routeSession.ID) servedByProxy := false + // priorGrant is the egress authority this attempt overwrote, if any. A + // replan of a session that was already proxy-served must be able to put it + // back: rolling back to the restored old plan leaves that plan's published + // proxy URL live, and a deleted grant would 404 it. + var priorGrant *playback.RecipeCard switch { case !mode.headerAuth: streamURL, servedByProxy = h.identityStreamURLV3(&routeSession, file, proxyNode) case mode.proxyEgress: - streamURL, servedByProxy = h.identityGrantStreamURLV3(r.Context(), &routeSession, file, proxyNode) + streamURL, servedByProxy, priorGrant = h.identityGrantStreamURLV3(r.Context(), &routeSession, file, proxyNode) } releaseProxyReservation := func() { if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { @@ -1050,6 +1073,7 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p if previousNodeURL != "" { h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) } + h.revokeStaleProxyGrantOnCommitV3(r.Context(), session.ID, mode, servedByProxy) h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy) unlock() }, @@ -1064,13 +1088,26 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p // that was never committed. if servedByProxy { releaseProxyReservation() - h.deleteProxyGrantV3(r.Context(), session.ID) + h.restoreProxyGrantV3(r.Context(), session.ID, priorGrant) } unlock() }, }, nil } +// revokeStaleProxyGrantOnCommitV3 drops the egress grant when an +// authorized-origins attempt commits onto a transport this server serves +// itself. A replan that moves a proxy-served session onto the API origin (or +// onto a local transcode) publishes a URL the proxy has no part in, and the +// surviving grant would keep the proxy authorized to serve the previous recipe +// for the rest of its TTL. +func (h *PlaybackHandler) revokeStaleProxyGrantOnCommitV3(ctx context.Context, sessionID string, mode mediaAuthModeV3, servedByProxy bool) { + if !mode.proxyEgress || servedByProxy { + return + } + h.deleteProxyGrantV3(ctx, sessionID) +} + // identityGrantStreamURLV3 builds the stream URL for a direct-play or // progressive-remux session that negotiated authorized media origins: an // absolute, credential-free proxy URL backed by a server-side grant, otherwise @@ -1086,38 +1123,71 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p // caller can release the planner reservation when it is not. A grant that // cannot be written is not fatal: this attempt simply stays on the API origin, // which is exactly the behavior of a header-authenticated attempt that -// negotiated no origins at all. -func (h *PlaybackHandler) identityGrantStreamURLV3(ctx context.Context, s *playback.Session, file *models.MediaFile, proxyNode *nodepool.Node) (string, bool) { +// negotiated no origins at all. The third value is the grant this write +// displaced, for the caller's rollback. +func (h *PlaybackHandler) identityGrantStreamURLV3(ctx context.Context, s *playback.Session, file *models.MediaFile, proxyNode *nodepool.Node) (string, bool, *playback.RecipeCard) { if proxyNode == nil || file == nil || s == nil { - return h.playbackStreamURL(s), false + return h.playbackStreamURL(s), false, nil } card := identityRecipeCard(s) card.InputPath = file.FilePath card.DVProfile = file.PrimaryDVProfile() card.AudioOnly = file.IsAudioOnly() - if !h.putProxyGrantV3(ctx, s.ID, card) { - return h.playbackStreamURL(s), false + prior, stored := h.putProxyGrantV3(ctx, s.ID, card) + if !stored { + return h.playbackStreamURL(s), false, nil } - return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + s.ID, true + return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + s.ID, true, prior } // putProxyGrantV3 stores the recipe a designated proxy origin serves this // session from, reporting whether the grant is actually retrievable. A replan -// overwrites the previous grant under the same session id. +// overwrites the previous grant under the same session id, so the grant it +// displaces is returned for the caller to thread into its rollback: a +// replacement that fails to commit restores the old plan, and that plan's +// already-published proxy URL is only serviceable while its grant exists. +// +// The overwrite is deliberately not staged behind the commit. Between this Put +// and the transport commit the previously published client URL resolves the new +// recipe — same session, same user, same media authority, bounded by the replan +// window — which is the accepted cost of keeping one grant per session. // // A disabled store is a negative answer rather than a silent success: it // accepts writes it cannot retrieve (the Redis-less integrated box), and // publishing a proxy URL against one would hand the client a route that 404s. -func (h *PlaybackHandler) putProxyGrantV3(ctx context.Context, sessionID string, card playback.RecipeCard) bool { +func (h *PlaybackHandler) putProxyGrantV3(ctx context.Context, sessionID string, card playback.RecipeCard) (*playback.RecipeCard, bool) { if h.ProxyGrantStore == nil || !h.ProxyGrantStore.Enabled() || sessionID == "" { - return false + return nil, false + } + prior, hadPrior := h.ProxyGrantStore.Get(ctx, sessionID) + if !hadPrior { + prior = nil } if err := h.ProxyGrantStore.Put(ctx, sessionID, card); err != nil { slog.WarnContext(ctx, "protocol v3 proxy egress grant write failed; serving from the API origin", "component", "api", "playback_session_id", sessionID, "error", err) - return false + return nil, false + } + return prior, true +} + +// restoreProxyGrantV3 undoes a replan's grant overwrite. A session that was +// already egressing from a proxy keeps serving its restored plan, so its grant +// has to come back rather than be revoked; a session that had none is revoked +// as before, because a grant for a transport that never committed would point a +// proxy at work that no longer exists. +func (h *PlaybackHandler) restoreProxyGrantV3(ctx context.Context, sessionID string, prior *playback.RecipeCard) { + if h == nil || h.ProxyGrantStore == nil || sessionID == "" { + return + } + if prior == nil { + h.deleteProxyGrantV3(ctx, sessionID) + return + } + if err := h.ProxyGrantStore.Put(context.WithoutCancel(ctx), sessionID, *prior); err != nil { + slog.WarnContext(ctx, "failed to restore the previous proxy egress grant", + "component", "api", "playback_session_id", sessionID, "error", err) } - return true } // deleteProxyGrantV3 revokes a session's proxy egress authority. It runs @@ -1579,6 +1649,9 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play } committed = true previous := h.tm.SwapTranscodeSession(session.ID, ts) + // A local transcode is never proxy-served, so an authorized-origins + // replan landing here has to revoke the grant it is replacing. + h.revokeStaleProxyGrantOnCommitV3(r.Context(), session.ID, mode, false) h.applyRemoteTransportMarkV3(r.Context(), session.ID, false) unlock() if previous != nil && previous != ts { @@ -1641,6 +1714,9 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla // actually be established; otherwise the client fetches the manifest from // this server and the local liveness path applies. servedByProxy := false + // See prepareIdentityTransportV3: the displaced grant is what a failed + // replan of an already-proxy-served session has to put back. + var priorGrant *playback.RecipeCard switch { case !mode.headerAuth: card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) @@ -1648,7 +1724,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla servedByProxy = nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") case mode.proxyEgress: card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) - url, servedByProxy = h.grantManifestURLV3(r.Context(), card, nodePlan.ProxyNode) + url, servedByProxy, priorGrant = h.grantManifestURLV3(r.Context(), card, nodePlan.ProxyNode) } committed := false previousNodeURL := session.TranscodeNodeURL @@ -1663,6 +1739,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla if previousNodeURL != "" { h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) } + h.revokeStaleProxyGrantOnCommitV3(r.Context(), session.ID, mode, servedByProxy) h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy) unlock() }, rollback: func() { @@ -1680,7 +1757,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla // An egress grant written for a transport that never committed would // point a proxy at a transcode that no longer exists. if servedByProxy { - h.deleteProxyGrantV3(r.Context(), session.ID) + h.restoreProxyGrantV3(r.Context(), session.ID, priorGrant) } unlock() }}, nil @@ -1702,13 +1779,18 @@ func remoteTranscodeRecipeCardV3(session *playback.Session, file *models.MediaFi // the manifest, so the same /stream/v3/{session_id}/... family serves both. // // Without a planned proxy — or when the grant cannot be stored — the client -// fetches the manifest from this server, which relays the same node. -func (h *PlaybackHandler) grantManifestURLV3(ctx context.Context, card playback.RecipeCard, proxyNode *nodepool.Node) (string, bool) { +// fetches the manifest from this server, which relays the same node. The third +// value is the grant this write displaced, for the caller's rollback. +func (h *PlaybackHandler) grantManifestURLV3(ctx context.Context, card playback.RecipeCard, proxyNode *nodepool.Node) (string, bool, *playback.RecipeCard) { localURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", card.SessionID) - if proxyNode == nil || !h.putProxyGrantV3(ctx, card.SessionID, card) { - return localURL, false + if proxyNode == nil { + return localURL, false, nil + } + prior, stored := h.putProxyGrantV3(ctx, card.SessionID, card) + if !stored { + return localURL, false, nil } - return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + card.SessionID + "/master.m3u8", true + return strings.TrimRight(proxyNode.URL, "/") + "/stream/v3/" + card.SessionID + "/master.m3u8", true, prior } func sourceExecutionMetadataV3(file *models.MediaFile, result playback.PlannerResultV3) playback.SourceExecutionMetadataV3 { diff --git a/internal/api/handlers/playback_v3_origins_test.go b/internal/api/handlers/playback_v3_origins_test.go index fe04e0048..9b3abe6d5 100644 --- a/internal/api/handlers/playback_v3_origins_test.go +++ b/internal/api/handlers/playback_v3_origins_test.go @@ -22,11 +22,24 @@ type recordingProxyGrantStoreV3 struct { putErr error cards map[string]playback.RecipeCard deleted []string + // ops is the ordered call log ("get", "put", "delete"), so a test can + // assert that a replan read the grant it displaced before overwriting it. + ops []string } func (s *recordingProxyGrantStoreV3) Enabled() bool { return !s.disabled } +func (s *recordingProxyGrantStoreV3) Get(_ context.Context, sessionID string) (*playback.RecipeCard, bool) { + s.ops = append(s.ops, "get") + card, ok := s.cards[sessionID] + if !ok { + return nil, false + } + return &card, true +} + func (s *recordingProxyGrantStoreV3) Put(_ context.Context, sessionID string, card playback.RecipeCard) error { + s.ops = append(s.ops, "put") if s.putErr != nil { return s.putErr } @@ -38,7 +51,9 @@ func (s *recordingProxyGrantStoreV3) Put(_ context.Context, sessionID string, ca } func (s *recordingProxyGrantStoreV3) Delete(_ context.Context, sessionID string) error { + s.ops = append(s.ops, "delete") s.deleted = append(s.deleted, sessionID) + delete(s.cards, sessionID) return nil } @@ -95,6 +110,87 @@ func TestPrepareTransportV3AuthorizedOriginsRestoreDirectPlayProxyEgress(t *test } } +// A replan overwrites the grant of a session that is already proxy-served. If +// the replacement never commits, the client is left on the OLD plan's proxy +// URL — which only resolves while the OLD grant exists. Rolling back therefore +// has to put the displaced grant back, not revoke it. +func TestPrepareTransportV3AuthorizedOriginsRollbackRestoresTheDisplacedGrant(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + priorCard := playback.RecipeCard{SessionID: "session-origin-replan", UserID: 7, InputPath: "/media/previous-plan.mkv"} + grants := &recordingProxyGrantStoreV3{cards: map[string]playback.RecipeCard{"session-origin-replan": priorCard}} + handler.ProxyGrantStore = grants + file := v3HandlerFixtureFile(t) + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-replan", UserID: 7, ProfileID: "profile-1"}, + file, + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + if got := grants.cards["session-origin-replan"].InputPath; got != file.FilePath { + t.Fatalf("grant media path after prepare = %q, want the replacement plan's %q", got, file.FilePath) + } + + transport.rollback() + + restored, ok := grants.cards["session-origin-replan"] + if !ok { + t.Fatal("rollback revoked the grant; the restored plan's published proxy URL now 404s") + } + if restored.InputPath != priorCard.InputPath { + t.Fatalf("restored grant media path = %q, want the previous plan's %q", restored.InputPath, priorCard.InputPath) + } + if len(grants.deleted) != 0 { + t.Fatalf("grants deleted = %v, want none: the session is still proxy-served", grants.deleted) + } + if grants.ops[0] != "get" { + t.Fatalf("store ops = %v, want the displaced grant read before it was overwritten", grants.ops) + } +} + +// The mirror defect: a replan that lands on a transport this server serves +// itself publishes a URL the proxy has no part in. Leaving the grant alive +// would keep the proxy authorized to serve the previous recipe for the rest of +// its TTL, so committing off the proxy revokes it. +func TestPrepareTransportV3AuthorizedOriginsCommitOffTheProxyRevokesTheGrant(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + // No proxy in the plan: direct play needs no server work, so this attempt + // legitimately commits onto the API-local identity route. + handler.NodePlanner = &recordingNodePlannerV3{} + grants := &recordingProxyGrantStoreV3{cards: map[string]playback.RecipeCard{ + "session-origin-offproxy": {SessionID: "session-origin-offproxy", UserID: 7, InputPath: "/media/previous-plan.mkv"}, + }} + handler.ProxyGrantStore = grants + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-origin-offproxy", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + playback.PlannerResultV3{Plan: identityProxyPlanV3(playback.DeliveryOriginalHTTPV3), PlayMethod: playback.PlayDirect}, + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare identity transport: %v", transportErr) + } + if transport.url != "/stream/session-origin-offproxy" { + t.Fatalf("stream url = %q, want the API-local route", transport.url) + } + + transport.commit() + + if len(grants.deleted) != 1 || grants.deleted[0] != "session-origin-offproxy" { + t.Fatalf("grants deleted on commit = %v, want the stale proxy authority revoked", grants.deleted) + } + if _, ok := grants.cards["session-origin-offproxy"]; ok { + t.Fatal("a grant survived a commit onto a transport the proxy does not serve") + } +} + // A remux egresses from the proxy too, and the grant has to carry the source // facts the proxy cannot look up: without them it would serve a subtly // different stream than the plan promised. @@ -303,6 +399,7 @@ func TestPrepareTransportV3AuthorizedOriginsPublishGrantBackedHLSManifest(t *tes func TestEscalateRefusedProgressiveRemuxV3SkipsEscalationWhenOriginsHaveAProxy(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.ProxyGrantStore = &recordingProxyGrantStoreV3{} escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) if transportErr != nil { @@ -318,6 +415,7 @@ func TestEscalateRefusedProgressiveRemuxV3SkipsEscalationWhenOriginsHaveAProxy(t func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAnyProxyOrigin(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) handler.NodePlanner = &recordingNodePlannerV3{} + handler.ProxyGrantStore = &recordingProxyGrantStoreV3{} escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) if transportErr != nil { @@ -328,6 +426,35 @@ func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAnyProxyOrigin(t } } +// A proxy the server can name but cannot authorize is no executor at all: the +// origin URL is only serviceable while a grant backs it. Without a usable grant +// store this process can never publish one, so the refusal is permanent and the +// escalation has to run — otherwise the remux sits on a retryable +// capacity_unavailable that nothing in this deployment will ever satisfy. +func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAUsableGrantStore(t *testing.T) { + for _, test := range []struct { + name string + store proxyGrantStoreV3 + }{ + {name: "no grant store", store: nil}, + {name: "grant store disabled", store: &recordingProxyGrantStoreV3{disabled: true}}, + } { + t.Run(test.name, func(t *testing.T) { + handler, input, result := escalationFixtureV3(t, true) + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.ProxyGrantStore = test.store + + escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) + if transportErr != nil { + t.Fatalf("escalation error = %#v", transportErr) + } + if escalated.Plan == nil || escalated.Plan.Delivery != playback.DeliveryRemuxHLSV3 { + t.Fatalf("escalated delivery = %#v, want %q", escalated.Plan, playback.DeliveryRemuxHLSV3) + } + }) + } +} + // assertNoPlaybackCredentialV3 fails when a published URL carries any playback // credential — the whole promise of the mode, on the proxy origin as much as on // the API one. diff --git a/internal/api/handlers/playback_v3_tokenless_test.go b/internal/api/handlers/playback_v3_tokenless_test.go index 362dce880..170a842bf 100644 --- a/internal/api/handlers/playback_v3_tokenless_test.go +++ b/internal/api/handlers/playback_v3_tokenless_test.go @@ -116,7 +116,7 @@ func TestPlaybackURLBuildersRefuseTokensForMediaAuthorizedSessions(t *testing.T) // must never fall back to minting the credential the mode removed. grants := &recordingProxyGrantStoreV3{} handler.ProxyGrantStore = grants - got, servedByProxy := handler.identityGrantStreamURLV3(context.Background(), secure, file, proxy) + got, servedByProxy, _ := handler.identityGrantStreamURLV3(context.Background(), secure, file, proxy) if !servedByProxy || got != proxy.URL+"/stream/v3/session-secure" { t.Fatalf("origins identity URL = %q (proxy %v), want the credential-free proxy route", got, servedByProxy) } @@ -125,7 +125,7 @@ func TestPlaybackURLBuildersRefuseTokensForMediaAuthorizedSessions(t *testing.T) t.Fatal("origins identity URL was published without a grant behind it") } - got, servedByProxy = handler.grantManifestURLV3(context.Background(), card, proxy) + got, servedByProxy, _ = handler.grantManifestURLV3(context.Background(), card, proxy) if !servedByProxy || got != proxy.URL+"/stream/v3/session-secure/master.m3u8" { t.Fatalf("origins manifest URL = %q (proxy %v), want the credential-free proxy manifest", got, servedByProxy) } diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index cf7c29027..15db0e46f 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -117,6 +117,12 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return + case playback.SessionUnauthorized: + // Defensive against invariant drift, not a reachable path: this caller + // resolves a non-zero user before loading. Falling through would + // dereference the nil session the status carries. + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return } file, err := h.fileResolver.GetByID(r.Context(), session.MediaFileID) diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index e6ffd3633..95f0fafad 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -398,6 +398,12 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques case playback.SessionForbidden: writeError(w, http.StatusForbidden, "Forbidden", "Session belongs to another user") return + case playback.SessionUnauthorized: + // Defensive against invariant drift, not a reachable path: this caller + // resolves a non-zero user before loading. Falling through would + // dereference the nil session the status carries. + writeError(w, http.StatusUnauthorized, "Unauthorized", "Authentication required") + return } transcodeSession := h.tm.GetTranscodeSession(playSession.UpstreamSessionID) diff --git a/internal/playback/resolver.go b/internal/playback/resolver.go index 7de821ecf..688115f92 100644 --- a/internal/playback/resolver.go +++ b/internal/playback/resolver.go @@ -55,6 +55,14 @@ func (c *ClientCapabilities) hasDetailedVideoEvidence() bool { // opt-in is refused: video_decode entries whose evidence tier cannot validate // them would otherwise be silently ignored. func (c *ClientCapabilities) NormalizeAndValidateVideoDecode() error { + // A present-but-unrecognized tier is a client bug, not a legacy payload: + // silently resolving it from the flat lists would hide a typo behind a + // working-looking answer, where the v3 playback path rejects it outright. + // Omitting the field entirely stays valid — that is what a legacy flat + // payload looks like. + if c.VideoEvidence != "" && !validCapabilityEvidenceV3(c.VideoEvidence) { + return errors.New("video_evidence must be exact, platform_attested, or declared") + } if len(c.VideoDecode) == 0 { return nil } diff --git a/internal/playback/resolver_test.go b/internal/playback/resolver_test.go index 226f83a61..686eb2044 100644 --- a/internal/playback/resolver_test.go +++ b/internal/playback/resolver_test.go @@ -369,6 +369,30 @@ func TestNormalizeAndValidateVideoDecode(t *testing.T) { }, wantErr: true, }, + { + // A typo'd tier on a flat payload is a client bug the v3 playback + // path rejects outright. Silently resolving it from the flat lists + // here would hide the bug behind a working-looking answer. + name: "unknown evidence tier without entries", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.CapabilityEvidenceV3("exat"), + CodecsVideo: []string{"h264"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + }, + wantErr: true, + }, + { + name: "unknown evidence tier with entries", + caps: playback.ClientCapabilities{ + VideoEvidence: playback.CapabilityEvidenceV3("platform-attested"), + CodecsVideo: []string{"av1"}, + VideoDecode: []playback.VideoDecodeCapabilityV3{{ + Codec: "av1", MaxWidth: 1920, MaxHeight: 1080, Hardware: true, + }}, + }, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { From 9b09ae377a32b0b103720128ebf4999794702caa Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:26:00 -0400 Subject: [PATCH 30/44] fix(playback): survive transcode-node restarts on tokenless attempts and stop charging unused proxies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A header-authenticated remote transcode published no stream token, so after a transcode-node restart neither the client nor the API relay had a recipe to forward and playback 404ed until a replan. The API now writes the transport's recipe card to the shared noderecipe store (keyed by transport id, like the jellycompat handoff), and the node's reconstruct path falls back to the store when no X-Silo-Stream-Token is present — the token was a recipe source, never the route's authorization. Recipes are deleted on every deliberate teardown (transport replacement, rollback, session stop/abort); the TTL only backstops a crashed API process. - When a start reserved a proxy+transcode pair but published a URL the proxy does not serve (unwritable egress grant, or the legacy no-token fallback), the planner kept charging the proxy's job slot and estimated bandwidth until the reservation aged out. New ReleaseSessionProxy drops only the proxy half; the transcode node keeps its charge because it is running the job. - The proxy-grant store interface is renamed recipeCardStoreV3 and shared by both handler fields, since it now carries two key spaces. Co-Authored-By: Claude Fable 5 --- internal/api/handlers/playback.go | 18 +- internal/api/handlers/playback_v3.go | 103 ++++++- .../handlers/playback_v3_node_recipe_test.go | 277 ++++++++++++++++++ .../api/handlers/playback_v3_origins_test.go | 43 +-- internal/api/handlers/playback_v3_test.go | 7 + .../handlers/playback_v3_tokenless_test.go | 2 +- internal/api/router.go | 4 + internal/nodepool/planner.go | 25 ++ internal/nodepool/planner_test.go | 42 +++ internal/transcodenode/server.go | 133 ++++++--- internal/transcodenode/server_test.go | 83 ++++++ 11 files changed, 652 insertions(+), 85 deletions(-) create mode 100644 internal/api/handlers/playback_v3_node_recipe_test.go diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 1a25903d5..9052b3b69 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -162,7 +162,15 @@ type PlaybackHandler struct { // ProxyGrantStore hands a proxy the recipe it serves a header-authenticated // session from. Optional: without it (or without Redis behind it) an attempt // that negotiated authorized_media_origins_v1 simply stays on the API origin. - ProxyGrantStore proxyGrantStoreV3 + ProxyGrantStore recipeCardStoreV3 + // NodeRecipeStore hands a transcode node the recipe it rebuilds a + // header-authenticated remote transcode from after its own restart, keyed by + // the transport id the node serves it under. A legacy attempt needs none — + // its client URL carries the recipe in a stream token — but a tokenless + // relayed request has nothing to reconstruct from. Optional and best effort: + // without it (or without Redis behind it) such a session replans instead of + // recovering, exactly as before. + NodeRecipeStore recipeCardStoreV3 ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item @@ -944,6 +952,10 @@ func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *play // recipe card it is never a reconstruction aid, so it is revoked on every // stop and abort. h.deleteProxyGrantV3(ctx, session.ID) + // The teardown above stopped the remote job, so its stored recipe must not + // outlive it: a buffered or retrying request would otherwise rebuild ffmpeg on + // the node for a transport that no longer exists. + h.deleteNodeRecipeV3(ctx, session.TranscodeTransportID) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -983,6 +995,10 @@ func (h *PlaybackHandler) finalizeSessionAbort(ctx context.Context, session *pla // recipe card it is never a reconstruction aid, so it is revoked on every // stop and abort. h.deleteProxyGrantV3(ctx, session.ID) + // The teardown above stopped the remote job, so its stored recipe must not + // outlive it: a buffered or retrying request would otherwise rebuild ffmpeg on + // the node for a transport that no longer exists. + h.deleteNodeRecipeV3(ctx, session.TranscodeTransportID) if syncNow { h.syncSessionsNow(ctx, syncReason) } diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 42d0f7d51..dfacd7cb4 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -50,6 +50,10 @@ const ( // the fetch helper's own 10s timeout: planning happens on the start // request path, where a slow node must degrade the union, not the user. v3NodeCapabilityPlanTimeout = 3 * time.Second + // The node-recipe handoff is restart insurance written while the client is + // blocked on the start response, so a stalled store must lose the insurance + // rather than the start. Matches the jellycompat handoff's budget. + nodeRecipeWriteTimeoutV3 = 2 * time.Second ) var errSubtitleStoreUnavailableV3 = errors.New("subtitle store unavailable") @@ -109,20 +113,22 @@ func headerAuthenticatedMediaV3(clientFeatures []string) mediaAuthModeV3 { } } -// proxyGrantStoreV3 hands a media-authorized session's recipe to the proxy that -// will serve it. The grant replaces the signed URL token as the proxy's -// instruction set; the proxy still authenticates the caller itself. -type proxyGrantStoreV3 interface { - // Enabled reports whether the store can actually carry a grant. A disabled - // store accepts Put silently, so a URL that only a stored grant can serve +// recipeCardStoreV3 is the shared control-plane store central hands a recipe +// card to another Silo process through (*noderecipe.Store). One instance owns +// one key space; the handler holds two of them, and what a stored card +// authorizes or rebuilds is the field's business, not the interface's — see +// PlaybackHandler.ProxyGrantStore and PlaybackHandler.NodeRecipeStore. +type recipeCardStoreV3 interface { + // Enabled reports whether the store can actually carry a card. A disabled + // store accepts Put silently, so a URL that only a stored card can serve // must not be published without checking it. Enabled() bool - // Get reads the grant a session currently egresses from. A replan uses it - // to remember the recipe it is about to overwrite, so a failed replacement - // can hand the restored plan its authority back. - Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) - Put(ctx context.Context, sessionID string, card playback.RecipeCard) error - Delete(ctx context.Context, sessionID string) error + // Get reads the card currently stored under key. A replan uses it to + // remember the card it is about to overwrite, so a failed replacement can + // hand the restored plan its authority back. + Get(ctx context.Context, key string) (*playback.RecipeCard, bool) + Put(ctx context.Context, key string, card playback.RecipeCard) error + Delete(ctx context.Context, key string) error } type transportErrorV3 struct { @@ -172,6 +178,14 @@ type sessionReservationReleaserV3 interface { ReleaseSession(string) } +// sessionProxyReservationReleaserV3 gives back only the proxy half of a node +// reservation, for a start that keeps its transcode node but publishes a URL the +// planned proxy does not serve. Optional: a planner without the method simply +// keeps the whole reservation until it ages out. *nodepool.Planner implements it. +type sessionProxyReservationReleaserV3 interface { + ReleaseSessionProxy(string) +} + func (e *transportErrorV3) Error() string { if e.cause != nil { return e.reason + ": " + e.cause.Error() @@ -1072,6 +1086,7 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p h.tm.CloseTranscodeSession(session.ID, "") if previousNodeURL != "" { h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + h.deleteNodeRecipeV3(r.Context(), previousTransportID) } h.revokeStaleProxyGrantOnCommitV3(r.Context(), session.ID, mode, servedByProxy) h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy) @@ -1204,6 +1219,45 @@ func (h *PlaybackHandler) deleteProxyGrantV3(ctx context.Context, sessionID stri } } +// putNodeRecipeV3 hands the transcode node the recipe it rebuilds this job from +// after a restart, keyed by the transport id the node serves it under (which is +// the id in every relayed node URL, not the playback session id). +// +// It exists because a header-authenticated attempt publishes no stream token, so +// neither the client nor this server's relay has a recipe to forward when the +// node comes back empty — the node would 404 until the client replanned. A node +// dying mid-stream is a normal event, so tokenless playback recovers from it the +// way a legacy token attempt already does. +// +// Best effort, exactly like the jellycompat handoff: the write is bounded and a +// failure only forfeits restart resilience for this session, never the start. +func (h *PlaybackHandler) putNodeRecipeV3(ctx context.Context, transportID string, card playback.RecipeCard) { + if h == nil || h.NodeRecipeStore == nil || transportID == "" { + return + } + putCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), nodeRecipeWriteTimeoutV3) + defer cancel() + if err := h.NodeRecipeStore.Put(putCtx, transportID, card); err != nil { + slog.WarnContext(ctx, "persist node transcode recipe failed; this session cannot survive a node restart", + "component", "api", "playback_session_id", card.SessionID, "transport", transportID, + "node", card.TranscodeNodeURL, "error", err) + } +} + +// deleteNodeRecipeV3 drops a transport's stored recipe so a buffered or retrying +// request cannot resurrect a transcode the server has replaced or ended. The +// store's TTL is only the backstop for the paths that never run (a crashed API +// process); every deliberate teardown deletes here. +func (h *PlaybackHandler) deleteNodeRecipeV3(ctx context.Context, transportID string) { + if h == nil || h.NodeRecipeStore == nil || transportID == "" { + return + } + if err := h.NodeRecipeStore.Delete(context.WithoutCancel(ctx), transportID); err != nil { + slog.WarnContext(ctx, "failed to drop the node transcode recipe", + "component", "api", "transport", transportID, "error", err) + } +} + // planIdentityProxyV3 selects the proxy node that will serve a direct-play or // progressive-remux session. These deliveries need no transcode node — the // bytes are either the source file or a single remux pipe — so the planner is @@ -1659,6 +1713,7 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play } if previousNodeURL != "" { h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + h.deleteNodeRecipeV3(r.Context(), previousTransportID) } ts.SetRestartHook(func(ctx context.Context) { h.maybeStartThrottler(ctx, ts) @@ -1708,6 +1763,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla h.tm.StopRemoteTranscode(transportID, node.URL) return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "The selected transcode node rejected the playback transport.", retryable: true} } + card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) url := fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID) // Either URL builder only returns an absolute proxy URL when a proxy was // planned and its authority (a signed token, or a stored grant) could @@ -1719,13 +1775,29 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla var priorGrant *playback.RecipeCard switch { case !mode.headerAuth: - card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) url = h.buildProxyManifestURL(card, nodePlan.ProxyNode, mode.headerAuth) servedByProxy = nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http") case mode.proxyEgress: - card := remoteTranscodeRecipeCardV3(session, file, node.URL, transportID, req, nodeResp) url, servedByProxy, priorGrant = h.grantManifestURLV3(r.Context(), card, nodePlan.ProxyNode) } + if mode.headerAuth { + // No client-visible URL carries a stream token in this mode, so neither the + // client nor the API relay can hand the node its recipe back after the node + // restarts. Store it for the node to fetch. Both sub-modes need it: the + // API-local relay is the fallback whenever a proxy origin is not used, and + // the proxy relays the same tokenless node URLs when it is. + h.putNodeRecipeV3(r.Context(), transportID, card) + } + if nodePlan.ProxyNode != nil && !servedByProxy { + // The planner charged a proxy for a stream that will not cross it (no + // writable grant, or a legacy no-token fallback). Give back the proxy half + // of the reservation now rather than let it pin that node's job slot and + // estimated bandwidth until it ages out; the transcode node keeps its half, + // because it is running the job. + if releaser, ok := h.NodePlanner.(sessionProxyReservationReleaserV3); ok { + releaser.ReleaseSessionProxy(session.ID) + } + } committed := false previousNodeURL := session.TranscodeNodeURL previousTransportID := remoteTransportID(session) @@ -1738,6 +1810,7 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla h.tm.CloseTranscodeSession(session.ID, "") if previousNodeURL != "" { h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + h.deleteNodeRecipeV3(r.Context(), previousTransportID) } h.revokeStaleProxyGrantOnCommitV3(r.Context(), session.ID, mode, servedByProxy) h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy) @@ -1748,6 +1821,8 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla } committed = true h.tm.StopRemoteTranscode(transportID, node.URL) + // The node job this recipe rebuilds is gone, so the recipe must go too. + h.deleteNodeRecipeV3(r.Context(), transportID) // The accepted node job is gone; drop the planner reservation too so // repeated failed starts cannot pin the node's max-job or bandwidth // budget until the reservation ages out. diff --git a/internal/api/handlers/playback_v3_node_recipe_test.go b/internal/api/handlers/playback_v3_node_recipe_test.go new file mode 100644 index 000000000..61d33a993 --- /dev/null +++ b/internal/api/handlers/playback_v3_node_recipe_test.go @@ -0,0 +1,277 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// remoteTranscodeNodeStubV3 is a pooled transcode node that advertises the H.264 +// recipe and accepts a start, so a test can drive prepareRemoteTransportV3 +// without ffmpeg. +func remoteTranscodeNodeStubV3(t *testing.T) *httptest.Server { + t.Helper() + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/hw-capabilities": + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + var request transcodenode.TranscodeStartRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Errorf("decode remote start: %v", err) + } + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{SessionID: request.SessionID, Status: "started"}) + default: + w.WriteHeader(http.StatusNoContent) + } + })) + t.Cleanup(node.Close) + return node +} + +func remoteHLSPlanV3() *playback.PlanV3 { + return &playback.PlanV3{ + PlanID: "plan:node-recipe", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{{Name: playback.TransformationVideoToH264V3, Executor: playback.ExecutorServerV3, RecipeVersion: playback.TransformationVideoToH264RecipeVersionV3}}, + } +} + +func remoteHLSResultV3() playback.PlannerResultV3 { + return playback.PlannerResultV3{Plan: remoteHLSPlanV3(), PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"} +} + +// A header-authenticated attempt publishes no stream token, so when the node +// restarts neither the client nor this server's relay has a recipe to hand back +// — the node 404s until the client replans. The recipe therefore goes into the +// shared store the node reads, keyed by the TRANSPORT id the node serves the job +// under (not the playback session id, which is what the client URL carries). +func TestPrepareTransportV3HeaderAuthStoresTheNodeRecipeForRestartRecovery(t *testing.T) { + for _, test := range []struct { + name string + mode mediaAuthModeV3 + }{ + // Both sub-modes need it: without authorized origins the API relays the + // node itself, and with them the API relay is still the fallback whenever + // the proxy URL is not published. + {name: "header auth only", mode: headerAuthenticatedMediaV3([]string{playback.FeatureHeaderAuthenticatedMediaV3})}, + {name: "authorized origins", mode: authorizedOriginsModeV3()}, + } { + t.Run(test.name, func(t *testing.T) { + node := remoteTranscodeNodeStubV3(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}, ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.ProxyGrantStore = &recordingRecipeCardStoreV3{} + recipes := &recordingRecipeCardStoreV3{} + handler.NodeRecipeStore = recipes + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-node-recipe", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + remoteHLSResultV3(), + test.mode) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + + card, ok := recipes.cards[transport.transportID] + if !ok { + t.Fatalf("no recipe stored under transport %q; a node restart would 404 this session", transport.transportID) + } + if card.TranscodeNodeURL != node.URL || card.TargetCodecVideo != "h264" || card.SegmentDuration <= 0 { + t.Fatalf("stored recipe = %#v, want the complete recipe the node accepted", card) + } + if _, keyedBySession := recipes.cards["session-node-recipe"]; keyedBySession { + t.Fatal("recipe stored under the playback session id; the node serves under the transport id") + } + + // A transport that never commits leaves no node job, so its recipe + // must not survive to rebuild one. + transport.rollback() + if len(recipes.deleted) != 1 || recipes.deleted[0] != transport.transportID { + t.Fatalf("recipes deleted on rollback = %v, want the transport's recipe dropped", recipes.deleted) + } + }) + } +} + +// A legacy attempt carries its whole recipe in the client's stream token, which +// the relay forwards to the node. It needs no stored copy, and writing one would +// put a media path in Redis for no reason. +func TestPrepareTransportV3LegacyAttemptStoresNoNodeRecipe(t *testing.T) { + node := remoteTranscodeNodeStubV3(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}}} + recipes := &recordingRecipeCardStoreV3{} + handler.NodeRecipeStore = recipes + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-legacy-recipe", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + remoteHLSResultV3(), + mediaAuthModeV3{}) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + + if len(recipes.cards) != 0 { + t.Fatalf("recipes stored = %v, want none for a token-carrying attempt", recipes.cards) + } +} + +// Committing a replacement stops the previous node process, so the recipe that +// rebuilds it has to go with it — otherwise a buffered request could resurrect +// the transport the replan just retired. +func TestPrepareTransportV3RemoteCommitDropsThePreviousTransportRecipe(t *testing.T) { + node := remoteTranscodeNodeStubV3(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}}} + const previousTransportID = "session-node-replan-plan0001-aaaabbbb" + recipes := &recordingRecipeCardStoreV3{cards: map[string]playback.RecipeCard{ + previousTransportID: {SessionID: "session-node-replan", TranscodeTransportID: previousTransportID}, + }} + handler.NodeRecipeStore = recipes + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ + ID: "session-node-replan", UserID: 7, ProfileID: "profile-1", + TranscodeNodeURL: node.URL, TranscodeTransportID: previousTransportID, + }, + v3HandlerFixtureFile(t), + remoteHLSResultV3(), + headerAuthenticatedMediaV3([]string{playback.FeatureHeaderAuthenticatedMediaV3})) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + + transport.commit() + + if len(recipes.deleted) != 1 || recipes.deleted[0] != previousTransportID { + t.Fatalf("recipes deleted on commit = %v, want the replaced transport's recipe dropped", recipes.deleted) + } + if _, ok := recipes.cards[transport.transportID]; !ok { + t.Fatal("commit dropped the recipe of the transport it just committed") + } +} + +// A stopped session's node job is gone, so its stored recipe must be too. +func TestFinalizeSessionStopDropsTheNodeRecipe(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + const transportID = "session-node-stop-plan0001-aaaabbbb" + recipes := &recordingRecipeCardStoreV3{cards: map[string]playback.RecipeCard{ + transportID: {SessionID: "session-node-stop", TranscodeTransportID: transportID}, + }} + handler.NodeRecipeStore = recipes + session := &playback.Session{ + ID: "session-node-stop", UserID: 7, ProfileID: "profile-1", + TranscodeNodeURL: "http://node-1", TranscodeTransportID: transportID, + } + + handler.finalizeSessionStop(context.Background(), session, false, "", true) + + if len(recipes.deleted) != 1 || recipes.deleted[0] != transportID { + t.Fatalf("recipes deleted on stop = %v, want the session's transport recipe dropped", recipes.deleted) + } +} + +// The planner charged a proxy for bytes that will not cross it. Keeping that +// half of the reservation makes a healthy proxy look saturated after a burst of +// grant-store failures, so it is given back as soon as the URL is settled — the +// transcode node keeps its half, because it is running the job. +func TestPrepareTransportV3ReleasesTheProxyHalfWhenTheManifestIsNotProxyServed(t *testing.T) { + for _, test := range []struct { + name string + jwtSecret string + mode mediaAuthModeV3 + grants recipeCardStoreV3 + }{ + { + name: "grant write failed", + jwtSecret: "test-secret", + mode: authorizedOriginsModeV3(), + grants: &recordingRecipeCardStoreV3{putErr: errors.New("redis is down")}, + }, + { + // The legacy no-token fallback leaks the same half: no signable + // token means the proxy URL cannot be addressed at all. + name: "legacy attempt with no signable token", + mode: mediaAuthModeV3{}, + grants: &recordingRecipeCardStoreV3{}, + }, + } { + t.Run(test.name, func(t *testing.T) { + node := remoteTranscodeNodeStubV3(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = test.jwtSecret + planner := &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}, ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.NodePlanner = planner + handler.ProxyGrantStore = test.grants + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-proxy-half", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + remoteHLSResultV3(), + test.mode) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + + if transport.url != "/playback/transcode/session-proxy-half/master.m3u8" { + t.Fatalf("manifest url = %q, want the API-relayed manifest", transport.url) + } + if len(planner.releasedProxy) != 1 || planner.releasedProxy[0] != "session-proxy-half" { + t.Fatalf("proxy-half releases = %v, want the unused proxy reservation given back", planner.releasedProxy) + } + if len(planner.released) != 0 { + t.Fatalf("whole-reservation releases = %v, want none: the transcode node is running the job", planner.released) + } + }) + } +} + +// A proxy that does serve the manifest keeps its reservation: the bytes really +// are going to cross it. +func TestPrepareTransportV3KeepsTheProxyReservationWhenTheProxyServes(t *testing.T) { + node := remoteTranscodeNodeStubV3(t) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + planner := &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}, ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} + handler.NodePlanner = planner + handler.ProxyGrantStore = &recordingRecipeCardStoreV3{} + + transport, transportErr := handler.prepareTransportV3( + httptest.NewRequest(http.MethodPost, "/", nil), + &playback.Session{ID: "session-proxy-served", UserID: 7, ProfileID: "profile-1"}, + v3HandlerFixtureFile(t), + remoteHLSResultV3(), + authorizedOriginsModeV3()) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + + if transport.url != "http://proxy-1/stream/v3/session-proxy-served/master.m3u8" { + t.Fatalf("manifest url = %q, want the proxy manifest", transport.url) + } + if len(planner.releasedProxy) != 0 { + t.Fatalf("proxy-half releases = %v, want none: the proxy is serving this stream", planner.releasedProxy) + } +} diff --git a/internal/api/handlers/playback_v3_origins_test.go b/internal/api/handlers/playback_v3_origins_test.go index 9b3abe6d5..3deab89bf 100644 --- a/internal/api/handlers/playback_v3_origins_test.go +++ b/internal/api/handlers/playback_v3_origins_test.go @@ -14,10 +14,11 @@ import ( "github.com/Silo-Server/silo-server/internal/transcodenode" ) -// recordingProxyGrantStoreV3 stands in for the shared Redis grant store: it -// records what a proxy would be told to serve, so a test can assert on the +// recordingRecipeCardStoreV3 stands in for either key space of the shared Redis +// recipe store: it records what a proxy would be told to serve, or what a +// restarted transcode node would rebuild from, so a test can assert on the // authority the URL depends on rather than only on the URL's shape. -type recordingProxyGrantStoreV3 struct { +type recordingRecipeCardStoreV3 struct { disabled bool putErr error cards map[string]playback.RecipeCard @@ -27,9 +28,9 @@ type recordingProxyGrantStoreV3 struct { ops []string } -func (s *recordingProxyGrantStoreV3) Enabled() bool { return !s.disabled } +func (s *recordingRecipeCardStoreV3) Enabled() bool { return !s.disabled } -func (s *recordingProxyGrantStoreV3) Get(_ context.Context, sessionID string) (*playback.RecipeCard, bool) { +func (s *recordingRecipeCardStoreV3) Get(_ context.Context, sessionID string) (*playback.RecipeCard, bool) { s.ops = append(s.ops, "get") card, ok := s.cards[sessionID] if !ok { @@ -38,7 +39,7 @@ func (s *recordingProxyGrantStoreV3) Get(_ context.Context, sessionID string) (* return &card, true } -func (s *recordingProxyGrantStoreV3) Put(_ context.Context, sessionID string, card playback.RecipeCard) error { +func (s *recordingRecipeCardStoreV3) Put(_ context.Context, sessionID string, card playback.RecipeCard) error { s.ops = append(s.ops, "put") if s.putErr != nil { return s.putErr @@ -50,7 +51,7 @@ func (s *recordingProxyGrantStoreV3) Put(_ context.Context, sessionID string, ca return nil } -func (s *recordingProxyGrantStoreV3) Delete(_ context.Context, sessionID string) error { +func (s *recordingRecipeCardStoreV3) Delete(_ context.Context, sessionID string) error { s.ops = append(s.ops, "delete") s.deleted = append(s.deleted, sessionID) delete(s.cards, sessionID) @@ -69,7 +70,7 @@ func TestPrepareTransportV3AuthorizedOriginsRestoreDirectPlayProxyEgress(t *test handler.JWTSecret = "test-secret" planner := &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} handler.NodePlanner = planner - grants := &recordingProxyGrantStoreV3{} + grants := &recordingRecipeCardStoreV3{} handler.ProxyGrantStore = grants file := v3HandlerFixtureFile(t) @@ -119,7 +120,7 @@ func TestPrepareTransportV3AuthorizedOriginsRollbackRestoresTheDisplacedGrant(t handler.JWTSecret = "test-secret" handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} priorCard := playback.RecipeCard{SessionID: "session-origin-replan", UserID: 7, InputPath: "/media/previous-plan.mkv"} - grants := &recordingProxyGrantStoreV3{cards: map[string]playback.RecipeCard{"session-origin-replan": priorCard}} + grants := &recordingRecipeCardStoreV3{cards: map[string]playback.RecipeCard{"session-origin-replan": priorCard}} handler.ProxyGrantStore = grants file := v3HandlerFixtureFile(t) @@ -163,7 +164,7 @@ func TestPrepareTransportV3AuthorizedOriginsCommitOffTheProxyRevokesTheGrant(t * // No proxy in the plan: direct play needs no server work, so this attempt // legitimately commits onto the API-local identity route. handler.NodePlanner = &recordingNodePlannerV3{} - grants := &recordingProxyGrantStoreV3{cards: map[string]playback.RecipeCard{ + grants := &recordingRecipeCardStoreV3{cards: map[string]playback.RecipeCard{ "session-origin-offproxy": {SessionID: "session-origin-offproxy", UserID: 7, InputPath: "/media/previous-plan.mkv"}, }} handler.ProxyGrantStore = grants @@ -200,7 +201,7 @@ func TestPrepareTransportV3AuthorizedOriginsCarryRemuxSourceFacts(t *testing.T) stubCopySeekAnchorV3(handler) proxy := capableProxyStubV3(t) handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: proxy.URL + "/"}}} - grants := &recordingProxyGrantStoreV3{} + grants := &recordingRecipeCardStoreV3{} handler.ProxyGrantStore = grants file := v3HandlerFixtureFile(t) @@ -239,10 +240,10 @@ func TestPrepareTransportV3AuthorizedOriginsCarryRemuxSourceFacts(t *testing.T) func TestPrepareTransportV3AuthorizedOriginsFallBackToTheAPIWhenTheGrantFails(t *testing.T) { for _, test := range []struct { name string - store *recordingProxyGrantStoreV3 + store *recordingRecipeCardStoreV3 }{ - {name: "write error", store: &recordingProxyGrantStoreV3{putErr: errors.New("redis is down")}}, - {name: "store disabled", store: &recordingProxyGrantStoreV3{disabled: true}}, + {name: "write error", store: &recordingRecipeCardStoreV3{putErr: errors.New("redis is down")}}, + {name: "store disabled", store: &recordingRecipeCardStoreV3{disabled: true}}, } { t.Run(test.name, func(t *testing.T) { handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) @@ -283,7 +284,7 @@ func TestPrepareTransportV3AuthorizedOriginsRefuseLocalRemuxWhenTheGrantFails(t proxy := capableProxyStubV3(t) planner := &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: proxy.URL}}} handler.NodePlanner = planner - handler.ProxyGrantStore = &recordingProxyGrantStoreV3{putErr: errors.New("redis is down")} + handler.ProxyGrantStore = &recordingRecipeCardStoreV3{putErr: errors.New("redis is down")} transport, transportErr := handler.prepareTransportV3( httptest.NewRequest(http.MethodPost, "/", nil), @@ -309,7 +310,7 @@ func TestPrepareTransportV3HeaderAuthOnlyStaysOnTheAPIOrigin(t *testing.T) { handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) handler.JWTSecret = "test-secret" handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} - grants := &recordingProxyGrantStoreV3{} + grants := &recordingRecipeCardStoreV3{} handler.ProxyGrantStore = grants transport, transportErr := handler.prepareTransportV3( @@ -357,7 +358,7 @@ func TestPrepareTransportV3AuthorizedOriginsPublishGrantBackedHLSManifest(t *tes handler.JWTSecret = "test-secret" planner := &recordingNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: node.URL}, ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} handler.NodePlanner = planner - grants := &recordingProxyGrantStoreV3{} + grants := &recordingRecipeCardStoreV3{} handler.ProxyGrantStore = grants plan := &playback.PlanV3{ @@ -399,7 +400,7 @@ func TestPrepareTransportV3AuthorizedOriginsPublishGrantBackedHLSManifest(t *tes func TestEscalateRefusedProgressiveRemuxV3SkipsEscalationWhenOriginsHaveAProxy(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) handler.NodePlanner = &recordingNodePlannerV3{plan: nodepool.Plan{ProxyNode: &nodepool.Node{URL: "http://proxy-1"}}} - handler.ProxyGrantStore = &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = &recordingRecipeCardStoreV3{} escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) if transportErr != nil { @@ -415,7 +416,7 @@ func TestEscalateRefusedProgressiveRemuxV3SkipsEscalationWhenOriginsHaveAProxy(t func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAnyProxyOrigin(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) handler.NodePlanner = &recordingNodePlannerV3{} - handler.ProxyGrantStore = &recordingProxyGrantStoreV3{} + handler.ProxyGrantStore = &recordingRecipeCardStoreV3{} escalated, transportErr := handler.escalateRefusedProgressiveRemuxV3(context.Background(), authorizedOriginsModeV3(), func() playback.PlannerInputV3 { return input }, result) if transportErr != nil { @@ -434,10 +435,10 @@ func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAnyProxyOrigin(t func TestEscalateRefusedProgressiveRemuxV3StillEscalatesWithoutAUsableGrantStore(t *testing.T) { for _, test := range []struct { name string - store proxyGrantStoreV3 + store recipeCardStoreV3 }{ {name: "no grant store", store: nil}, - {name: "grant store disabled", store: &recordingProxyGrantStoreV3{disabled: true}}, + {name: "grant store disabled", store: &recordingRecipeCardStoreV3{disabled: true}}, } { t.Run(test.name, func(t *testing.T) { handler, input, result := escalationFixtureV3(t, true) diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go index 29eff8703..251313009 100644 --- a/internal/api/handlers/playback_v3_test.go +++ b/internal/api/handlers/playback_v3_test.go @@ -3909,6 +3909,9 @@ type recordingNodePlannerV3 struct { estBitrateKbps int plannedSessionID string released []string + // releasedProxy records the proxy-half releases: a start that keeps its + // transcode node but publishes a URL the planned proxy does not serve. + releasedProxy []string } func (p *recordingNodePlannerV3) PlanSession(sessionID, _ string, needsTranscode bool, estBitrateKbps int) nodepool.Plan { @@ -3922,6 +3925,10 @@ func (p *recordingNodePlannerV3) ReleaseSession(sessionID string) { p.released = append(p.released, sessionID) } +func (p *recordingNodePlannerV3) ReleaseSessionProxy(sessionID string) { + p.releasedProxy = append(p.releasedProxy, sessionID) +} + // PlanSessionWith mirrors the real planner: the eligibility predicate narrows // the pool before selection, so a proxy that cannot execute the recipe is // skipped rather than picked and then rejected. diff --git a/internal/api/handlers/playback_v3_tokenless_test.go b/internal/api/handlers/playback_v3_tokenless_test.go index 170a842bf..48a98a90b 100644 --- a/internal/api/handlers/playback_v3_tokenless_test.go +++ b/internal/api/handlers/playback_v3_tokenless_test.go @@ -114,7 +114,7 @@ func TestPlaybackURLBuildersRefuseTokensForMediaAuthorizedSessions(t *testing.T) // The authorized-origins builders publish the same proxy origin the legacy // ones do, but address it by session id against a stored grant — so they // must never fall back to minting the credential the mode removed. - grants := &recordingProxyGrantStoreV3{} + grants := &recordingRecipeCardStoreV3{} handler.ProxyGrantStore = grants got, servedByProxy, _ := handler.identityGrantStreamURLV3(context.Background(), secure, file, proxy) if !servedByProxy || got != proxy.URL+"/stream/v3/session-secure" { diff --git a/internal/api/router.go b/internal/api/router.go index 8a2a24159..16de45595 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1011,6 +1011,10 @@ func NewRouter(deps Dependencies) chi.Router { // from the pool instead of this server. Nil-safe: without Redis the // store reports itself disabled and every such attempt stays API-local. playbackHandler.ProxyGrantStore = noderecipe.NewProxyGrantStore(deps.RedisClient, 0) + // Hand transcode nodes the recipes they rebuild header-authenticated remote + // transcodes from after a restart. Same nil-safety: without Redis such a + // session replans instead of recovering, as it did before. + playbackHandler.NodeRecipeStore = noderecipe.NewStore(deps.RedisClient, 0) if deps.Config != nil { playbackHandler.PlaybackConfig = func() config.PlaybackConfig { return deps.CurrentConfig().Playback diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 2388e9d0c..b84ba6584 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -320,6 +320,31 @@ func (p *Planner) ReleaseSession(sessionID string) { p.mu.Unlock() } +// ReleaseSessionProxy drops only the proxy half of a session's reservation, +// leaving its transcode node charged. A start that selected both nodes but ends +// up publishing a URL the proxy does not serve (its egress grant could not be +// written, or the attempt fell back to the API-relayed manifest) would otherwise +// keep charging that proxy's job slot and estimated bandwidth for a stream no +// byte will cross it — enough grant-store failures and a healthy proxy looks +// saturated. The transcode node is still running the job, so its half stands. +func (p *Planner) ReleaseSessionProxy(sessionID string) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + res, ok := p.reserved[sessionID] + if !ok { + return + } + res.proxyURL = "" + res.kbps = 0 + if res.transcodeURL == "" { + // Nothing left to bridge; drop the entry rather than wait out its age. + delete(p.reserved, sessionID) + } +} + // ReserveTranscodeWork selects the least-loaded healthy transcode node while // sharing the same health-bridging reservation accounting as playback. Unlike // a playback session it does not require a proxy partner: the completed file diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index 85e213aec..d21c10e49 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -129,6 +129,48 @@ func TestReleaseSessionDropsProvisionalReservation(t *testing.T) { } } +// A start that selected both nodes but publishes a URL the proxy does not serve +// must give the proxy's job slot and estimated bandwidth back while the +// transcode node keeps running the job. Asserted through selection, which is +// what the accounting exists to drive. +func TestReleaseSessionProxyFreesTheProxyHalfAndKeepsTheTranscode(t *testing.T) { + proxy := proxyNode(1, "http://proxy-1", nil) + proxy.MaxJobs = intPtr(1) + proxy.MaxBandwidthKbps = intPtr(10_000) + transcode := transcodeNode(2, "http://tc-1", nil, 0) + transcode.MaxJobs = intPtr(1) + f := newFixture([]*Node{proxy}, []*Node{transcode}) + + plan := f.planner.PlanSession("s1", "", true, 8_000) + if plan.TranscodeNode == nil || plan.ProxyNode == nil { + t.Fatalf("plan = %+v, want both halves reserved", plan) + } + // Both halves are charged, so nothing else fits on the proxy. + if got := f.planner.PlanSession("s2", "", false, 2_000).ProxyNode; got != nil { + t.Fatalf("proxy admitted %+v while its reservation stands", got) + } + + f.planner.ReleaseSessionProxy("s1") + + // 8 Mbps only fits if BOTH the job slot and the bandwidth charge were + // released; the estimate alone would leave 2 Mbps of headroom. + if got := f.planner.PlanSession("s2", "", false, 8_000).ProxyNode; got == nil { + t.Fatal("released proxy half still blocked the proxy") + } + // The transcode node is still running s1, so its slot is still charged. + if got := f.planner.PlanSession("s3", "", true, 0).TranscodeNode; got != nil { + t.Fatalf("transcode node admitted %+v; only the proxy half was released", got) + } + + // Nil-safe, and an unknown session is a no-op rather than a phantom entry. + var absent *Planner + absent.ReleaseSessionProxy("s1") + f.planner.ReleaseSessionProxy("never-planned") + if _, ok := f.planner.reserved["never-planned"]; ok { + t.Fatal("releasing an unknown session created a reservation") + } +} + func TestDegradedGroupExcludesItsTranscodeNodes(t *testing.T) { unhealthyProxy := proxyNode(1, "http://proxy-a", strPtr("rack-a")) unhealthyProxy.Healthy = false diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 0f9f87504..ea50d59e7 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -419,11 +419,15 @@ func (s *Server) reapSession(sessionID string, session *playback.TranscodeSessio } // recipeStore reads a remote transcode's reconstruction recipe written by central -// at transcode start. The jellycompat node-hop token is identity-only by design — -// not because a Jellyfin client can't round-trip it, but because the recipe is -// mutated in place and the client can't be driven to refresh a stale token, so the -// authoritative recipe lives server-side (see internal/noderecipe). On a node -// restart the node fetches it here instead of 404ing. *noderecipe.Store implements it. +// at transcode start, keyed by the transport id this node serves the job under. +// It is the reconstruct source for every flow whose request cannot carry a +// complete recipe itself: the jellycompat node-hop token is identity-only by +// design — not because a Jellyfin client can't round-trip it, but because the +// recipe is mutated in place and the client can't be driven to refresh a stale +// token — and a header-authenticated (tokenless) attempt publishes no credential +// at all, so the relayed request carries nothing to rebuild from (see +// internal/noderecipe). On a node restart the node fetches the recipe here +// instead of 404ing. *noderecipe.Store implements it. type recipeStore interface { Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) // Delete drops a session's recipe so a buffered/retrying request after a node @@ -433,8 +437,9 @@ type recipeStore interface { } // SetRecipeStore wires the control-plane recipe store so this node can rebuild a -// jellycompat transcode after its own restart. Optional; without it a recipe-less -// (jellycompat) token cannot reconstruct and the request 404s as before. +// jellycompat or header-authenticated transcode after its own restart. Optional; +// without it a request that carries no complete recipe of its own cannot +// reconstruct and 404s as before. func (s *Server) SetRecipeStore(store recipeStore) { s.recipeStore = store } @@ -871,48 +876,54 @@ func (s *Server) requireApprovedInputPath(w http.ResponseWriter, r *http.Request } // reconstructFromToken rebuilds a transcode session this node lost to its own -// restart. The proxy forwards the client's verified stream token in the -// X-Silo-Stream-Token header; the token carries the full byte-affecting recipe -// (the former Postgres "recipe card"), so the node can re-spawn ffmpeg seeked to -// the requested segment rather than 404ing — mirroring the integrated server's -// token-carried reconstruct. Returns nil when the request carries no usable -// transcode token, which the caller renders as a genuine not-found. +// restart, from whichever recipe source the request has. +// +// A legacy attempt forwards the client's verified stream token in the +// X-Silo-Stream-Token header, and a native token carries the full byte-affecting +// recipe (the former Postgres "recipe card"), so the node can re-spawn ffmpeg +// seeked to the requested segment rather than 404ing — mirroring the integrated +// server's token-carried reconstruct. +// +// Two flows reach this path with no usable token at all and rebuild from the +// control-plane recipe store instead: jellycompat, whose node-hop token is +// identity-only by design (see internal/noderecipe), and a header-authenticated +// (tokenless) attempt, where no client-visible URL carries a credential and the +// relayed request therefore has no token to forward. The token was never this +// route's authorization — the static bearer already authenticated the caller — +// so its absence only removes a recipe source, never a permission. +// +// Returns nil when no source yields a complete transcode recipe for the session +// id in the URL, which the caller renders as a genuine not-found. // // requestedSegment is the segment the client is fetching, or negative on the // manifest path. Reconstruction is single-flighted per session id so concurrent // manifest and segment requests for the same lost session share one ffmpeg. func (s *Server) reconstructFromToken(r *http.Request, sessionID string, requestedSegment int) *playback.TranscodeSession { - tokenStr := r.Header.Get("X-Silo-Stream-Token") - if tokenStr == "" { - return nil - } - cfg := s.watcher.Config() - if cfg == nil { - return nil - } - claims, err := streamtoken.Verify(tokenStr, cfg.Auth.JWTSecret) - if err != nil { - slog.WarnContext(r.Context(), "transcode node reconstruct: invalid stream token", "component", "transcodenode", "error", err, - "session", sessionID, "playback_session_id", sessionID) - return nil - } - card := playback.RecipeCardFromClaims(claims) - // The token's recipe must be a transcode card for the session id in the URL: a - // mismatch is a forged or stale request, and direct/remux cards carry no encode - // parameters to rebuild. An empty PlayMethod is a transcode card (back-compat). - expectedTransportID := card.SessionID - if card.TranscodeTransportID != "" { - expectedTransportID = card.TranscodeTransportID - } - if expectedTransportID != sessionID || (card.PlayMethod != "" && card.PlayMethod != playback.PlayTranscode) { - return nil + var card playback.RecipeCard + tokenComplete := false + if tokenStr := r.Header.Get("X-Silo-Stream-Token"); tokenStr != "" { + cfg := s.watcher.Config() + if cfg == nil { + return nil + } + claims, err := streamtoken.Verify(tokenStr, cfg.Auth.JWTSecret) + if err != nil { + slog.WarnContext(r.Context(), "transcode node reconstruct: invalid stream token", "component", "transcodenode", "error", err, + "session", sessionID, "playback_session_id", sessionID) + return nil + } + card = playback.RecipeCardFromClaims(claims) + // A presented token's recipe must be a transcode card for the session id in + // the URL: a mismatch is a forged or stale request, and direct/remux cards + // carry no encode parameters to rebuild. An empty PlayMethod is a transcode + // card (back-compat). + if !recipeServesTransport(card, sessionID) { + return nil + } + tokenComplete = recipeIsComplete(card) } - // A native token carries the full byte-affecting recipe. The jellycompat node - // hop signs an identity-only token by design (see internal/noderecipe for why), - // so its card decodes with no encode parameters. For the jellycompat case the - // recipe is fetched from the control-plane recipe store below; without that - // store there is nothing to rebuild from, so 404. - tokenComplete := card.SegmentDuration > 0 && card.TargetCodecVideo != "" + // Without a complete token recipe the store is the only remaining source; with + // no store wired there is nothing to rebuild from, so 404. if !tokenComplete && s.recipeStore == nil { return nil } @@ -928,12 +939,13 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request } resolved := card if !tokenComplete { - // Recipe-less (jellycompat) token: fetch the recipe central wrote to the - // control-plane store at transcode start. A miss / incomplete recipe is a - // genuine not-found (404), never a spawn from a bad recipe. + // No complete token recipe (jellycompat's identity-only token, or a + // header-authenticated attempt with no token at all): fetch the recipe + // central wrote to the control-plane store at transcode start. A miss, + // a recipe for another transport, or an incomplete one is a genuine + // not-found (404), never a spawn from a bad recipe. fetched, ok := s.recipeStore.Get(r.Context(), sessionID) - if !ok || fetched == nil || fetched.SessionID != sessionID || - fetched.SegmentDuration <= 0 || fetched.TargetCodecVideo == "" { + if !ok || fetched == nil || !recipeServesTransport(*fetched, sessionID) || !recipeIsComplete(*fetched) { return (*playback.TranscodeSession)(nil), nil } resolved = *fetched @@ -946,6 +958,31 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request return nil } +// recipeServesTransport reports whether a recipe card describes the transcode +// this node serves under transportID — the id in the node-facing URL. +// +// The two writers key that id differently and both shapes are accepted. A native +// v3 remote transcode runs under a plan-scoped transport id (so a prepared +// successor can coexist with its predecessor), recorded on the card as +// TranscodeTransportID; jellycompat runs under the upstream playback session id +// and leaves the field empty, so SessionID is the transport id there. A card +// that matches neither is a forged, stale, or misrouted request. An empty +// PlayMethod counts as a transcode card (back-compat). +func recipeServesTransport(card playback.RecipeCard, transportID string) bool { + expected := card.SessionID + if card.TranscodeTransportID != "" { + expected = card.TranscodeTransportID + } + return expected == transportID && (card.PlayMethod == "" || card.PlayMethod == playback.PlayTranscode) +} + +// recipeIsComplete reports whether a recipe carries the encode parameters +// ffmpeg needs. An identity-only card (the jellycompat node-hop token) is not +// complete and has to be resolved against the control-plane store. +func recipeIsComplete(card playback.RecipeCard) bool { + return card.SegmentDuration > 0 && card.TargetCodecVideo != "" +} + // spawnReconstruct re-spawns ffmpeg for a lost session from its recipe card and // registers it in the live map. It is only ever called inside the per-session // single-flight in reconstructFromToken, so it is the sole writer racing to diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go index 27794e1af..2ef729809 100644 --- a/internal/transcodenode/server_test.go +++ b/internal/transcodenode/server_test.go @@ -731,6 +731,89 @@ func TestReconstructFromToken_JellycompatRecipeFetch(t *testing.T) { }) } +// nativeTransportCard is the shape central stores for a header-authenticated +// remote transcode: the recipe is keyed by the plan-scoped TRANSPORT id the node +// serves it under, which is not the playback session id. +func nativeTransportCard(sessionID, transportID string) *playback.RecipeCard { + return &playback.RecipeCard{ + SessionID: sessionID, + TranscodeTransportID: transportID, + PlayMethod: playback.PlayTranscode, + InputPath: "/media/movie.mkv", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + } +} + +// A header-authenticated attempt publishes no stream token, so nothing forwards +// one to this node — the request that arrives after a node restart carries only +// the static bearer that already authorized it. The stored recipe is then the +// only reconstruct source, and it is keyed by the transport id in the URL, so +// the node must accept the native (TranscodeTransportID) card shape too. +func TestReconstructFromToken_TokenlessRebuildsFromTheStoredTransportRecipe(t *testing.T) { + const sessionID = "sess-tokenless-1" + const transportID = sessionID + "-plan1234-abcd1234" + + s := newTestServer(t) + s.tracker = nodesessions.NewTracker(nil, "http://node", "node", "transcode") + ffmpegPath := filepath.Join(t.TempDir(), "looping-ffmpeg.sh") + if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nwhile :; do sleep 0.1; done\n"), 0o755); err != nil { + t.Fatal(err) + } + s.watcher.Config().Playback.FFmpegPath = ffmpegPath + store := &stubRecipeStore{ok: true, card: nativeTransportCard(sessionID, transportID)} + s.SetRecipeStore(store) + + session := s.reconstructFromToken(requestWithToken(transportID, ""), transportID, 5) + if session == nil { + t.Fatal("tokenless request did not reconstruct; a node restart would 404 this session until the client replans") + } + defer session.CloseProcess() + if store.hits != 1 { + t.Fatalf("recipe store consulted %d times, want 1", store.hits) + } + if got := session.Opts().SessionID; got != transportID { + t.Fatalf("rebuilt session id = %q, want the transport id %q the node serves under", got, transportID) + } + if got := session.Opts().StartSegmentNumber; got != 5 { + t.Fatalf("rebuilt start segment = %d, want the segment the client is fetching", got) + } +} + +// Without a recipe to rebuild from, a tokenless request is still a genuine +// not-found: the node must never spawn ffmpeg on a guess. +func TestReconstructFromToken_TokenlessWithoutARecipeIsNotFound(t *testing.T) { + const transportID = "sess-tokenless-2-plan1234-abcd1234" + + t.Run("no recipe store wired", func(t *testing.T) { + s := newTestServer(t) + if got := s.reconstructFromToken(requestWithToken(transportID, ""), transportID, 5); got != nil { + t.Fatalf("expected nil without a recipe store, got %v", got) + } + }) + + t.Run("store miss", func(t *testing.T) { + s := newTestServer(t) + store := &stubRecipeStore{ok: false} + s.SetRecipeStore(store) + if got := s.reconstructFromToken(requestWithToken(transportID, ""), transportID, 5); got != nil { + t.Fatalf("expected nil on store miss, got %v", got) + } + if store.hits != 1 { + t.Fatalf("recipe store consulted %d times, want 1", store.hits) + } + }) + + t.Run("stored recipe for another transport", func(t *testing.T) { + s := newTestServer(t) + s.SetRecipeStore(&stubRecipeStore{ok: true, card: nativeTransportCard("sess-tokenless-2", "some-other-transport")}) + if got := s.reconstructFromToken(requestWithToken(transportID, ""), transportID, 5); got != nil { + t.Fatalf("expected nil for a recipe keyed to another transport, got %v", got) + } + }) +} + // handleStop is a deliberate teardown, so it must drop the session's recipe to // stop a buffered/retrying post-restart request from reconstructing a brand-new // ffmpeg for an already-stopped session. A zero-value TranscodeSession needs no From 413ec6bf26eb5562188661bb4bd41a80c99fc737 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:32:33 -0400 Subject: [PATCH 31/44] test(transcodenode): check CloseProcess error in tokenless reconstruct test golangci-lint errcheck failed CI on the new changed line. Co-Authored-By: Claude Fable 5 --- internal/transcodenode/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go index 2ef729809..b845d7711 100644 --- a/internal/transcodenode/server_test.go +++ b/internal/transcodenode/server_test.go @@ -769,7 +769,7 @@ func TestReconstructFromToken_TokenlessRebuildsFromTheStoredTransportRecipe(t *t if session == nil { t.Fatal("tokenless request did not reconstruct; a node restart would 404 this session until the client replans") } - defer session.CloseProcess() + defer func() { _ = session.CloseProcess() }() if store.hits != 1 { t.Fatalf("recipe store consulted %d times, want 1", store.hits) } From 080c577d302e0568935b0018e63725c8db271e3d Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:01:54 -0400 Subject: [PATCH 32/44] fix(streamtelemetry): enrol tokenless /stream/v3 proxy routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge left the five credential-free grant routes registered but unclassified, so bytes served through authorized_media_origins_v1 were invisible to stream telemetry. Enrol them: - Declare GET+HEAD /stream/v3/{session_id} (playback), GET+HEAD .../master.m3u8 (manifest) and GET .../segment/{name} (playback), all viewer egress and capability-relevant, and wrap each registration in observeProxy. - Give them CanonicalSessionKey "verified_media_grant" rather than the "verified_stream_token" the proxyRoute helper hardcodes. The field is descriptive — it is only compared in sameDeclaration and emitted into the route manifest, and no code branches on its value — but these routes prove entitlement with a Redis grant plus the caller's own bearer token, never a stream token, so labelling them otherwise would be false. - Attach the viewer in relayGrantToTranscodeNode, the single path both grant transcode handlers take. The proxy->node hop itself stays internal_relay. Co-Authored-By: Claude Fable 5 --- internal/proxy/media_routes.go | 21 ++++++++++++++++++++- internal/proxy/mediagrant.go | 4 ++++ internal/proxy/server.go | 10 +++++----- internal/proxy/testdata/media_routes.txt | 20 ++++++++++---------- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/internal/proxy/media_routes.go b/internal/proxy/media_routes.go index f26cf76f4..98bb3895d 100644 --- a/internal/proxy/media_routes.go +++ b/internal/proxy/media_routes.go @@ -16,6 +16,14 @@ var proxyMediaRoutes = []streamtelemetry.MediaRoute{ proxyRoute(http.MethodGet, "/stream/transcode/{token}/master.m3u8", streamtelemetry.ClassManifest, true), proxyRoute(http.MethodHead, "/stream/transcode/{token}/master.m3u8", streamtelemetry.ClassManifest, true), proxyRoute(http.MethodGet, "/stream/transcode/{token}/segment/{name}", streamtelemetry.ClassPlayback, true), + // authorized_media_origins_v1: same viewer egress, different proof of + // entitlement — a Redis grant plus the caller's own bearer token, never a + // stream token — so these carry their own canonical session key. + grantRoute(http.MethodGet, "/stream/v3/{session_id}", streamtelemetry.ClassPlayback, true), + grantRoute(http.MethodHead, "/stream/v3/{session_id}", streamtelemetry.ClassPlayback, true), + grantRoute(http.MethodGet, "/stream/v3/{session_id}/master.m3u8", streamtelemetry.ClassManifest, true), + grantRoute(http.MethodHead, "/stream/v3/{session_id}/master.m3u8", streamtelemetry.ClassManifest, true), + grantRoute(http.MethodGet, "/stream/v3/{session_id}/segment/{name}", streamtelemetry.ClassPlayback, true), proxyRoute(http.MethodGet, "/stream/subtitles/{token}/{track}", streamtelemetry.ClassPlayback, true), proxyRoute(http.MethodGet, "/stream/subtitles/{token}/{track}/fonts", streamtelemetry.ClassPlayback, true), proxyRoute(http.MethodGet, "/downloads/file/{token}", streamtelemetry.ClassTransfer, false), @@ -23,8 +31,19 @@ var proxyMediaRoutes = []streamtelemetry.MediaRoute{ } func proxyRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { + return proxyRouteWithKey(method, pattern, class, capRelevant, "verified_stream_token") +} + +// grantRoute declares a credential-free /stream/v3 route. It is viewer egress +// like every other proxy media route; only the session key differs, because the +// identity comes from an authorized grant rather than a verified stream token. +func grantRoute(method, pattern string, class streamtelemetry.Class, capRelevant bool) streamtelemetry.MediaRoute { + return proxyRouteWithKey(method, pattern, class, capRelevant, "verified_media_grant") +} + +func proxyRouteWithKey(method, pattern string, class streamtelemetry.Class, capRelevant bool, sessionKey string) streamtelemetry.MediaRoute { return streamtelemetry.MediaRoute{Family: streamtelemetry.FamilyProxy, Method: method, Pattern: pattern, - Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: "verified_stream_token", + Class: class, Role: streamtelemetry.RoleViewerEgress, CanonicalSessionKey: sessionKey, CapRelevant: capRelevant, Enrolled: true, Capture: proxyCapture(pattern)} } diff --git a/internal/proxy/mediagrant.go b/internal/proxy/mediagrant.go index 2997c4772..4baaf1e34 100644 --- a/internal/proxy/mediagrant.go +++ b/internal/proxy/mediagrant.go @@ -174,6 +174,10 @@ func (s *Server) handleGrantTranscodeSegment(w http.ResponseWriter, r *http.Requ // after its own restart), so the credential the client was promised it would // never see stays strictly on the proxy→node hop. func (s *Server) relayGrantToTranscodeNode(w http.ResponseWriter, r *http.Request, claims *streamtoken.Claims, path string) { + // The token transcode routes attach in their handlers; this is the only path + // the two grant transcode handlers take, so attaching once here is the + // equivalent hook. The proxy→node hop itself stays internal_relay. + attachStream(r.Context(), claims) cfg := s.watcher.Config() forwardToken := "" if cfg != nil && cfg.Auth.JWTSecret != "" { diff --git a/internal/proxy/server.go b/internal/proxy/server.go index c08fb1911..4e3486c85 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -167,11 +167,11 @@ func (s *Server) Handler() http.Handler { // Credential-free grant routes (authorized_media_origins_v1). Same media // bytes as the token routes above, addressed by session id and // authorized by the caller's own Authorization header. - r.Head("/stream/v3/{session_id}", s.handleGrantIdentity) - r.Get("/stream/v3/{session_id}", s.handleGrantIdentity) - r.Head("/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest) - r.Get("/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest) - r.Get("/stream/v3/{session_id}/segment/{name}", s.handleGrantTranscodeSegment) + r.Head("/stream/v3/{session_id}", observeProxy(s.telemetry, http.MethodHead, "/stream/v3/{session_id}", s.handleGrantIdentity)) + r.Get("/stream/v3/{session_id}", observeProxy(s.telemetry, http.MethodGet, "/stream/v3/{session_id}", s.handleGrantIdentity)) + r.Head("/stream/v3/{session_id}/master.m3u8", observeProxy(s.telemetry, http.MethodHead, "/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest)) + r.Get("/stream/v3/{session_id}/master.m3u8", observeProxy(s.telemetry, http.MethodGet, "/stream/v3/{session_id}/master.m3u8", s.handleGrantTranscodeManifest)) + r.Get("/stream/v3/{session_id}/segment/{name}", observeProxy(s.telemetry, http.MethodGet, "/stream/v3/{session_id}/segment/{name}", s.handleGrantTranscodeSegment)) r.Get("/stream/subtitles/{token}/{track}/fonts", observeProxy(s.telemetry, http.MethodGet, "/stream/subtitles/{token}/{track}/fonts", s.handleSubtitleFonts)) r.Get("/stream/subtitles/{token}/{track}", observeProxy(s.telemetry, http.MethodGet, "/stream/subtitles/{token}/{track}", s.handleSubtitle)) r.Head("/downloads/file/{token}", observeProxy(s.telemetry, http.MethodHead, "/downloads/file/{token}", s.handleDownloadFile)) diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index 9dcc83db6..068ccf770 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -14,11 +14,11 @@ GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true tr GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true true -GET /stream/v3/{session_id} non-media -HEAD /stream/v3/{session_id} non-media -GET /stream/v3/{session_id}/master.m3u8 non-media -HEAD /stream/v3/{session_id}/master.m3u8 non-media -GET /stream/v3/{session_id}/segment/{name} non-media +GET /stream/v3/{session_id} media playback viewer_egress true true +HEAD /stream/v3/{session_id} media playback viewer_egress true true +GET /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true +HEAD /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true +GET /stream/v3/{session_id}/segment/{name} media playback viewer_egress true true # fixture 2 POST /admin/force-reload non-media GET /api/v1/health non-media @@ -35,8 +35,8 @@ GET /stream/subtitles/{token}/{track}/fonts media playback viewer_egress true tr GET /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true HEAD /stream/transcode/{token}/master.m3u8 media manifest viewer_egress true true GET /stream/transcode/{token}/segment/{name} media playback viewer_egress true true -GET /stream/v3/{session_id} non-media -HEAD /stream/v3/{session_id} non-media -GET /stream/v3/{session_id}/master.m3u8 non-media -HEAD /stream/v3/{session_id}/master.m3u8 non-media -GET /stream/v3/{session_id}/segment/{name} non-media +GET /stream/v3/{session_id} media playback viewer_egress true true +HEAD /stream/v3/{session_id} media playback viewer_egress true true +GET /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true +HEAD /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true +GET /stream/v3/{session_id}/segment/{name} media playback viewer_egress true true From 003e9baef0d7c0319795b3e24a0e728f6da37107 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:53:08 -0400 Subject: [PATCH 33/44] feat(streamtelemetry): enable by default and derive distributed mode from redis Stream telemetry measured nothing on a fresh install: both switches were opt-in, so the parity comparison every P1 threshold depends on only ever ran where someone had already read the design doc. Observation is process-local, off the hot path and bounded, so the safer default is on. SILO_STREAM_TELEMETRY_ENABLED now defaults to true and is a per-process kill switch; SILO_STREAM_TELEMETRY_FAMILIES still narrows observation or drops one misbehaving family without losing the rest. SILO_STREAM_TELEMETRY_DISTRIBUTED is no longer a flag the operator has to keep in sync with their topology: unset, the mode follows whether Redis is configured, so a single-container install stays on LocalStore and a cluster merges. Setting it pins the mode either way, and a rejected distributed configuration pins it off so the derivation cannot re-enable exactly what was just refused. Both switches read a set-but-unparseable value as false rather than as the default (envutil.BoolDefault). For a default-on flag that means a typo in the kill switch turns telemetry OFF, which is the fail-safe direction: the operator was reaching for "stop observing", and a mistyped disable that quietly left the feature running is the failure that costs them. Co-Authored-By: Claude Fable 5 --- cmd/silo/main.go | 17 +++- docs/admin-api.md | 5 ++ docs/design/2026-08-17-stream-telemetry.md | 35 ++++++--- docs/feature-changelog.md | 1 + internal/envutil/bool.go | 23 ++++++ internal/envutil/bool_test.go | 90 ++++++++++++++++++++++ internal/streamtelemetry/config.go | 43 +++++++++-- internal/streamtelemetry/config_test.go | 72 ++++++++++++++++- internal/streamtelemetry/registry_test.go | 38 +++++++++ 9 files changed, 304 insertions(+), 20 deletions(-) create mode 100644 internal/envutil/bool_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 0be99bb34..5f7574b75 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -171,12 +171,21 @@ func registerClientIPConfigReload(watcher *nodeconfig.Watcher, resolver *clienti } // newStreamTelemetryRegistry builds the telemetry registry for this process, -// preferring the Redis-backed store in distributed mode. It never falls back to -// LocalStore on a failed ping: cache.NewRedisClient builds a lazy client that -// never dials, and a Redis restart mid-deploy must not strand a publisher -// local-only for the life of the process. +// preferring the Redis-backed store in distributed mode. Distributed mode is +// derived from whether Redis is configured unless the operator pinned +// SILO_STREAM_TELEMETRY_DISTRIBUTED: a single-process deployment then stays on +// the local store and a clustered one merges, without either being asked to set +// a variable that only restates its own topology. Every process builds its +// registry here, so the derivation belongs in this function rather than at the +// call sites. It never falls back to LocalStore on a failed ping: +// cache.NewRedisClient builds a lazy client that never dials, and a Redis +// restart mid-deploy must not strand a publisher local-only for the life of the +// process. func newStreamTelemetryRegistry(ctx context.Context, nodeID string, redisClient *redis.Client) *streamtelemetry.Registry { streamTelemetryConfig := streamtelemetry.ConfigFromEnv(nodeID) + if !streamTelemetryConfig.DistributedExplicit { + streamTelemetryConfig.Distributed = redisClient != nil + } store := streamtelemetry.GlobalSnapshotStore(streamtelemetry.NewLocalStore()) if streamTelemetryConfig.Enabled && streamTelemetryConfig.Distributed { if redisClient != nil { diff --git a/docs/admin-api.md b/docs/admin-api.md index 57ab7beb0..f29fb8b46 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -21,6 +21,11 @@ session. Design: [`docs/design/2026-08-17-stream-telemetry.md`](design/2026-08-1 The view is served from a bounded-staleness cache with single-flight refresh, so several admins polling this route pay at most one rebuild per TTL. +Stream telemetry runs by default, so this route reports on an unconfigured +server. An `enabled: false` body means this process was switched off with +`SILO_STREAM_TELEMETRY_ENABLED=false`, or that a bad core setting disabled it — +the startup log names the variable in that case. + ### Response Always `200 OK`. "Nothing to compare" is expressed in the body rather than as an diff --git a/docs/design/2026-08-17-stream-telemetry.md b/docs/design/2026-08-17-stream-telemetry.md index edff5867a..cb838285c 100644 --- a/docs/design/2026-08-17-stream-telemetry.md +++ b/docs/design/2026-08-17-stream-telemetry.md @@ -585,6 +585,10 @@ native, so defaulting them on would widen instrumentation across two more live b paths on upgrade alone. "Set the variable before deploying" is a runbook, not a safe default. +Since `SILO_STREAM_TELEMETRY_ENABLED` now defaults on, this gate is what keeps the +shared-process families off: the master switch decides whether a process observes at +all, and the family list decides how far that observation reaches. + **Rollout.** Name a shared-process family explicitly to enable it, one at a time. The same variable is the kill switch — drop one misbehaving family without losing the rest. The resolved set is logged at startup. An unrecognised name disables telemetry entirely @@ -894,16 +898,24 @@ All settings are read once at startup. Invalid **core** settings disable telemet log the offending variable as an error; invalid **distributed-only** settings disable distributed mode while leaving local observation running. +Telemetry runs unless it is switched off, and distributed mode follows the deployment: +`SILO_STREAM_TELEMETRY_ENABLED=false` is the per-process kill switch, +`SILO_STREAM_TELEMETRY_FAMILIES` narrows or kills individual families, and the merge is +used whenever Redis is configured. Both switches fail towards off — a value that is set +but cannot be parsed is treated as `false`, not as the default, because an operator who +mistypes a kill switch was reaching for "stop", and a mistyped disable that quietly left +the feature running is the failure that costs them. + | Variable | Default | Scope | Meaning | |---|---:|---|---| -| `SILO_STREAM_TELEMETRY_ENABLED` | `false` | core | Master switch, per process. | +| `SILO_STREAM_TELEMETRY_ENABLED` | `true` | core | Master switch, per process. Set it to `false` to stop observing; a value that cannot be parsed also reads as off. | | `SILO_STREAM_TELEMETRY_FAMILIES` | `native,proxy,transcode_node` | core | Which route families are wrapped. Also the kill switch. | | `SILO_STREAM_TELEMETRY_SWEEP_INTERVAL` | `1s` | core | Collector period. | | `SILO_STREAM_TELEMETRY_RETENTION` | `5m` | core | How long a session survives its last observation. | | `SILO_STREAM_TELEMETRY_MAX_SESSIONS` | `10000` | core | Local session cap. | | `SILO_STREAM_TELEMETRY_MAX_TRANSFERS` | `10000` | core | Local transfer cap. | | `SILO_STREAM_TELEMETRY_MAX_OBSERVATIONS` | `50000` | core | Local in-flight observation cap. | -| `SILO_STREAM_TELEMETRY_DISTRIBUTED` | `false` | distributed | Publish and read snapshots through Redis. | +| `SILO_STREAM_TELEMETRY_DISTRIBUTED` | auto (on when Redis is configured) | distributed | Publish and read snapshots through Redis. Setting it pins the mode either way and stops the derivation; a rejected distributed configuration also pins it off. | | `SILO_STREAM_TELEMETRY_FRESHNESS` | `5s` | distributed | Maximum usable snapshot age; at least three sweep intervals. | | `SILO_STREAM_TELEMETRY_MEMBERSHIP_TTL` | `60s` | distributed | Heartbeat age after which a publisher has departed; must exceed freshness. | | `SILO_STREAM_TELEMETRY_KEY_PREFIX` | `silo:stelem` | distributed | Non-empty, whitespace-free Redis namespace. | @@ -921,20 +933,25 @@ self-heals when Redis returns. ## Operating it -**Turning it on is the next task, and everything in P1 depends on it.** Every remaining -threshold is a guess until the merged view has been compared against what admins see -today. +**It is on.** Every remaining threshold is a guess until the merged view has been +compared against what admins see today, and that comparison only happens at scale if +observation is the default rather than something each deployment has to opt into. A +fresh install observes the default family set, and merges through Redis whenever Redis +is configured; nothing has to be set to get a parity read. ```bash -# 1. default family set: native, proxy, transcode_node -SILO_STREAM_TELEMETRY_ENABLED=true -SILO_STREAM_TELEMETRY_DISTRIBUTED=true +# 1. nothing to set: default family set native, proxy, transcode_node, +# distributed merge on wherever Redis is configured. +curl -fsS localhost:8091/api/v1/admin/stream-telemetry/parity # 2. read repeatedly, over days — one report is a sample, not proof -curl -fsS localhost:8091/api/v1/admin/stream-telemetry/parity # 3. widen one family at a time, only after the previous one is quiet SILO_STREAM_TELEMETRY_FAMILIES=native,proxy,transcode_node,jellycompat + +# 4. back out: kill one family, or the whole process's observation +SILO_STREAM_TELEMETRY_FAMILIES=native,transcode_node +SILO_STREAM_TELEMETRY_ENABLED=false ``` What to watch: diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index a95e472a1..f212624fa 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -10,6 +10,7 @@ Playback protocol v3 now advertises the engine-neutral `authorized_media_origins ### Measure delivered bytes on every serving path Silo now measures what it actually sends, rather than trusting what a client reports it is watching. - Every byte-serving route across the API server, Jellyfin-compatibility layer, standalone proxy, audiobook listener and transcode nodes reports what it served, to whom and how fast, off the hot path. +- Measurement is on by default and needs no configuration. `SILO_STREAM_TELEMETRY_ENABLED=false` is the per-process kill switch, `SILO_STREAM_TELEMETRY_FAMILIES` narrows observation or kills one misbehaving family without losing the rest, and the distributed merge turns itself on wherever Redis is configured, so a single-node install measures locally and a cluster merges without either setting a variable. - Adds `GET /api/v1/admin/stream-telemetry/parity`, which puts the merged measurement beside the two live-session views admins read today and diffs them. See [docs/admin-api.md](admin-api.md). - Makes no decisions: nothing is blocked, throttled or ended, and no existing admin view was repointed onto it. - Fixes four defects on the byte paths themselves — proxied streams recorded against no owner, the proxy's own address recorded as the viewer's, the kernel sendfile fast path dead through the proxy chain, and stream tokens with no reliable creation time. diff --git a/internal/envutil/bool.go b/internal/envutil/bool.go index 3f180823f..0cab8de15 100644 --- a/internal/envutil/bool.go +++ b/internal/envutil/bool.go @@ -27,3 +27,26 @@ func Truthy(value string) bool { // Bool reports whether the named environment variable is set to a truthy value. func Bool(name string) bool { return Truthy(os.Getenv(name)) } + +// BoolDefault reports whether the named environment variable is on, falling back +// to def when the variable is unset or empty — whitespace-only counts as empty, +// since a value that survives a shell only as spaces was never really supplied. +// +// A value that IS present but unrecognised ("flase", "no", "0") reads as false +// rather than as def. For a flag that defaults on, that means a typo in the kill +// switch turns the flag OFF, which is the fail-safe direction: the operator was +// reaching for "off", and a mistyped disable that silently left the feature +// running is the failure that actually hurts. +func BoolDefault(name string, def bool) bool { + if !IsSet(name) { + return def + } + return Truthy(os.Getenv(name)) +} + +// IsSet reports whether the named environment variable carries a non-empty value +// once surrounding whitespace is trimmed. It answers "did the operator touch this +// knob?", which a default-on flag has to ask separately from "is it on?" — an +// unset knob and one explicitly set to false want different behaviour when +// something else would otherwise derive the value. +func IsSet(name string) bool { return strings.TrimSpace(os.Getenv(name)) != "" } diff --git a/internal/envutil/bool_test.go b/internal/envutil/bool_test.go new file mode 100644 index 000000000..56d82028b --- /dev/null +++ b/internal/envutil/bool_test.go @@ -0,0 +1,90 @@ +package envutil + +import "testing" + +const testEnv = "SILO_ENVUTIL_TEST_FLAG" + +func TestTruthy(t *testing.T) { + for _, test := range []struct { + value string + want bool + }{ + {"1", true}, {"true", true}, {"TRUE", true}, {" True ", true}, + {"yes", true}, {"on", true}, {"enabled", true}, + {"", false}, {" ", false}, {"0", false}, {"false", false}, + {"no", false}, {"off", false}, {"disabled", false}, {"flase", false}, + } { + if got := Truthy(test.value); got != test.want { + t.Errorf("Truthy(%q) = %v, want %v", test.value, got, test.want) + } + } +} + +func TestBool(t *testing.T) { + t.Run("unset", func(t *testing.T) { + if Bool(testEnv) { + t.Fatal("unset variable read as on") + } + }) + t.Run("set", func(t *testing.T) { + t.Setenv(testEnv, "yes") + if !Bool(testEnv) { + t.Fatal("truthy variable read as off") + } + }) +} + +// A default-on flag is only safe if a garbled value lands on "off": the operator +// who mistyped the kill switch was reaching for off, not for the default. +func TestBoolDefault(t *testing.T) { + for _, test := range []struct { + name string + value string + set bool + def bool + want bool + }{ + {"unset keeps a true default", "", false, true, true}, + {"unset keeps a false default", "", false, false, false}, + {"empty keeps the default", "", true, true, true}, + {"whitespace keeps the default", " ", true, true, true}, + {"explicit false overrides a true default", "false", true, true, false}, + {"explicit true overrides a false default", "true", true, false, true}, + {"malformed value reads as off under a true default", "flase", true, true, false}, + {"malformed value reads as off under a false default", "flase", true, false, false}, + } { + t.Run(test.name, func(t *testing.T) { + if test.set { + t.Setenv(testEnv, test.value) + } + if got := BoolDefault(testEnv, test.def); got != test.want { + t.Fatalf("BoolDefault(%q, %v) = %v, want %v", test.value, test.def, got, test.want) + } + }) + } +} + +func TestIsSet(t *testing.T) { + for _, test := range []struct { + name string + value string + set bool + want bool + }{ + {"unset", "", false, false}, + {"empty", "", true, false}, + {"whitespace only", " \t ", true, false}, + {"false is still set", "false", true, true}, + {"true is set", "true", true, true}, + {"garbage is set", "flase", true, true}, + } { + t.Run(test.name, func(t *testing.T) { + if test.set { + t.Setenv(testEnv, test.value) + } + if got := IsSet(testEnv); got != test.want { + t.Fatalf("IsSet(%q) = %v, want %v", test.value, got, test.want) + } + }) + } +} diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index fc7982f48..ab6de008b 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -47,11 +47,23 @@ var defaultObservedFamilies = map[Family]bool{ } type Config struct { + // Enabled turns observation on for this process, and defaults ON: + // SILO_STREAM_TELEMETRY_ENABLED=false is the per-process kill switch. A value + // that is set but unparseable also reads as off (envutil.BoolDefault), so a + // mistyped kill switch fails towards silence rather than towards running. Enabled bool NodeID string PublisherID string PublisherEpoch int64 - Distributed bool + // Distributed publishes and reads snapshots through Redis. ConfigFromEnv only + // reads the variable; when it was not set, wiring derives the mode from + // whether Redis is configured, so a single-process deployment stays local and + // a clustered one merges without either having to name a second variable. + Distributed bool + // DistributedExplicit reports that Distributed is already settled and must not + // be auto-derived — either the operator set SILO_STREAM_TELEMETRY_DISTRIBUTED, + // or an invalid distributed configuration has forced the mode off. + DistributedExplicit bool // Families narrows which route families are observed. Empty means // defaultObservedFamilies. It is a kill switch as much as a rollout control: // one misbehaving family can be dropped without losing all observation. @@ -105,18 +117,24 @@ func DefaultConfig(nodeID string) Config { } } -// ConfigFromEnv returns a safe configuration. Invalid core settings disable -// telemetry; invalid distributed-only settings retain local telemetry. +// ConfigFromEnv returns a safe configuration. Telemetry is on unless +// SILO_STREAM_TELEMETRY_ENABLED turns it off, and distributed mode is left for +// the caller to derive from Redis availability unless the operator pinned +// SILO_STREAM_TELEMETRY_DISTRIBUTED. Invalid core settings disable telemetry; +// invalid distributed-only settings retain local telemetry. func ConfigFromEnv(nodeID string) Config { cfg := DefaultConfig(nodeID) - cfg.Enabled = envutil.Bool(enabledEnv) + cfg.Enabled = envutil.BoolDefault(enabledEnv, true) coreInvalid := make([]string, 0) distributedInvalid := make([]string, 0) // The operator only owns the variables they actually set. The cross-checks // below relate two knobs, and a violation involving an unset knob is not the - // operator's mistake — it is a default that has to move. + // operator's mistake — it is a default that has to move. Only the timing and + // sizing knobs are ever cross-checked; ENABLED and DISTRIBUTED are never + // blamed by crossCheckFailed, so defaulting them on cannot misreport anyone. explicit := make(map[string]bool) cfg.Distributed = envutil.Bool(distributedEnv) + cfg.DistributedExplicit = envutil.IsSet(distributedEnv) parseDuration := func(name string, dst *time.Duration) { value := strings.TrimSpace(os.Getenv(name)) if value == "" { @@ -245,6 +263,10 @@ func ConfigFromEnv(nodeID string) Config { if cfg.MembershipTTL > time.Duration(1<<63-1)/10 { crossCheckFailed(membershipTTLEnv) } + // Telemetry now runs by default, so this error is the common shape of a + // misconfiguration: the operator broke a variable and lost observation they + // never asked for. The warn branch is reserved for a process that had already + // been switched off. if len(coreInvalid) > 0 { if cfg.Enabled { cfg.Enabled = false @@ -254,12 +276,21 @@ func ConfigFromEnv(nodeID string) Config { } } if len(distributedInvalid) > 0 { - if cfg.Distributed { + // Now that the mode is derived, "off" and "would have stayed off" are + // different outcomes: a rejected config that suppresses a merge the + // deployment would otherwise have run is an error, while a process that + // pinned the mode off loses nothing and only needs the noise recorded. + if cfg.Distributed || !cfg.DistributedExplicit { cfg.Distributed = false slog.Error("stream telemetry distributed mode disabled because configuration is invalid", "variables", strings.Join(distributedInvalid, ",")) } else { slog.Warn("ignoring invalid distributed stream telemetry configuration", "variables", strings.Join(distributedInvalid, ",")) } + // A rejected distributed configuration settles the mode as firmly as the + // operator setting the variable would. Without this, the caller's "no + // DISTRIBUTED variable means derive it from Redis" rule would turn + // distributed mode straight back on with the configuration just refused. + cfg.DistributedExplicit = true } return cfg } diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index 103dffc0a..bc93e7345 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -6,14 +6,73 @@ import ( ) func TestConfigFromEnvValidation(t *testing.T) { + // Telemetry observes unless it is switched off, and leaves distributed mode + // unsettled so wiring can derive it from Redis. t.Run("defaults", func(t *testing.T) { clearConfigEnv(t) cfg := ConfigFromEnv("node") - if cfg.Enabled || cfg.Distributed || cfg.SweepInterval != time.Second || cfg.Retention != 5*time.Minute || cfg.MaxObservations != 50_000 || + if !cfg.Enabled || cfg.Distributed || cfg.DistributedExplicit || cfg.SweepInterval != time.Second || cfg.Retention != 5*time.Minute || cfg.MaxObservations != 50_000 || cfg.Freshness != 5*time.Second || cfg.MembershipTTL != time.Minute || cfg.KeyPrefix != "silo:stelem" || cfg.FullResyncEvery != 60 || cfg.MaxPublishers != 256 || cfg.MaxMergedSessions != 50_000 || cfg.MaxMergedTransfers != 50_000 { t.Fatalf("defaults = %+v", cfg) } }) + // The kill switch has to work, and has to fail towards off: an operator who + // mistypes it was reaching for "stop observing", so a value nobody can parse + // stops observation rather than quietly leaving it running. + for _, test := range []struct { + name string + value string + enabled bool + }{ + {"empty stays on", "", true}, + {"whitespace stays on", " ", true}, + {"explicit false kills", "false", false}, + {"explicit off kills", "off", false}, + {"malformed value kills", "flase", false}, + {"explicit true stays on", "true", true}, + } { + t.Run("enabled switch: "+test.name, func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, test.value) + if cfg := ConfigFromEnv("node"); cfg.Enabled != test.enabled { + t.Fatalf("SILO_STREAM_TELEMETRY_ENABLED=%q gave Enabled=%v, want %v", test.value, cfg.Enabled, test.enabled) + } + }) + } + // Wiring derives distributed mode from Redis only when the operator left the + // variable alone, so "set but false" and "unset" must stay distinguishable. + for _, test := range []struct { + name string + value string + distributed bool + explicit bool + }{ + {"unset leaves the mode to the caller", "", false, false}, + {"whitespace leaves the mode to the caller", " ", false, false}, + {"explicit true pins it on", "true", true, true}, + {"explicit false pins it off", "false", false, true}, + {"malformed value pins it off", "flase", false, true}, + } { + t.Run("distributed switch: "+test.name, func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(distributedEnv, test.value) + cfg := ConfigFromEnv("node") + if cfg.Distributed != test.distributed || cfg.DistributedExplicit != test.explicit { + t.Fatalf("SILO_STREAM_TELEMETRY_DISTRIBUTED=%q gave %v/%v, want %v/%v", test.value, cfg.Distributed, cfg.DistributedExplicit, test.distributed, test.explicit) + } + }) + } + // A rejected distributed configuration has to pin the mode off too, or the + // caller's Redis derivation would re-enable exactly what was just refused. + t.Run("rejected distributed config pins the mode off", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(freshnessEnv, "10s") + t.Setenv(membershipTTLEnv, "10s") + cfg := ConfigFromEnv("node") + if !cfg.Enabled || cfg.Distributed || !cfg.DistributedExplicit { + t.Fatalf("config = %+v", cfg) + } + }) t.Run("valid distributed overrides", func(t *testing.T) { clearConfigEnv(t) t.Setenv(enabledEnv, "true") @@ -50,8 +109,17 @@ func TestConfigFromEnvValidation(t *testing.T) { t.Fatalf("invalid config remained enabled: %+v", cfg) } }) + t.Run("invalid core disables the default-on process", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(maxTransfersEnv, "not-a-number") + cfg := ConfigFromEnv("node") + if cfg.Enabled || cfg.MaxTransfers != 10_000 { + t.Fatalf("invalid config remained enabled: %+v", cfg) + } + }) t.Run("invalid disabled is ignored", func(t *testing.T) { clearConfigEnv(t) + t.Setenv(enabledEnv, "false") t.Setenv(maxTransfersEnv, "not-a-number") cfg := ConfigFromEnv("node") if cfg.Enabled || cfg.MaxTransfers != 10_000 { @@ -75,6 +143,8 @@ func TestConfigFromEnvValidation(t *testing.T) { } t.Run("invalid distributed while disabled warns and stays disabled", func(t *testing.T) { clearConfigEnv(t) + t.Setenv(enabledEnv, "false") + t.Setenv(distributedEnv, "false") t.Setenv(maxPublishersEnv, "0") cfg := ConfigFromEnv("node") if cfg.Enabled || cfg.Distributed { diff --git a/internal/streamtelemetry/registry_test.go b/internal/streamtelemetry/registry_test.go index fe224803a..2a6059e80 100644 --- a/internal/streamtelemetry/registry_test.go +++ b/internal/streamtelemetry/registry_test.go @@ -410,6 +410,44 @@ func TestRegistryGlobalView(t *testing.T) { } } +// The single-process, Redis-less deployment is the default one now that +// telemetry ships on: observed traffic has to reach a complete merged view +// through LocalStore alone, with no publisher marked missing or stale. If this +// breaks, every household running one container gets a degraded parity read. +func TestLocalOnlyViewIsCompleteForASinglePublisher(t *testing.T) { + cfg := testConfig() + cfg.Retention = time.Minute + store := NewLocalStore() + registry := NewRegistry(cfg, store, slog.New(slog.DiscardHandler)) + handler := registry.Observe(testRoute(ClassPlayback))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Attach(r.Context(), testAttachment("session-1")) + _, _ = w.Write([]byte("payload")) + })) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/media/x", nil)) + + at := time.Now() + originalNow := now + now = func() time.Time { return at } + defer func() { now = originalNow }() + snapshot := registry.Sweep() + if err := store.Publish(context.Background(), snapshot); err != nil { + t.Fatal(err) + } + view, err := registry.GlobalView(context.Background()) + if err != nil { + t.Fatal(err) + } + if !view.Complete || len(view.IncompleteReasons) != 0 { + t.Fatalf("local-only view degraded: complete=%v reasons=%v", view.Complete, view.IncompleteReasons) + } + if len(view.Publishers) != 1 || len(view.Sessions) != 1 { + t.Fatalf("view = %d publishers, %d sessions", len(view.Publishers), len(view.Sessions)) + } + if session := view.Sessions[0]; session.SessionID != "session-1" || session.ViewerBytesAccepted != 7 { + t.Fatalf("session = %+v", session) + } +} + func TestLocalStoreDeepCopies(t *testing.T) { store := NewLocalStore() source := Snapshot{Sessions: []SessionView{{ViewerIPs: []string{"one"}, Routes: []RouteActivityView{{Pattern: "/one"}}, Outcomes: map[httpstream.StreamOutcome]int64{"completed": 1}}}} From 6301b7e4f4f48f7e581b41eea32e05ca1c7aaf7d Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:05:37 -0400 Subject: [PATCH 34/44] feat(streamtelemetry): observe every route family by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staged per-family rollout set (native, proxy, transcode_node) is removed by owner decision: SILO_STREAM_TELEMETRY_FAMILIES left unset now observes all five declared families (native, jellycompat, proxy, abs, transcode_node) instead of a curated subset. The variable stays as a narrowing/kill knob — naming it takes families away rather than staging them in. Adds streamtelemetry.AllFamilies as the single canonical family list so ObservesFamily and ObservedFamilies don't hand-duplicate it, updates the design doc's family-gate section and env table to match present-tense behavior (keeping the original staged-rollout narrative as history), and updates the feature changelog to say every family is observed out of the box. Co-Authored-By: Claude Fable 5 --- docs/design/2026-08-17-stream-telemetry.md | 57 +++++++++++----------- docs/feature-changelog.md | 2 +- internal/streamtelemetry/config.go | 42 +++++++--------- internal/streamtelemetry/config_test.go | 40 ++++++++++----- internal/streamtelemetry/route.go | 6 +++ 5 files changed, 81 insertions(+), 66 deletions(-) diff --git a/docs/design/2026-08-17-stream-telemetry.md b/docs/design/2026-08-17-stream-telemetry.md index cb838285c..5bb9bfe90 100644 --- a/docs/design/2026-08-17-stream-telemetry.md +++ b/docs/design/2026-08-17-stream-telemetry.md @@ -571,30 +571,28 @@ hole and the ABS proxy-attribution bug are live defects on `main` regardless of project. Four pieces plus one the phase did not originally name — the rolling write deadline's 64 MiB `ReadFrom` slice (§4.4). Details in §4.3 and §4.4. -### P0b — local shadow telemetry ✅ built, all five families enrolled +### P0b — local shadow telemetry ✅ built, all five families enrolled and observed by default Process-local, observation-only. `Observation`, `LogicalPlaybackSession`, `Transfer`, -release-fold, bounded retention (§2.2). One router family at a time, benchmarked before -the next. - -**The family gate.** `SILO_STREAM_TELEMETRY_FAMILIES` defaults to -`native,proxy,transcode_node`. The default is deliberately **not** every family: proxy -and transcode node are separate processes where `SILO_STREAM_TELEMETRY_ENABLED` is -already a per-family switch, while jellycompat and ABS share the API process with -native, so defaulting them on would widen instrumentation across two more live byte -paths on upgrade alone. "Set the variable before deploying" is a runbook, not a safe -default. - -Since `SILO_STREAM_TELEMETRY_ENABLED` now defaults on, this gate is what keeps the -shared-process families off: the master switch decides whether a process observes at -all, and the family list decides how far that observation reaches. - -**Rollout.** Name a shared-process family explicitly to enable it, one at a time. The -same variable is the kill switch — drop one misbehaving family without losing the rest. -The resolved set is logged at startup. An unrecognised name disables telemetry entirely -and names the variable, because a typo that silently observed nothing would be worse -than no telemetry. Once a family has run in production, move it into -`defaultObservedFamilies` in `internal/streamtelemetry/config.go`. +release-fold, bounded retention (§2.2). Originally rolled out one router family at a +time, benchmarked before the next widening (see the soak notes below); the staged +default has since been removed by owner decision, and all five families are now +observed as soon as telemetry is enabled. + +**The family gate.** `SILO_STREAM_TELEMETRY_FAMILIES` defaults to every declared +family — `native`, `proxy`, `transcode_node`, `jellycompat`, `abs`. The variable exists +to narrow observation or drop one misbehaving family without losing the rest, not to +stage a rollout: naming it takes away families rather than adding them. + +Since `SILO_STREAM_TELEMETRY_ENABLED` now defaults on, that master switch decides +whether a process observes at all, and the family list — left unset by default — no +longer restricts how far that observation reaches within a process. + +**Historical rollout (P0 soak).** The initial production rollout named the variable +explicitly and widened it one family at a time — `native`, then `+jellycompat` — the +same variable served as both the staged-rollout control and the kill switch. That +staging discipline is retained below as a record of how the soak was run; it no longer +describes the present-day default, which observes every family unless narrowed. ### P0c — distributed read-only view ✅ built @@ -909,7 +907,7 @@ the feature running is the failure that costs them. | Variable | Default | Scope | Meaning | |---|---:|---|---| | `SILO_STREAM_TELEMETRY_ENABLED` | `true` | core | Master switch, per process. Set it to `false` to stop observing; a value that cannot be parsed also reads as off. | -| `SILO_STREAM_TELEMETRY_FAMILIES` | `native,proxy,transcode_node` | core | Which route families are wrapped. Also the kill switch. | +| `SILO_STREAM_TELEMETRY_FAMILIES` | all five (`native,proxy,transcode_node,jellycompat,abs`) | core | Which route families are wrapped. Narrows or kills observation; naming it takes families away rather than staging them in. | | `SILO_STREAM_TELEMETRY_SWEEP_INTERVAL` | `1s` | core | Collector period. | | `SILO_STREAM_TELEMETRY_RETENTION` | `5m` | core | How long a session survives its last observation. | | `SILO_STREAM_TELEMETRY_MAX_SESSIONS` | `10000` | core | Local session cap. | @@ -936,20 +934,21 @@ self-heals when Redis returns. **It is on.** Every remaining threshold is a guess until the merged view has been compared against what admins see today, and that comparison only happens at scale if observation is the default rather than something each deployment has to opt into. A -fresh install observes the default family set, and merges through Redis whenever Redis +fresh install observes every declared family, and merges through Redis whenever Redis is configured; nothing has to be set to get a parity read. ```bash -# 1. nothing to set: default family set native, proxy, transcode_node, -# distributed merge on wherever Redis is configured. +# 1. nothing to set: every family observed by default (native, proxy, +# transcode_node, jellycompat, abs), distributed merge on wherever Redis is +# configured. curl -fsS localhost:8091/api/v1/admin/stream-telemetry/parity # 2. read repeatedly, over days — one report is a sample, not proof -# 3. widen one family at a time, only after the previous one is quiet -SILO_STREAM_TELEMETRY_FAMILIES=native,proxy,transcode_node,jellycompat +# 3. narrow to specific families, or drop one that is misbehaving +SILO_STREAM_TELEMETRY_FAMILIES=native,proxy,transcode_node -# 4. back out: kill one family, or the whole process's observation +# 4. back out further, or kill the whole process's observation SILO_STREAM_TELEMETRY_FAMILIES=native,transcode_node SILO_STREAM_TELEMETRY_ENABLED=false ``` diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index f212624fa..d88a3b17f 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -10,7 +10,7 @@ Playback protocol v3 now advertises the engine-neutral `authorized_media_origins ### Measure delivered bytes on every serving path Silo now measures what it actually sends, rather than trusting what a client reports it is watching. - Every byte-serving route across the API server, Jellyfin-compatibility layer, standalone proxy, audiobook listener and transcode nodes reports what it served, to whom and how fast, off the hot path. -- Measurement is on by default and needs no configuration. `SILO_STREAM_TELEMETRY_ENABLED=false` is the per-process kill switch, `SILO_STREAM_TELEMETRY_FAMILIES` narrows observation or kills one misbehaving family without losing the rest, and the distributed merge turns itself on wherever Redis is configured, so a single-node install measures locally and a cluster merges without either setting a variable. +- Measurement is on by default and needs no configuration: every media route family — native, jellycompat, proxy, audiobooks and transcode nodes — is observed out of the box. `SILO_STREAM_TELEMETRY_ENABLED=false` is the per-process kill switch, `SILO_STREAM_TELEMETRY_FAMILIES` narrows observation or kills one misbehaving family without losing the rest, and the distributed merge turns itself on wherever Redis is configured, so a single-node install measures locally and a cluster merges without either setting a variable. - Adds `GET /api/v1/admin/stream-telemetry/parity`, which puts the merged measurement beside the two live-session views admins read today and diffs them. See [docs/admin-api.md](admin-api.md). - Makes no decisions: nothing is blocked, throttled or ended, and no existing admin view was repointed onto it. - Fixes four defects on the byte paths themselves — proxied streams recorded against no owner, the proxy's own address recorded as the viewer's, the kernel sendfile fast path dead through the proxy chain, and stream tokens with no reliable creation time. diff --git a/internal/streamtelemetry/config.go b/internal/streamtelemetry/config.go index ab6de008b..08d7768ca 100644 --- a/internal/streamtelemetry/config.go +++ b/internal/streamtelemetry/config.go @@ -31,21 +31,6 @@ const ( viewTTLEnv = "SILO_STREAM_TELEMETRY_VIEW_TTL" ) -// defaultObservedFamilies is the set observed when SILO_STREAM_TELEMETRY_FAMILIES -// is unset. It is deliberately NOT "every declared family": jellycompat and ABS -// share the API process with native, so defaulting them on would widen -// instrumentation across a live byte path on upgrade alone, which is exactly what -// §6's one-family-at-a-time rollout exists to prevent. Proxy and transcode node -// are separate processes, so their own SILO_STREAM_TELEMETRY_ENABLED already gates -// them and they stay in the default set. Name a family in the variable to observe -// it; move it in here once it has run in production, and delete this set when all -// five have. -var defaultObservedFamilies = map[Family]bool{ - FamilyNative: true, - FamilyProxy: true, - FamilyTranscodeNode: true, -} - type Config struct { // Enabled turns observation on for this process, and defaults ON: // SILO_STREAM_TELEMETRY_ENABLED=false is the per-process kill switch. A value @@ -64,9 +49,10 @@ type Config struct { // be auto-derived — either the operator set SILO_STREAM_TELEMETRY_DISTRIBUTED, // or an invalid distributed configuration has forced the mode off. DistributedExplicit bool - // Families narrows which route families are observed. Empty means - // defaultObservedFamilies. It is a kill switch as much as a rollout control: - // one misbehaving family can be dropped without losing all observation. + // Families narrows which route families are observed. Empty means every + // declared family (AllFamilies) — observation is on for all five by default. + // The variable exists to narrow observation or drop one misbehaving family + // without losing the rest; it is a kill switch, not a staged rollout. Families map[Family]bool SweepInterval time.Duration @@ -296,10 +282,12 @@ func ConfigFromEnv(nodeID string) Config { } // ObservesFamily reports whether routes in this family are wrapped. It is read -// once per route at mount time, never on the hot path. +// once per route at mount time, never on the hot path. An unset +// SILO_STREAM_TELEMETRY_FAMILIES observes every declared family; naming the +// variable narrows or kills observation from there. func (c Config) ObservesFamily(family Family) bool { if len(c.Families) == 0 { - return defaultObservedFamilies[family] + return true } return c.Families[family] } @@ -307,12 +295,16 @@ func (c Config) ObservesFamily(family Family) bool { // ObservedFamilies lists the observed families in a stable order, for the // startup log that makes the resolved set visible. func (c Config) ObservedFamilies() []string { - set := c.Families - if len(set) == 0 { - set = defaultObservedFamilies + if len(c.Families) == 0 { + names := make([]string, 0, len(AllFamilies)) + for _, family := range AllFamilies { + names = append(names, string(family)) + } + sort.Strings(names) + return names } - names := make([]string, 0, len(set)) - for family, observed := range set { + names := make([]string, 0, len(c.Families)) + for family, observed := range c.Families { if observed { names = append(names, string(family)) } diff --git a/internal/streamtelemetry/config_test.go b/internal/streamtelemetry/config_test.go index bc93e7345..3cc04bcc9 100644 --- a/internal/streamtelemetry/config_test.go +++ b/internal/streamtelemetry/config_test.go @@ -258,28 +258,46 @@ func clearConfigEnv(t *testing.T) { } } -// The family gate is what makes a shared-process family (jellycompat, ABS) safe -// to enroll: it is both the staged-rollout control §6 asks for and a kill switch -// for one misbehaving family that keeps the rest observing. +// The family gate is a narrowing/kill switch, not a staged rollout: every +// declared family is observed by default, and naming +// SILO_STREAM_TELEMETRY_FAMILIES narrows observation or drops one misbehaving +// family while the rest keep observing. func TestConfigFamilyGate(t *testing.T) { - t.Run("unset observes the shipped set only", func(t *testing.T) { + t.Run("unset observes every declared family", func(t *testing.T) { clearConfigEnv(t) cfg := ConfigFromEnv("node") if len(cfg.Families) != 0 { t.Fatalf("families = %+v, want unset", cfg.Families) } - for _, family := range []Family{FamilyNative, FamilyProxy, FamilyTranscodeNode} { + for _, family := range AllFamilies { if !cfg.ObservesFamily(family) { t.Fatalf("%s not observed by default", family) } } - // Widening the default would instrument two more live byte paths in the - // API process on upgrade alone. That has to be an explicit opt-in. - for _, family := range []Family{FamilyJellycompat, FamilyABS} { + if got := cfg.ObservedFamilies(); len(got) != 5 || + got[0] != "abs" || got[1] != "jellycompat" || got[2] != "native" || got[3] != "proxy" || got[4] != "transcode_node" { + t.Fatalf("observed families = %v", got) + } + }) + t.Run("FAMILIES=proxy observes only proxy", func(t *testing.T) { + clearConfigEnv(t) + t.Setenv(enabledEnv, "true") + t.Setenv(familiesEnv, "proxy") + cfg := ConfigFromEnv("node") + if !cfg.Enabled { + t.Fatalf("config = %+v", cfg) + } + if !cfg.ObservesFamily(FamilyProxy) { + t.Fatal("proxy not observed") + } + for _, family := range []Family{FamilyNative, FamilyJellycompat, FamilyABS, FamilyTranscodeNode} { if cfg.ObservesFamily(family) { - t.Fatalf("%s observed by default", family) + t.Fatalf("%s observed despite FAMILIES=proxy", family) } } + if got := cfg.ObservedFamilies(); len(got) != 1 || got[0] != "proxy" { + t.Fatalf("observed families = %v", got) + } }) t.Run("explicit list narrows and widens", func(t *testing.T) { clearConfigEnv(t) @@ -312,12 +330,12 @@ func TestConfigFamilyGate(t *testing.T) { t.Fatal("a typo in the family list must disable telemetry rather than silently observe nothing") } }) - t.Run("only whitespace falls back to the default set", func(t *testing.T) { + t.Run("only whitespace falls back to observing every family", func(t *testing.T) { clearConfigEnv(t) t.Setenv(enabledEnv, "true") t.Setenv(familiesEnv, " , ") cfg := ConfigFromEnv("node") - if !cfg.Enabled || !cfg.ObservesFamily(FamilyNative) || cfg.ObservesFamily(FamilyABS) { + if !cfg.Enabled || !cfg.ObservesFamily(FamilyNative) || !cfg.ObservesFamily(FamilyABS) { t.Fatalf("config = %+v", cfg) } }) diff --git a/internal/streamtelemetry/route.go b/internal/streamtelemetry/route.go index e2b041204..c6028ed29 100644 --- a/internal/streamtelemetry/route.go +++ b/internal/streamtelemetry/route.go @@ -32,6 +32,12 @@ const ( RoleProducer Role = "producer" ) +// AllFamilies lists every declared route family, in stable sorted order. It is +// the canonical set Config.ObservesFamily and Config.ObservedFamilies fall +// back to when SILO_STREAM_TELEMETRY_FAMILIES is unset, so the five families +// are named once rather than duplicated across both functions. +var AllFamilies = []Family{FamilyABS, FamilyJellycompat, FamilyNative, FamilyProxy, FamilyTranscodeNode} + type MediaRoute struct { Family Family Method string From 5af2a097f681bba0bd21faf97a74e4984798cc7e Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:46:50 -0400 Subject: [PATCH 35/44] feat(scanner): persist H.264 copy-safety verdicts and move analysis off browse paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-PPS copy-safety scan ran on media-page load and was forgotten on every restart, re-reading the opening seconds of every browsed H.264 file — painfully slow on remote storage. The verdict is now persisted on media_files (self-validating against file size+mtime, so in-place rewrites invalidate it without writer coordination), the scan window drops from 15s to 5s, browse pages never trigger the scan (EnsureProbeOnly), and concurrent first scans share one ffmpeg via singleflight. The lazy path stays fail-closed and stateless on errors. Related issue: N/A — narrow fix Co-Authored-By: Claude Fable 5 --- docs/feature-changelog.md | 7 + internal/api/handlers/playback.go | 7 + internal/catalog/detail.go | 33 +- internal/catalog/detail_prepare_files_test.go | 78 ++++ internal/chapterthumbs/service.go | 7 +- internal/chapterthumbs/service_test.go | 2 +- internal/models/media.go | 23 +- internal/scanner/file_repo.go | 37 ++ internal/scanner/pps.go | 9 +- internal/scanner/probe_repair.go | 166 +++++++- .../probe_repair_copy_safety_persist_test.go | 380 ++++++++++++++++++ ...823182731_persist_multiple_pps_verdict.sql | 19 + 12 files changed, 730 insertions(+), 38 deletions(-) create mode 100644 internal/catalog/detail_prepare_files_test.go create mode 100644 internal/scanner/probe_repair_copy_safety_persist_test.go create mode 100644 migrations/sql/20260823182731_persist_multiple_pps_verdict.sql diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index d88a3b17f..edcf76943 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -2,6 +2,13 @@ ## 2026-08-23 +### Browsing no longer waits on H.264 stream-copy analysis +Silo checks each H.264 file once for a bitstream quirk that makes stream-copying unsafe. That check reads the opening seconds of the file, and it used to run while a media page was loading and be forgotten on every restart — so browsing a library, especially after a reboot, re-read part of every H.264 file. On remote or cloud storage that was the difference between an instant page and a slow one. + +Three things change. Media pages no longer trigger the analysis at all; it now happens when a play is actually being prepared, so browsing is fast regardless of where the files live. The result is stored on the file instead of being kept only in memory, so it survives restarts and is computed at most once per file. And the check itself reads 5 seconds instead of 15. + +A file that changes on disk is re-checked automatically: the stored answer is only trusted while the file's size and modification time still match, so re-encoding or replacing a file in place invalidates it without any manual step. Nothing is recorded when an analysis fails, so a transient error never turns into a stale verdict — the next request simply retries. No configuration changes, and playback behavior is unchanged. + ### Serve tokenless playback from proxy nodes again Playback protocol v3 now advertises the engine-neutral `authorized_media_origins_v1` opt-in, which a client sends together with `header_authenticated_media_v1`. Plans for such an attempt may return absolute, still credential-free media URLs on server-designated proxy origins (`/stream/v3/...`), so direct play, progressive remux, and HLS egress from the node pool instead of the API server. The proxy validates the caller's own access token against the same live login session the API checks, so revoking a session stops proxy playback immediately; replans and every other control-plane call stay on the API. A client that sends only `header_authenticated_media_v1` keeps today's API-local behavior unchanged, and so does a deployment with no proxy pool. diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index e0806b9c5..7f413695a 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -123,8 +123,15 @@ type PlaybackFileVersionFetcher interface { GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error) } +// PlaybackProbeEnsurer repairs probe metadata and resolves the H.264 +// copy-safety verdict. Playback keeps the full Ensure: the planner consumes +// the verdict to decide whether a video stream-copy is safe. +// +// EnsureProbeOnly is declared so the same value satisfies catalog's narrower +// browse-side contract when it is handed to the detail service. type PlaybackProbeEnsurer interface { Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) } type PlaybackChapterThumbnailQueuer interface { diff --git a/internal/catalog/detail.go b/internal/catalog/detail.go index ddbabefa5..a4471ccb0 100644 --- a/internal/catalog/detail.go +++ b/internal/catalog/detail.go @@ -44,7 +44,11 @@ type batchDurationFetcher interface { } type PlaybackProbeEnsurer interface { + // Ensure repairs probe metadata and resolves the H.264 copy-safety + // verdict; EnsureProbeOnly does the repair alone. Browse surfaces use the + // latter — see prepareBrowseFiles. Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) } type ChapterThumbnailQueuer interface { @@ -1359,7 +1363,7 @@ func (s *DetailService) buildExtraItemDetail(ctx context.Context, contentID stri return nil, fmt.Errorf("fetching extra files: %w", err) } files = FilterMediaFilesByAccess(files, filter) - files = s.preparePlaybackFiles(ctx, files) + files = s.prepareBrowseFiles(ctx, files) detail := &ItemDetail{ ContentID: extra.ContentID, @@ -1846,7 +1850,7 @@ func (s *DetailService) buildMediaItemDetail(ctx context.Context, item *models.M if item.Type == "audiobook" { sortAudiobookMediaFiles(files) } - files = s.preparePlaybackFiles(ctx, files) + files = s.prepareBrowseFiles(ctx, files) detail.Versions, detail.PlaybackVariants, detail.Subtitles, detail.Intro, detail.Credits, detail.Recap, detail.Preview = s.buildPlaybackInfo( ctx, files, @@ -2697,7 +2701,7 @@ func (s *DetailService) buildEpisodeDetail(ctx context.Context, episode *models. return nil, fmt.Errorf("fetching file versions: %w", err) } files = FilterMediaFilesByAccess(files, filter) - files = s.preparePlaybackFiles(ctx, files) + files = s.prepareBrowseFiles(ctx, files) detail.Versions, detail.PlaybackVariants, detail.Subtitles, detail.Intro, detail.Credits, detail.Recap, detail.Preview = s.buildPlaybackInfo( ctx, files, @@ -3694,7 +3698,22 @@ func fileIDOrZero(version *FileVersion) int { return version.FileID } +// preparePlaybackFiles repairs probe metadata and resolves the H.264 +// copy-safety verdict. Used by the watch surfaces, where a play is being +// prepared and the verdict is about to matter. func (s *DetailService) preparePlaybackFiles(ctx context.Context, files []*models.MediaFile) []*models.MediaFile { + return s.prepareFiles(ctx, files, true) +} + +// prepareBrowseFiles repairs probe metadata only. Item, episode and extra +// detail pages never consume the copy-safety verdict — it is not serialized +// into their responses — so scanning for it there was pure warm-up that cost a +// multi-second read per H.264 file on remote storage. +func (s *DetailService) prepareBrowseFiles(ctx context.Context, files []*models.MediaFile) []*models.MediaFile { + return s.prepareFiles(ctx, files, false) +} + +func (s *DetailService) prepareFiles(ctx context.Context, files []*models.MediaFile, withCopySafety bool) []*models.MediaFile { if len(files) == 0 { return files } @@ -3705,7 +3724,13 @@ func (s *DetailService) preparePlaybackFiles(ctx context.Context, files []*model continue } if s.probeEnsurer != nil { - ensured, err := s.probeEnsurer.Ensure(ctx, file) + var ensured *models.MediaFile + var err error + if withCopySafety { + ensured, err = s.probeEnsurer.Ensure(ctx, file) + } else { + ensured, err = s.probeEnsurer.EnsureProbeOnly(ctx, file) + } if err == nil && ensured != nil { file = ensured } diff --git a/internal/catalog/detail_prepare_files_test.go b/internal/catalog/detail_prepare_files_test.go new file mode 100644 index 000000000..f667ce414 --- /dev/null +++ b/internal/catalog/detail_prepare_files_test.go @@ -0,0 +1,78 @@ +package catalog + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// recordingProbeEnsurer records which half of the ensurer contract each +// prepare path asks for. +type recordingProbeEnsurer struct { + fullCalls []int + probeCalls []int +} + +func (e *recordingProbeEnsurer) Ensure(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.fullCalls = append(e.fullCalls, file.ID) + return file, nil +} + +func (e *recordingProbeEnsurer) EnsureProbeOnly(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.probeCalls = append(e.probeCalls, file.ID) + return file, nil +} + +// Browse detail must never trigger the H.264 copy-safety scan: the verdict is +// not serialized into those responses, so the scan is pure warm-up and its +// read is what made first-time browsing slow on remote storage. +func TestPrepareBrowseFilesSkipsCopySafety(t *testing.T) { + ensurer := &recordingProbeEnsurer{} + svc := &DetailService{probeEnsurer: ensurer} + files := []*models.MediaFile{{ID: 1}, {ID: 2}} + + prepared := svc.prepareBrowseFiles(context.Background(), files) + + if len(prepared) != 2 { + t.Fatalf("prepareBrowseFiles() returned %d files, want 2", len(prepared)) + } + if len(ensurer.fullCalls) != 0 { + t.Fatalf("browse path called Ensure for %v, want no copy-safety scans", ensurer.fullCalls) + } + if len(ensurer.probeCalls) != 2 { + t.Fatalf("browse path called EnsureProbeOnly %d times, want 2 — probe repair must still run", len(ensurer.probeCalls)) + } +} + +// The watch surfaces are where a play is being prepared, so they keep the +// full ensure and warm the verdict while the user looks at the Play button. +func TestPreparePlaybackFilesKeepsCopySafety(t *testing.T) { + ensurer := &recordingProbeEnsurer{} + svc := &DetailService{probeEnsurer: ensurer} + files := []*models.MediaFile{{ID: 1}, {ID: 2}} + + prepared := svc.preparePlaybackFiles(context.Background(), files) + + if len(prepared) != 2 { + t.Fatalf("preparePlaybackFiles() returned %d files, want 2", len(prepared)) + } + if len(ensurer.fullCalls) != 2 { + t.Fatalf("watch path called Ensure %d times, want 2", len(ensurer.fullCalls)) + } + if len(ensurer.probeCalls) != 0 { + t.Fatalf("watch path called EnsureProbeOnly for %v, want the full ensure", ensurer.probeCalls) + } +} + +func TestPrepareFilesWithoutEnsurerPassesFilesThrough(t *testing.T) { + svc := &DetailService{} + files := []*models.MediaFile{{ID: 1}, nil, {ID: 2}} + + if got := len(svc.prepareBrowseFiles(context.Background(), files)); got != 2 { + t.Fatalf("prepareBrowseFiles() returned %d files, want 2 (nil entries dropped)", got) + } + if got := len(svc.preparePlaybackFiles(context.Background(), files)); got != 2 { + t.Fatalf("preparePlaybackFiles() returned %d files, want 2 (nil entries dropped)", got) + } +} diff --git a/internal/chapterthumbs/service.go b/internal/chapterthumbs/service.go index 12f6d2f50..cd180c9f5 100644 --- a/internal/chapterthumbs/service.go +++ b/internal/chapterthumbs/service.go @@ -67,8 +67,11 @@ type FolderRepository interface { GetByID(ctx context.Context, id int) (*models.MediaFolder, error) } +// ProbeEnsurer repairs probe metadata. Only the repair half is needed here: +// chapter extraction reads Chapters, never the H.264 copy-safety verdict, so +// this deliberately does not ask for the bitstream scan. type ProbeEnsurer interface { - Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) } type SettingsReader interface { @@ -541,7 +544,7 @@ func (s *Service) ensureChapters(ctx context.Context, file *models.MediaFile, no return file, nil } - ensured, err := s.probeEnsurer.Ensure(ctx, file) + ensured, err := s.probeEnsurer.EnsureProbeOnly(ctx, file) if err == nil && ensured != nil { return ensured, nil } diff --git a/internal/chapterthumbs/service_test.go b/internal/chapterthumbs/service_test.go index ce1799059..b0b712dd9 100644 --- a/internal/chapterthumbs/service_test.go +++ b/internal/chapterthumbs/service_test.go @@ -221,7 +221,7 @@ type testProbeEnsurer struct { err error } -func (e testProbeEnsurer) Ensure(context.Context, *models.MediaFile) (*models.MediaFile, error) { +func (e testProbeEnsurer) EnsureProbeOnly(context.Context, *models.MediaFile) (*models.MediaFile, error) { if e.err != nil { return nil, e.err } diff --git a/internal/models/media.go b/internal/models/media.go index 2eac43f00..8e7175a61 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -113,12 +113,23 @@ type MediaFile struct { PresentationPartTotal int MultiEpisodeStart int MultiEpisodeEnd int - ProbeSource string // arrs, local - ProbeUpdatedAt *time.Time - MatchAttemptedAt *time.Time - MissingSince *time.Time - CreatedAt time.Time - UpdatedAt time.Time + // MultiplePPS is the persisted H.264 multi-PPS copy-safety verdict; nil + // means the file has never been successfully analyzed. It is trusted only + // when MultiplePPSScanSize and MultiplePPSScanMtime still match the file's + // current size and mtime, so a rewritten file self-invalidates without any + // coordination from the writers that touch media_files. + // + // json:"-" on all three: MediaFile is not a client-facing shape, and the + // runtime copy-safety signal clients do act on lives on VideoTrack. + MultiplePPS *bool `json:"-"` + MultiplePPSScanSize *int64 `json:"-"` + MultiplePPSScanMtime *time.Time `json:"-"` + ProbeSource string // arrs, local + ProbeUpdatedAt *time.Time + MatchAttemptedAt *time.Time + MissingSince *time.Time + CreatedAt time.Time + UpdatedAt time.Time } // MediaChapter represents a single media chapter derived from embedded file metadata. diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 7c2605b0f..10c72c152 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -59,6 +59,7 @@ const fileColumns = `id, content_id, episode_id, extra_id, season_number, episod edition_raw, edition_key, edition_confidence, edition_source, presentation_kind, presentation_group_key, presentation_part_index, presentation_part_total, multi_episode_start, multi_episode_end, + multiple_pps, multiple_pps_scan_size, multiple_pps_scan_mtime, probe_source, probe_updated_at, match_attempted_at, missing_since, created_at, updated_at` const overlayFileColumns = `content_id, episode_id, media_folder_id, file_path, @@ -82,6 +83,7 @@ const mfFileColumns = `mf.id, mf.content_id, mf.episode_id, mf.extra_id, mf.seas mf.edition_raw, mf.edition_key, mf.edition_confidence, mf.edition_source, mf.presentation_kind, mf.presentation_group_key, mf.presentation_part_index, mf.presentation_part_total, mf.multi_episode_start, mf.multi_episode_end, + mf.multiple_pps, mf.multiple_pps_scan_size, mf.multiple_pps_scan_mtime, mf.probe_source, mf.probe_updated_at, mf.match_attempted_at, mf.missing_since, mf.created_at, mf.updated_at` // scanMediaFile scans a single row into a *models.MediaFile. @@ -196,6 +198,9 @@ func scanMediaFile(row pgx.Row) (*models.MediaFile, error) { &presentationPartTotal, &multiEpisodeStart, &multiEpisodeEnd, + &f.MultiplePPS, + &f.MultiplePPSScanSize, + &f.MultiplePPSScanMtime, &probeSource, &f.ProbeUpdatedAt, &f.MatchAttemptedAt, @@ -506,6 +511,9 @@ func scanMediaFiles(rows pgx.Rows) ([]*models.MediaFile, error) { &presentationPartTotal, &multiEpisodeStart, &multiEpisodeEnd, + &f.MultiplePPS, + &f.MultiplePPSScanSize, + &f.MultiplePPSScanMtime, &probeSource, &f.ProbeUpdatedAt, &f.MatchAttemptedAt, @@ -1172,6 +1180,35 @@ func (r *FileRepository) SetChapterThumbnailFailure( return nil } +// UpdateMultiplePPS records the H.264 multi-PPS copy-safety verdict together +// with the size and mtime it was computed from, so a later read can tell +// whether the file has been rewritten since. +// +// It deliberately does not go through Upsert: that path also clears +// match_suppressed_at and missing_since, which a copy-safety scan has no +// business touching. +func (r *FileRepository) UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error { + tag, err := r.pool.Exec(ctx, ` + UPDATE media_files + SET multiple_pps = $2, + multiple_pps_scan_size = $3, + multiple_pps_scan_mtime = $4, + updated_at = NOW() + WHERE id = $1`, + fileID, + multiplePPS, + scanSize, + scanMtime, + ) + if err != nil { + return fmt.Errorf("updating multiple pps verdict: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrFileNotFound + } + return nil +} + // segmentState tracks the mutable per-segment fields used by UpsertMarkers. // Each segment kind (intro, credits, recap, preview) has an independent state // that the apply step mutates if the priority check allows the write. diff --git a/internal/scanner/pps.go b/internal/scanner/pps.go index 3725af4df..ea005bd48 100644 --- a/internal/scanner/pps.go +++ b/internal/scanner/pps.go @@ -10,10 +10,11 @@ import ( // copySafetyScanSeconds bounds how much of the stream the multi-PPS scan // demuxes. Affected encoders emit every PPS variant within the opening GOPs -// (all four in the reference file appear inside the first two seconds); a -// generous window catches slower rotations while staying a stream-copy, so the -// scan finishes in well under a second regardless of runtime. -const copySafetyScanSeconds = 15 +// (all four in the reference file appear inside the first two seconds), so a +// few seconds of headroom catches slower rotations. Kept short because the +// window is bytes read off the media store: on remote storage the read, not +// the demux, is what costs. +const copySafetyScanSeconds = 5 // DetectMultiplePPSH264 reports whether an H.264 stream redefines the same // pic_parameter_set_id in-band with more than one distinct content within the diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index 201665604..6b11ec79b 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -3,11 +3,13 @@ package scanner import ( "context" "log/slog" + "strconv" "strings" "sync" "time" "github.com/Silo-Server/silo-server/internal/models" + "golang.org/x/sync/singleflight" ) // NeedsCriticalProbeRepair reports whether playback-critical probe metadata is @@ -84,6 +86,12 @@ func videoTracksMissingColorRange(tracks []models.VideoTrack) bool { return false } +// copySafetyWriter persists a multi-PPS verdict. *FileRepository satisfies it; +// the indirection keeps the ensurer testable without a database. +type copySafetyWriter interface { + UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error +} + // PlaybackProbeEnsurer repairs missing playback-critical probe metadata on // demand by running a local ffprobe and persisting the result. type PlaybackProbeEnsurer struct { @@ -91,27 +99,76 @@ type PlaybackProbeEnsurer struct { ffprobePath string ffmpegPath string timeout time.Duration + // copySafetyRepo persists multi-PPS verdicts. Normally the same + // *FileRepository as fileRepo; tests substitute a double. + copySafetyRepo copySafetyWriter // copySafety memoizes the multi-PPS bitstream scan per file for the life of - // the process. It is never persisted: the scan runs on the first playback - // after a restart and is recomputed lazily thereafter. + // the process, in front of the persisted media_files verdict. Both layers + // are validated against the file's current size and mtime. copySafety sync.Map // file ID -> copySafetyResult + // copySafetyFlight collapses concurrent first scans of the same file so a + // burst of playback/detail requests spawns one ffmpeg, not one each. + copySafetyFlight singleflight.Group } type copySafetyResult struct { size int64 + mtime *time.Time multi bool } +// matches reports whether a memoized verdict still describes the given file. +func (r copySafetyResult) matches(file *models.MediaFile) bool { + if r.size != file.FileSize { + return false + } + if r.mtime == nil || file.FileModifiedAt == nil { + // A verdict recorded without an mtime can only be trusted on size. + return r.mtime == nil && file.FileModifiedAt == nil + } + return sameFileModifiedAt(r.mtime, *file.FileModifiedAt) +} + func NewPlaybackProbeEnsurer(fileRepo *FileRepository, ffprobePath, ffmpegPath string, timeout time.Duration) *PlaybackProbeEnsurer { - return &PlaybackProbeEnsurer{ + e := &PlaybackProbeEnsurer{ fileRepo: fileRepo, ffprobePath: ffprobePath, ffmpegPath: ffmpegPath, timeout: timeout, } + if fileRepo != nil { + e.copySafetyRepo = fileRepo + } + return e } +// Ensure repairs playback-critical probe metadata and resolves the H.264 +// copy-safety verdict. Use it where a play is being prepared — the planner +// consumes the verdict to decide whether a video stream-copy is safe. func (e *PlaybackProbeEnsurer) Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { + current, err := e.ensureProbeRepair(ctx, file) + if err != nil || current == nil || e == nil { + return current, err + } + + // Copy-safety analysis is independent of critical probe repair: an + // already-probed file still needs its multi-PPS verdict before the planner + // can decide whether a video stream-copy is safe. + return e.ensureCopySafety(ctx, current) +} + +// EnsureProbeOnly repairs playback-critical probe metadata and stops there. +// +// Browse surfaces (item, episode and extra detail pages) use this: they never +// consume the copy-safety verdict — VideoTrack.MultiplePPS is json:"-" and +// never reaches a client — so running the bitstream scan there was pure +// warm-up, and it is exactly what made first-time browsing slow on remote +// storage. The verdict is resolved when a play is actually being prepared. +func (e *PlaybackProbeEnsurer) EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { + return e.ensureProbeRepair(ctx, file) +} + +func (e *PlaybackProbeEnsurer) ensureProbeRepair(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { if file == nil || e == nil || e.fileRepo == nil { return file, nil } @@ -140,38 +197,39 @@ func (e *PlaybackProbeEnsurer) Ensure(ctx context.Context, file *models.MediaFil current = repaired } - // Copy-safety analysis is independent of critical probe repair: an - // already-probed file still needs its one-time multi-PPS scan before the - // planner can decide whether a video stream-copy is safe. - return e.ensureCopySafety(ctx, current) + return current, nil } -// ensureCopySafety computes the multi-PPS copy-safety flag for H.264 files at -// playback start and stamps it on an in-memory copy of the file. The result is -// memoized per process and never written to the database, so it is recomputed -// on the first play after a restart. +// ensureCopySafety resolves the multi-PPS copy-safety flag for H.264 files at +// playback start and stamps it on an in-memory copy of the file. It answers +// from the process cache first, then from the verdict persisted on the +// media_files row, and only then runs the bitstream scan — so a restart no +// longer re-reads the opening seconds of every browsed H.264 file. func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { if !needsCopySafetyProbe(file) || strings.TrimSpace(e.ffmpegPath) == "" { return file, nil } if cached, ok := e.copySafety.Load(file.ID); ok { - if result, ok := cached.(copySafetyResult); ok && result.size == file.FileSize { + if result, ok := cached.(copySafetyResult); ok && result.matches(file) { return fileWithMultiplePPS(file, result.multi), nil } } - timeout := e.timeout - if timeout < 30*time.Second { - timeout = 30 * time.Second + // A persisted verdict is self-validating: it is only honored while the + // recorded size and mtime still describe the file, so a rewrite in place + // falls through to a rescan without any writer having to clear it. + if multi, ok := persistedCopySafetyVerdict(file); ok { + e.storeCopySafety(file, multi) + return fileWithMultiplePPS(file, multi), nil } - scanCtx, cancel := context.WithTimeout(ctx, timeout) - multi, err := DetectMultiplePPSH264(scanCtx, e.ffmpegPath, file.FilePath) - cancel() + + multi, err := e.scanAndPersistCopySafety(ctx, file) if err != nil { // Unknown safety must not fail open to the video-copy path this probe is - // intended to guard. Leave MultiplePPS unset and do not cache the result, - // so a later request retries the scan without misreporting the cause. + // intended to guard. Leave MultiplePPS unset and do not cache or persist + // the result, so a later request retries the scan without misreporting + // the cause. slog.WarnContext(ctx, "video copy-safety scan failed; disabling stream copy", "component", "scanner", "file_id", file.ID, @@ -180,10 +238,76 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model return fileWithCopySafety(file, nil, true), nil } - e.copySafety.Store(file.ID, copySafetyResult{size: file.FileSize, multi: multi}) return fileWithMultiplePPS(file, multi), nil } +// scanAndPersistCopySafety runs the multi-PPS bitstream scan, persists the +// verdict, and memoizes it. Concurrent callers for the same file share one +// scan; a failed database write is logged and the scan result is still used, +// since it is correct for this request and the next one will retry the write. +func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { + fileID := file.ID + filePath := file.FilePath + fileSize := file.FileSize + fileModifiedAt := file.FileModifiedAt + + multi, err, _ := e.copySafetyFlight.Do(strconv.Itoa(fileID), func() (any, error) { + timeout := e.timeout + if timeout < 30*time.Second { + timeout = 30 * time.Second + } + scanCtx, cancel := context.WithTimeout(ctx, timeout) + multi, err := DetectMultiplePPSH264(scanCtx, e.ffmpegPath, filePath) + cancel() + if err != nil { + return false, err + } + + if e.copySafetyRepo != nil && fileModifiedAt != nil { + if writeErr := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, multi, fileSize, *fileModifiedAt); writeErr != nil { + slog.WarnContext(ctx, "persisting video copy-safety verdict failed", + "component", "scanner", + "file_id", fileID, + "error", writeErr, + ) + } + } + e.storeCopySafety(file, multi) + return multi, nil + }) + if err != nil { + return false, err + } + result, _ := multi.(bool) + return result, nil +} + +func (e *PlaybackProbeEnsurer) storeCopySafety(file *models.MediaFile, multi bool) { + entry := copySafetyResult{size: file.FileSize, multi: multi} + if file.FileModifiedAt != nil { + mtime := *file.FileModifiedAt + entry.mtime = &mtime + } + e.copySafety.Store(file.ID, entry) +} + +// persistedCopySafetyVerdict returns the multi-PPS verdict stored on the +// media_files row, and whether it is still valid for the file as it stands. A +// verdict is valid only when it was computed from the same size and mtime the +// row now reports. +func persistedCopySafetyVerdict(file *models.MediaFile) (bool, bool) { + if file == nil || file.MultiplePPS == nil || file.MultiplePPSScanSize == nil || file.MultiplePPSScanMtime == nil { + return false, false + } + if *file.MultiplePPSScanSize != file.FileSize { + return false, false + } + if file.FileModifiedAt == nil || !sameFileModifiedAt(file.MultiplePPSScanMtime, *file.FileModifiedAt) { + return false, false + } + return *file.MultiplePPS, true +} + // fileWithMultiplePPS returns a shallow copy of file with the (runtime-only) // MultiplePPS flag set on its first video track, without mutating the caller's // file or its shared VideoTracks slice. diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go new file mode 100644 index 000000000..07a3064b7 --- /dev/null +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -0,0 +1,380 @@ +package scanner + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// conflictingPPSAnnexB is a two-NAL Annex-B stream that redefines +// pic_parameter_set_id 0 with two different payloads — what +// DetectMultiplePPSH264 reports as multi-PPS. Written as printf octal escapes +// so a /bin/sh stub can emit it: 00 00 01 68 80 | 00 00 01 68 C0. +const conflictingPPSAnnexB = `\000\000\001\150\200\000\000\001\150\300` + +// fakeFFmpeg writes a stub ffmpeg that appends one line to a log file per +// invocation and emits the given printf-escaped payload on stdout. It returns +// the stub's path and a func reporting how many times it ran. +func fakeFFmpeg(t *testing.T, stdoutPayload string, delay time.Duration) (string, func() int) { + t.Helper() + dir := t.TempDir() + logPath := filepath.Join(dir, "invocations.log") + ffmpegPath := filepath.Join(dir, "ffmpeg") + sleep := "" + if delay > 0 { + sleep = fmt.Sprintf("sleep %.2f\n", delay.Seconds()) + } + script := fmt.Sprintf("#!/bin/sh\necho run >> %q\n%sprintf '%s'\n", logPath, sleep, stdoutPayload) + if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + return ffmpegPath, func() int { + data, err := os.ReadFile(logPath) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read fake ffmpeg log: %v", err) + } + runs := 0 + for _, b := range data { + if b == '\n' { + runs++ + } + } + return runs + } +} + +type recordedPPSWrite struct { + fileID int + multiplePPS bool + scanSize int64 + scanMtime time.Time +} + +type fakeCopySafetyWriter struct { + mu sync.Mutex + writes []recordedPPSWrite + err error +} + +func (w *fakeCopySafetyWriter) UpdateMultiplePPS(_ context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error { + w.mu.Lock() + defer w.mu.Unlock() + w.writes = append(w.writes, recordedPPSWrite{ + fileID: fileID, + multiplePPS: multiplePPS, + scanSize: scanSize, + scanMtime: scanMtime, + }) + return w.err +} + +func (w *fakeCopySafetyWriter) recorded() []recordedPPSWrite { + w.mu.Lock() + defer w.mu.Unlock() + return append([]recordedPPSWrite(nil), w.writes...) +} + +func copySafetyTestFile(mtime time.Time) *models.MediaFile { + modified := mtime + return &models.MediaFile{ + ID: 42, + FilePath: "/library/movie.mkv", + FileSize: 1234, + FileModifiedAt: &modified, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + } +} + +func TestEnsureCopySafetyUsesPersistedVerdictWithoutScanning(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, "", 0) + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + file := copySafetyTestFile(mtime) + verdict := true + scanSize := file.FileSize + scanMtime := mtime + file.MultiplePPS = &verdict + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &scanMtime + + got, err := ensurer.ensureCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 0 { + t.Fatalf("ffmpeg ran %d times, want 0 for a valid persisted verdict", runs()) + } + track := got.VideoTracks[0] + if track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("MultiplePPS = %v, want true from the persisted verdict", track.MultiplePPS) + } + if !track.VideoCopyUnsafe { + t.Fatal("VideoCopyUnsafe = false, want true for a multi-PPS file") + } + if _, ok := ensurer.copySafety.Load(file.ID); !ok { + t.Fatal("persisted verdict was not promoted into the in-memory cache") + } +} + +func TestEnsureCopySafetyRescansStaleVerdict(t *testing.T) { + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + + tests := []struct { + name string + scanSize int64 + scanMtime time.Time + }{ + {name: "size mismatch", scanSize: 999, scanMtime: mtime}, + {name: "mtime mismatch", scanSize: 1234, scanMtime: mtime.Add(time.Hour)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + + file := copySafetyTestFile(mtime) + verdict := false + scanSize := tc.scanSize + scanMtime := tc.scanMtime + file.MultiplePPS = &verdict + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &scanMtime + + got, err := ensurer.ensureCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1 for a stale persisted verdict", runs()) + } + track := got.VideoTracks[0] + if track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("MultiplePPS = %v, want the rescanned true, not the stale false", track.MultiplePPS) + } + }) + } +} + +func TestEnsureCopySafetyPersistsScanResult(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + file := copySafetyTestFile(mtime) + + got, err := ensurer.ensureCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1", runs()) + } + writes := writer.recorded() + if len(writes) != 1 { + t.Fatalf("UpdateMultiplePPS called %d times, want 1", len(writes)) + } + want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234, scanMtime: mtime} + if writes[0] != want { + t.Fatalf("UpdateMultiplePPS(%+v), want %+v", writes[0], want) + } + if track := got.VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("MultiplePPS = %v, want true", track.MultiplePPS) + } +} + +func TestEnsureCopySafetyScanSurvivesPersistFailure(t *testing.T) { + ffmpegPath, _ := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{err: fmt.Errorf("database unavailable")} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + got, err := ensurer.ensureCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v, want the scan result to be used anyway", err) + } + if track := got.VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("MultiplePPS = %v, want the scan result despite the failed write", track.MultiplePPS) + } +} + +func TestEnsureCopySafetyWithoutRepoDoesNotPanic(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + if _, err := ensurer.ensureCopySafety(context.Background(), file); err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1", runs()) + } +} + +// A failed scan must stay fail-closed and stateless: nothing is written, so a +// transient error never becomes sticky state on the row and the next request +// retries cleanly. +func TestEnsureCopySafetyFailureRecordsNothing(t *testing.T) { + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ + ffmpegPath: filepath.Join(t.TempDir(), "missing-ffmpeg"), + copySafetyRepo: writer, + } + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + got, err := ensurer.ensureCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if !got.VideoTracks[0].VideoCopyUnsafe { + t.Fatal("VideoCopyUnsafe = false, want true after an inconclusive scan") + } + if writes := writer.recorded(); len(writes) != 0 { + t.Fatalf("failed scan recorded %d verdicts, want 0", len(writes)) + } +} + +// Browse surfaces call EnsureProbeOnly. The copy-safety verdict never reaches +// a client from those responses, so scanning there was pure warm-up — and the +// read it costs is what made first-time browsing slow on remote storage. +func TestEnsureProbeOnlySkipsCopySafetyScan(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + got, err := ensurer.EnsureProbeOnly(context.Background(), file) + if err != nil { + t.Fatalf("EnsureProbeOnly() error = %v", err) + } + if runs() != 0 { + t.Fatalf("ffmpeg ran %d times for a browse-detail load, want 0", runs()) + } + if got.VideoTracks[0].MultiplePPS != nil { + t.Fatal("EnsureProbeOnly() resolved the copy-safety verdict") + } + if got.VideoTracks[0].VideoCopyUnsafe { + t.Fatal("EnsureProbeOnly() marked the file copy-unsafe") + } + if writes := writer.recorded(); len(writes) != 0 { + t.Fatalf("EnsureProbeOnly() persisted %d verdicts, want 0", len(writes)) + } + + // The same ensurer still scans when a play is being prepared. + if _, err := ensurer.Ensure(context.Background(), file); err != nil { + t.Fatalf("Ensure() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times for a playback load, want 1", runs()) + } +} + +func TestEnsureCopySafetyConcurrentCallsScanOnce(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 200*time.Millisecond) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + + const callers = 8 + var wg sync.WaitGroup + results := make([]*models.MediaFile, callers) + errs := make([]error, callers) + start := make(chan struct{}) + for i := 0; i < callers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i], errs[i] = ensurer.ensureCopySafety(context.Background(), copySafetyTestFile(mtime)) + }(i) + } + close(start) + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Fatalf("ensureCopySafety() caller %d error = %v", i, err) + } + if track := results[i].VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("caller %d MultiplePPS = %v, want true", i, track.MultiplePPS) + } + } + if got := runs(); got != 1 { + t.Fatalf("ffmpeg ran %d times for %d concurrent callers, want 1", got, callers) + } + if writes := writer.recorded(); len(writes) != 1 { + t.Fatalf("UpdateMultiplePPS called %d times, want 1", len(writes)) + } +} + +func TestPersistedCopySafetyVerdict(t *testing.T) { + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + base := func() *models.MediaFile { + file := copySafetyTestFile(mtime) + verdict := true + scanSize := file.FileSize + scanMtime := mtime + file.MultiplePPS = &verdict + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &scanMtime + return file + } + + tests := []struct { + name string + mutate func(*models.MediaFile) + wantMulti bool + wantOK bool + }{ + {name: "valid", mutate: func(*models.MediaFile) {}, wantMulti: true, wantOK: true}, + {name: "never scanned", mutate: func(f *models.MediaFile) { f.MultiplePPS = nil }}, + {name: "missing scan size", mutate: func(f *models.MediaFile) { f.MultiplePPSScanSize = nil }}, + {name: "missing scan mtime", mutate: func(f *models.MediaFile) { f.MultiplePPSScanMtime = nil }}, + {name: "size drifted", mutate: func(f *models.MediaFile) { f.FileSize = 4321 }}, + { + name: "mtime drifted", + mutate: func(f *models.MediaFile) { + later := mtime.Add(time.Second) + f.FileModifiedAt = &later + }, + }, + {name: "file mtime unknown", mutate: func(f *models.MediaFile) { f.FileModifiedAt = nil }}, + { + name: "sub-microsecond mtime drift is absorbed", + mutate: func(f *models.MediaFile) { + jittered := mtime.Add(17 * time.Nanosecond).Local() + f.FileModifiedAt = &jittered + }, + wantMulti: true, + wantOK: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + file := base() + tc.mutate(file) + multi, ok := persistedCopySafetyVerdict(file) + if ok != tc.wantOK || multi != tc.wantMulti { + t.Fatalf("persistedCopySafetyVerdict() = (%v, %v), want (%v, %v)", multi, ok, tc.wantMulti, tc.wantOK) + } + }) + } +} diff --git a/migrations/sql/20260823182731_persist_multiple_pps_verdict.sql b/migrations/sql/20260823182731_persist_multiple_pps_verdict.sql new file mode 100644 index 000000000..f495c51b7 --- /dev/null +++ b/migrations/sql/20260823182731_persist_multiple_pps_verdict.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +-- Persist the H.264 multi-PPS copy-safety verdict so the bitstream scan is not +-- re-run on every process restart. The verdict is self-validating: it is only +-- trusted when the recorded scan size and mtime still match the media_files +-- row, so any rewrite of the file invalidates it without writer coordination. +ALTER TABLE public.media_files + ADD COLUMN multiple_pps boolean, + ADD COLUMN multiple_pps_scan_size bigint, + ADD COLUMN multiple_pps_scan_mtime timestamptz; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE public.media_files + DROP COLUMN IF EXISTS multiple_pps_scan_mtime, + DROP COLUMN IF EXISTS multiple_pps_scan_size, + DROP COLUMN IF EXISTS multiple_pps; +-- +goose StatementEnd From f027119f9263dfebdcdb23d179d022ae7f2be867 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:58:05 -0400 Subject: [PATCH 36/44] feat(playback): optimistic remux race with server-initiated plan invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an H.264 file's copy-safety verdict is unknown, playback no longer blocks on the bitstream scan: the planner issues the remux optimistically, the scan runs behind the plan, and an unsafe verdict withdraws it. Sessions that negotiated the new plan_invalidated_v1 feature get a pushed plan_invalidated realtime command and switch via their normal failure_recovery replan; everything else — including today's mobile apps — is stopped and recovers onto a transcode through the persisted verdict. Watch pages and playback start now never wait on the scan. jellycompat sessions are exempt: their route selection does not consult the verdict yet. Web client implements the feature; Apple/Android follow-ups tracked in their repos. Related issue: #135 Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 96 +++- .../fixtures/valid/capability_response.json | 1 + .../v3/fixtures/valid/decision_response.json | 1 + docs/feature-changelog.md | 7 + internal/api/handlers/playback.go | 50 +- .../api/handlers/playback_copy_safety_test.go | 117 +++++ internal/api/handlers/playback_realtime.go | 38 ++ internal/api/handlers/playback_v3.go | 13 + internal/api/handlers/session_ws.go | 15 + .../session_ws_plan_invalidated_test.go | 71 +++ internal/api/router.go | 23 + internal/catalog/detail.go | 50 +- internal/catalog/detail_prepare_files_test.go | 82 ++- internal/models/media.go | 63 +++ internal/playback/command_dispatcher_test.go | 12 + internal/playback/copy_safety_notifier.go | 364 ++++++++++++++ .../playback/copy_safety_notifier_test.go | 466 ++++++++++++++++++ internal/playback/copy_safety_race.go | 136 +++++ internal/playback/copy_safety_race_test.go | 234 +++++++++ internal/playback/protocol_v3.go | 29 +- internal/playback/protocol_v3_test.go | 65 +++ internal/playback/realtime.go | 38 ++ internal/playback/realtime_test.go | 42 ++ internal/playback/resolver.go | 13 +- .../protocol_v3/capability_response.json | 1 + .../protocol_v3/conformance_matrix.json | 1 + .../protocol_v3/decision_response.json | 1 + internal/scanner/file_repo.go | 5 +- internal/scanner/probe_repair.go | 125 +++-- .../probe_repair_copy_safety_cached_test.go | 212 ++++++++ .../probe_repair_copy_safety_persist_test.go | 64 ++- internal/scanner/scanner.go | 2 +- .../player/components/VideoPlayer.test.tsx | 65 ++- web/src/player/components/VideoPlayer.tsx | 33 +- web/src/player/components/WatchPage.test.ts | 1 + web/src/player/components/WatchPage.tsx | 1 + web/src/player/hooks/usePlaybackRealtime.ts | 17 +- .../player/hooks/usePlaybackSession.test.ts | 173 +++++++ web/src/player/hooks/usePlaybackSession.ts | 39 ++ web/src/player/playback-session-wire-v3.ts | 37 +- web/src/player/protocol-v3.ts | 11 + web/src/player/realtime-protocol.test.ts | 45 ++ web/src/player/realtime-protocol.ts | 51 +- 43 files changed, 2800 insertions(+), 110 deletions(-) create mode 100644 internal/api/handlers/playback_copy_safety_test.go create mode 100644 internal/api/handlers/session_ws_plan_invalidated_test.go create mode 100644 internal/playback/copy_safety_notifier.go create mode 100644 internal/playback/copy_safety_notifier_test.go create mode 100644 internal/playback/copy_safety_race.go create mode 100644 internal/playback/copy_safety_race_test.go create mode 100644 internal/scanner/probe_repair_copy_safety_cached_test.go diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index def7a96ec..8c6d177b9 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -48,7 +48,9 @@ The server never claims something it did not verify. **Route events are diagnostics, not control.** A client reports what happened (`first_frame`, `plan_failed`, `terminal`) so the server can learn; the report never changes the session. Playback recovery goes through replan (§6), which is a -request with a response, not a fire-and-forget event. +request with a response, not a fire-and-forget event. The server may *ask* for a +replan — the `plan_invalidated` realtime command, §6.1 — but even then the plan +only changes when the client comes back through the replan endpoint. Two consequences worth stating early, because they surprise implementers: @@ -99,13 +101,13 @@ the document is always the full one: "features": ["playback_plan_v3", "neutral_playback_v3_contract_v1", "layout_aware_passthrough", "playback_route_diagnostics", "device_quirks_v1", "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1", "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", - "plan_source_duration_v1"], + "plan_invalidated_v1", "plan_source_duration_v1"], "deliveries": ["original_http", "server_remux_progressive", "server_remux_hls", "server_transcode_hls"], "transformations": [{"name": "audio_to_aac", "executor": "server", "recipe_version": "1", "validated_claims": ["audio_decode"]}] } ``` -The twelve feature strings above are the full set this server version advertises: +The thirteen feature strings above are the full set this server version advertises: | Feature | What it promises | | --- | --- | @@ -120,6 +122,7 @@ The twelve feature strings above are the full set this server version advertises | `header_authenticated_media_v1` | An opted-in client receives media URLs without signed credentials in their query or path, and authenticates every media request with its normal Authorization header (§4.1) | | `authorized_media_origins_v1` | Meaningful only with the token above: the client also honors credential-free absolute media URLs on server-designated proxy origins, which restores distributed egress for a header-authenticated attempt (§4.1) | | `software_video_decode_v1` | Exact/platform-attested clients may qualify bounded `video_decode[]` entries with `hardware: false` for direct/original delivery; without the opt-in those evidence tiers remain hardware-only (§3) | +| `plan_invalidated_v1` | The client can be told mid-session that the plan it is playing was withdrawn, over the realtime `plan_invalidated` command, and replans off it. A session that did not negotiate it is stopped instead (§6.1) | | `plan_source_duration_v1` | `source.duration_seconds` is populated when known, so its absence means *unknown* rather than *unsupported* (§5) | That last one is the reason feature detection is a list and not a version @@ -732,6 +735,93 @@ sends — including an explicit list that omits one, which is otherwise a valid way to drop a feature. Seek replans never replace the feature list at all. Changing any of these modes means stopping and starting a new attempt. +### 6.1 `plan_invalidated_v1` — the server withdraws a plan + +Every other control message in this protocol travels client → server. This one +does not: `plan_invalidated` is the only **server-initiated control push**, and +it exists because the server can learn a route is wrong *after* the plan is +already playing. + +The concrete case is H.264 stream-copy safety. Some encoders redefine the same +`pic_parameter_set_id` in-band with conflicting content, which cannot be copied +into an avc1/fMP4 segment (§4). Detecting it means reading the opening seconds +of the source, which on remote storage costs seconds — so the server no longer +waits for it. An unresolved verdict plans **optimistically** (a remux is +allowed), the scan runs behind the issued plan, and if it comes back unsafe the +plan that was handed out has to be taken back. + +The push is a realtime **command** on the session control socket +(`GET /playback/sessions/{session_id}/control/ws`) — acked and answered like +any other, not a fire-and-forget event: + +```json +{ + "type": "command", + "command_id": "…", + "session_id": "…", + "name": "plan_invalidated", + "reason": "video_copy_unsafe", + "deadline_ms": 8000, + "payload": {"reason": "video_copy_unsafe", "plan_id": ""} +} +``` + +`payload.plan_id` names the plan being withdrawn, which is not necessarily the +one on screen: a client that has already replanned past it has nothing to do and +completes the command as a no-op. That is why the field is required — acting +without checking it would evict a route the server never complained about. + +A client that advertises `plan_invalidated_v1` in `client_features` promises to: + +1. send `{"type":"ack","status":"accepted"}` immediately, +2. run its ordinary recovery replan — `operation: "failure_recovery"`, with the + invalidated plan's `plan_attempt_key` in `attempted_plan_keys` so the copy + route is excluded deterministically (the now-persisted verdict excludes it + too), and +3. send `{"type":"result","status":"completed"}` when the replan is done. + +**Everything else is a session stop.** The server pushes the command only to a +session that negotiated the feature *and* holds a live realtime connection. No +feature, no connection, no `completed` result within `deadline_ms` (an ack +alone does not stop the clock), or a `rejected` result, and the session is +terminated instead. That is deliberate, and it is the whole +backwards-compatibility story: a client shipped before this token sees its +session end, runs the recovery it already has, and its fresh attempt is planned +against the persisted verdict — which lands it on a transcode. No client has to +implement anything to stay correct; the feature only buys a seamless switch +instead of a stopped session. + +An inconclusive scan changes nothing: nothing is persisted, no command is +pushed, and live sessions keep the route they were given. Only a positive +"this source cannot be copied" verdict withdraws a plan. + +Three scoping rules keep the stop from firing where it cannot help: + +- **The verdict is about the effective file.** A session whose *requested* + edition turned out to be copy-unsafe, but which is streaming a different + edition after the 4K guard or a version replan, is left alone: the bytes it is + serving are copy-safe. +- **A session still being established is given time.** A session exists in the + session manager before its attempt record is written and long before its + client can open a realtime channel. A verdict landing inside that window would + see a session it cannot tell and stop one that is mid-start, so a session that + would otherwise be stopped waits out `CopySafetySessionSettleWindow` and is + examined once more; by then it is normally reachable and gets the command. +- **Jellyfin-compatibility sessions are exempt.** That surface decides direct + stream from the device profile and the catalog version, never from the + copy-safety verdict, so a stopped compat client reconnects onto the identical + remux. The stop is only correct for clients whose recovery re-decides the + route, which for a Silo client means planning against the persisted verdict. + +That persisted verdict is read from the `media_files` row on every path that +plans a route — start, replan, and the v2 resolver — not only from the probe +ensurer's in-memory stamp. A replan that did not see it would simply walk from +one stream-copy delivery to the other. + +Delivery is in-process: the replica that owns the session owns its realtime +connection, so a verdict resolved on one node acts on the sessions that node is +serving. + --- ## 7. Registries diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json index 230cd0820..58fa7acee 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json @@ -15,6 +15,7 @@ "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_invalidated_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json index b194163f6..3e57dcb70 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json @@ -12,6 +12,7 @@ "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_invalidated_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index edcf76943..a71065745 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -2,6 +2,13 @@ ## 2026-08-23 +### Never wait on H.264 stream-copy analysis to start playing +The check that decides whether an H.264 file can be stream-copied reads the opening seconds of the source, which on remote or cloud storage takes seconds. It used to run before playback could start, and a file it had never seen was held at the Play button; when the check itself failed, playback fell back to a full transcode even though nothing had actually proven the file unsafe. + +It no longer runs on the request path at all. A file with no stored verdict is now played optimistically — the cheap stream-copy route — and the analysis runs behind the stream that is already playing. Watch pages behave the same way: they start the analysis and render immediately. + +If the analysis then finds the file genuinely cannot be copied, Silo moves the sessions playing it off that route. Clients that advertise the new `plan_invalidated_v1` capability are told to switch, and they re-plan onto a transcode without the viewer seeing more than a brief rebuffer. Any other client — including every app version shipped before this change — has its session ended and recovers the way it already does; because the verdict is now stored, its next attempt starts on the transcode directly. An analysis that fails or is inconclusive changes nothing: nothing is stored and playback continues untouched, instead of the old behavior of transcoding on a failed check. + ### Browsing no longer waits on H.264 stream-copy analysis Silo checks each H.264 file once for a bitstream quirk that makes stream-copying unsafe. That check reads the opening seconds of the file, and it used to run while a media page was loading and be forgotten on every restart — so browsing a library, especially after a reboot, re-read part of every H.264 file. On remote or cloud storage that was the difference between an instant page and a slow one. diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 7f413695a..c02bfeb00 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -123,15 +123,26 @@ type PlaybackFileVersionFetcher interface { GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error) } -// PlaybackProbeEnsurer repairs probe metadata and resolves the H.264 -// copy-safety verdict. Playback keeps the full Ensure: the planner consumes -// the verdict to decide whether a video stream-copy is safe. +// PlaybackProbeEnsurer repairs probe metadata and stamps the H.264 copy-safety +// verdict when it is already known. // -// EnsureProbeOnly is declared so the same value satisfies catalog's narrower -// browse-side contract when it is handed to the detail service. +// It deliberately exposes no blocking variant: a play must never wait on the +// multi-second bitstream scan, so an unknown verdict is planned optimistically +// and resolved behind the play (see PlaybackCopySafetyRacer). +// +// EnsureProbeOnly is declared because this interface is also the type the +// router carries the shared ensurer in when handing it to the catalog and +// chapter-thumbnail services, which repair probe metadata and nothing else. type PlaybackProbeEnsurer interface { - Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + EnsureCopySafetyCached(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) +} + +// PlaybackCopySafetyRacer resolves an unknown H.264 copy-safety verdict out of +// band, after a plan that stream-copies video has already been issued. +// *playback.CopySafetyRace implements it. +type PlaybackCopySafetyRacer interface { + RaceScanForPlan(fileID int, plan *playback.PlanV3) } type PlaybackChapterThumbnailQueuer interface { @@ -179,14 +190,18 @@ type PlaybackHandler struct { // relayed request has nothing to reconstruct from. Optional and best effort: // without it (or without Redis behind it) such a session replans instead of // recovering, exactly as before. - NodeRecipeStore recipeCardStoreV3 - ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks - EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series - ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item - OriginalLangLookup PlaybackOriginalLanguageLookup - SettingsRepo PlaybackSettingsReader // optional; reads server settings (e.g., allow_4k_transcode) - FileVersionFetcher PlaybackFileVersionFetcher // optional; queries sibling file versions for 4K guard - ProbeEnsurer PlaybackProbeEnsurer // optional; repairs missing probe metadata on demand + NodeRecipeStore recipeCardStoreV3 + ItemAccess PlaybackItemAccessChecker // optional; enables file authorization checks + EpisodeLookup PlaybackEpisodeLookup // optional; resolves episode files to their series + ExtraLookup PlaybackExtraLookup // optional; resolves extras files to their parent item + OriginalLangLookup PlaybackOriginalLanguageLookup + SettingsRepo PlaybackSettingsReader // optional; reads server settings (e.g., allow_4k_transcode) + FileVersionFetcher PlaybackFileVersionFetcher // optional; queries sibling file versions for 4K guard + ProbeEnsurer PlaybackProbeEnsurer // optional; repairs missing probe metadata on demand + // CopySafetyRacer resolves an unknown H.264 copy-safety verdict behind an + // already-issued stream-copy plan. Optional: without it an unknown verdict + // simply stays unknown and the copy route is never withdrawn. + CopySafetyRacer PlaybackCopySafetyRacer ChapterThumbnailQueuer PlaybackChapterThumbnailQueuer IntroAnalyzer IntroEpisodeAnalyzer IntroRepository PlaybackIntroEligibilityChecker @@ -390,11 +405,16 @@ func semanticPlayMethod(s *playback.Session) playback.PlayMethod { return s.PlayMethod } +// ensurePlaybackProbe repairs probe metadata and stamps the H.264 copy-safety +// verdict when it is already known. It never runs the bitstream scan: an +// unknown verdict plans optimistically (the planner reads nil MultiplePPS as +// "copy is allowed") and is resolved asynchronously once the plan is issued, so +// starting playback never waits on a multi-second read of the source. func (h *PlaybackHandler) ensurePlaybackProbe(ctx context.Context, file *models.MediaFile) *models.MediaFile { if h == nil || h.ProbeEnsurer == nil || file == nil { return file } - repaired, err := h.ProbeEnsurer.Ensure(ctx, file) + repaired, err := h.ProbeEnsurer.EnsureCopySafetyCached(ctx, file) if err != nil { slog.WarnContext(ctx, "playback probe repair failed", "component", "api", "file_id", file.ID, "path", file.FilePath, "error", err) return file diff --git a/internal/api/handlers/playback_copy_safety_test.go b/internal/api/handlers/playback_copy_safety_test.go new file mode 100644 index 000000000..5f866a92d --- /dev/null +++ b/internal/api/handlers/playback_copy_safety_test.go @@ -0,0 +1,117 @@ +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// blockingProbeEnsurer fails the test if the start path asks for anything but +// the cached-only ensure — the other variants can run the multi-second +// bitstream scan, which must never happen on a request path. +type blockingProbeEnsurer struct { + t *testing.T + mu sync.Mutex + cachedCalls int +} + +func (e *blockingProbeEnsurer) EnsureProbeOnly(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.t.Helper() + e.t.Fatal("playback start called EnsureProbeOnly; it must resolve a known copy-safety verdict") + return file, nil +} + +func (e *blockingProbeEnsurer) EnsureCopySafetyCached(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.cachedCalls++ + return file, nil +} + +func (e *blockingProbeEnsurer) calls() int { + e.mu.Lock() + defer e.mu.Unlock() + return e.cachedCalls +} + +type recordingCopySafetyRacer struct { + mu sync.Mutex + plans []playback.DeliveryV3 + files []int +} + +func (r *recordingCopySafetyRacer) RaceScanForPlan(fileID int, plan *playback.PlanV3) { + r.mu.Lock() + defer r.mu.Unlock() + r.files = append(r.files, fileID) + if plan != nil { + r.plans = append(r.plans, plan.Delivery) + } +} + +func (r *recordingCopySafetyRacer) raced() ([]int, []playback.DeliveryV3) { + r.mu.Lock() + defer r.mu.Unlock() + return append([]int(nil), r.files...), append([]playback.DeliveryV3(nil), r.plans...) +} + +// Starting playback must never wait on the H.264 copy-safety scan: it takes the +// cached-only ensure, and the plan it issues is handed to the racer that +// resolves the verdict behind it. +func TestHandleStartPlaybackV3DoesNotBlockOnCopySafetyScan(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := v3HandlerFixtureFile(t) + ensurer := &blockingProbeEnsurer{t: t} + racer := &recordingCopySafetyRacer{} + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.ProbeEnsurer = ensurer + handler.CopySafetyRacer = racer + + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", + strings.NewReader(marshalV3StartRequest(t, v3HandlerStartRequest()))).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + if ensurer.calls() == 0 { + t.Fatal("start did not resolve the cached copy-safety verdict") + } + files, _ := racer.raced() + if len(files) != 1 || files[0] != file.ID { + t.Fatalf("raced files = %v, want the planned file %d handed to the racer", files, file.ID) + } +} + +// The route test is the racer's; the handler's job is only to hand it the plan +// it issued, for every route, and to stay silent when no racer is wired. +func TestRaceCopySafetyV3HandsThePlanToTheRacer(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + racer := &recordingCopySafetyRacer{} + handler.CopySafetyRacer = racer + + handler.raceCopySafetyV3(42, &playback.PlanV3{PlanID: "plan-1", Delivery: playback.DeliveryRemuxHLSV3}) + handler.raceCopySafetyV3(0, &playback.PlanV3{PlanID: "plan-2", Delivery: playback.DeliveryRemuxHLSV3}) + handler.raceCopySafetyV3(43, nil) + + files, deliveries := racer.raced() + if len(files) != 1 || files[0] != 42 { + t.Fatalf("raced files = %v, want only the valid file/plan pair", files) + } + if len(deliveries) != 1 || deliveries[0] != playback.DeliveryRemuxHLSV3 { + t.Fatalf("raced deliveries = %v, want the issued plan's delivery", deliveries) + } + + without := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + without.raceCopySafetyV3(42, &playback.PlanV3{PlanID: "plan-1", Delivery: playback.DeliveryRemuxHLSV3}) +} diff --git a/internal/api/handlers/playback_realtime.go b/internal/api/handlers/playback_realtime.go index 47c0f16d5..0bf7ddd73 100644 --- a/internal/api/handlers/playback_realtime.go +++ b/internal/api/handlers/playback_realtime.go @@ -63,6 +63,44 @@ func (h *PlaybackHandler) abortPlaybackSessionByID(ctx context.Context, sessionI return h.abortPlaybackSession(ctx, session) } +// CopySafetyPlaybackControl adapts the playback handler to the session control +// playback.CopySafetyNotifier needs: the notifier owns the decision to withdraw +// a plan, the handler owns realtime command bookkeeping and session teardown. +type CopySafetyPlaybackControl struct { + playback *PlaybackHandler +} + +// NewCopySafetyPlaybackControl returns the adapter, or nil without a handler. +func NewCopySafetyPlaybackControl(handler *PlaybackHandler) *CopySafetyPlaybackControl { + if handler == nil { + return nil + } + return &CopySafetyPlaybackControl{playback: handler} +} + +func (c *CopySafetyPlaybackControl) RememberRealtimeCommand(commandID, sessionID string, name playback.CommandName) { + if c == nil { + return + } + c.playback.rememberRealtimeCommand(commandID, sessionID, name) +} + +func (c *CopySafetyPlaybackControl) ForgetRealtimeCommand(commandID string) { + if c == nil { + return + } + c.playback.forgetRealtimeCommand(commandID) +} + +// StopSession ends the session as a system teardown, not a user stop: the +// recipe card is kept so the client's recovery can rebuild from it. +func (c *CopySafetyPlaybackControl) StopSession(ctx context.Context, sessionID string) error { + if c == nil { + return playback.ErrSessionNotFound + } + return c.playback.stopPlaybackSessionByID(ctx, sessionID, false) +} + func (h *PlaybackHandler) rememberRealtimeCommand(commandID, sessionID string, name playback.CommandName) { if h == nil || commandID == "" || sessionID == "" { return diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 0d5797c65..403dc782c 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -860,6 +860,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr h.ChapterThumbnailQueuer.QueuePriorityFileAtPosition(r.Context(), effectiveFile.ID, session.Position) } h.maybeQueueLazyPlaybackMarkers(r.Context(), session, effectiveFile) + h.raceCopySafetyV3(effectiveFile.ID, result.Plan) h.persistSeriesSelectionsV3(r.Context(), userID, profileID, effectiveFile, plannedAudioTrackIndexV3(result, audioIndex)) h.syncSessionsNow(r.Context(), "v3_start") h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, Event: playback.RouteEventPlanSelectedV3, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputContextID: req.ClientPlaybackContext.Output.OutputContextID}, UserID: userID, ProfileID: profileID, ClientName: clientInfo.Name, ClientVersion: clientInfo.Version, ClientBuild: clientInfo.Build, ClientChannel: clientInfo.Channel, ClientModel: req.ClientPlaybackContext.Device.Model}) @@ -2300,9 +2301,21 @@ func (h *PlaybackHandler) HandleReplanPlaybackV3(w http.ResponseWriter, r *http. transport.afterDurableCommit() } } + h.raceCopySafetyV3(updated.EffectiveMediaFileID, response.PlaybackPlan) writeJSON(w, http.StatusOK, response) } +// raceCopySafetyV3 resolves an unknown H.264 copy-safety verdict behind a plan +// that stream-copies video. It is called after the durable commit on both the +// start and replan paths, so the scan only ever chases a route a client was +// actually handed, and it returns immediately — no response waits on it. +func (h *PlaybackHandler) raceCopySafetyV3(fileID int, plan *playback.PlanV3) { + if h == nil || h.CopySafetyRacer == nil || fileID <= 0 || plan == nil { + return + } + h.CopySafetyRacer.RaceScanForPlan(fileID, plan) +} + func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.AttemptRecordV3, req playback.ReplanRequestV3) (playback.DecisionResponseV3, playback.AttemptRecordV3, *preparedTransportV3, *transportErrorV3) { reservationHeld := false reservationHandedOff := false diff --git a/internal/api/handlers/session_ws.go b/internal/api/handlers/session_ws.go index 0b56ad713..b6338c7d2 100644 --- a/internal/api/handlers/session_ws.go +++ b/internal/api/handlers/session_ws.go @@ -178,6 +178,18 @@ func (h *PlaybackHandler) handleRealtimeClientMessage(sessionID string, data []b return playback.ErrInvalidRealtimePayload } if result.Status != playback.RealtimeResultStatusCompleted { + // A rejected plan_invalidated leaves the client running a route the + // server has withdrawn, and the tracker's deadline was already + // canceled by the result. Fall back to the same session stop an + // unnegotiated client gets; its recovery replans against the + // persisted verdict. + if record.Name == playback.CommandPlanInvalidated { + slog.Warn("client rejected a plan invalidation; stopping the session", + "session", sessionID, "playback_session_id", sessionID, "error", result.Error) + if err := h.stopPlaybackSessionByID(context.Background(), sessionID, false); err != nil && !errors.Is(err, playback.ErrSessionNotFound) { + slog.Error("failed to stop playback after a rejected plan invalidation", "session", sessionID, "playback_session_id", sessionID, "error", err) + } + } return nil } switch record.Name { @@ -186,6 +198,9 @@ func (h *PlaybackHandler) handleRealtimeClientMessage(sessionID string, data []b if err != nil && !errors.Is(err, playback.ErrSessionNotFound) { slog.Error("failed to stop playback after realtime completion", "session", sessionID, "playback_session_id", sessionID, "error", err) } + case playback.CommandPlanInvalidated: + // Completion means the client replanned itself; the session stays + // alive on its replacement plan and nothing else is required here. } return nil default: diff --git a/internal/api/handlers/session_ws_plan_invalidated_test.go b/internal/api/handlers/session_ws_plan_invalidated_test.go new file mode 100644 index 000000000..6d3c5dfc1 --- /dev/null +++ b/internal/api/handlers/session_ws_plan_invalidated_test.go @@ -0,0 +1,71 @@ +package handlers + +import ( + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +func realtimeResultMessage(t *testing.T, sessionID, commandID string, status playback.RealtimeResultStatus) []byte { + t.Helper() + data, err := json.Marshal(playback.ResultEnvelope{ + Type: playback.RealtimeMessageTypeResult, + CommandID: commandID, + SessionID: sessionID, + Status: status, + }) + if err != nil { + t.Fatalf("marshal result envelope: %v", err) + } + return data +} + +// A client that refuses a plan invalidation is left running a route the server +// has withdrawn, and its rejection already canceled the command deadline — +// so the rejection itself has to stop the session. +func TestRealtimeRejectedPlanInvalidationStopsSession(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr) + handler.CommandTracker = playback.NewCommandTracker() + defer handler.CommandTracker.Close() + + session, err := sessionMgr.StartSession(1, "profile-1", 100, playback.PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + handler.rememberRealtimeCommand("cmd-1", session.ID, playback.CommandPlanInvalidated) + + if err := handler.handleRealtimeClientMessage(session.ID, + realtimeResultMessage(t, session.ID, "cmd-1", playback.RealtimeResultStatusRejected)); err != nil { + t.Fatalf("handleRealtimeClientMessage: %v", err) + } + + if _, err := sessionMgr.GetSession(session.ID); err == nil { + t.Fatal("session survived a rejected plan invalidation, want it stopped") + } +} + +// A completed invalidation means the client replanned itself: the session must +// stay alive on its replacement plan. +func TestRealtimeCompletedPlanInvalidationKeepsSession(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr) + handler.CommandTracker = playback.NewCommandTracker() + defer handler.CommandTracker.Close() + + session, err := sessionMgr.StartSession(1, "profile-1", 100, playback.PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + handler.rememberRealtimeCommand("cmd-1", session.ID, playback.CommandPlanInvalidated) + + if err := handler.handleRealtimeClientMessage(session.ID, + realtimeResultMessage(t, session.ID, "cmd-1", playback.RealtimeResultStatusCompleted)); err != nil { + t.Fatalf("handleRealtimeClientMessage: %v", err) + } + + if _, err := sessionMgr.GetSession(session.ID); err != nil { + t.Fatalf("GetSession after a completed replan: %v, want the session kept", err) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index f1f924646..cab28d88e 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1049,6 +1049,29 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.MarkerUpserter = deps.FileRepo } playbackHandler.MarkerUpdateNotifier = playback.NewMarkerUpdateNotifier(deps.SessionMgr, realtimeHub) + // Optimistic remux: a play is never blocked on the H.264 copy-safety + // scan, so the scan runs behind the issued plan and the notifier moves + // any session that is already stream-copying an unsafe source off that + // route (or stops it, for a client that cannot be told). Both halves + // need the same probe ensurer the playback and detail surfaces use. + if copySafetyScanner, ok := deps.ProbeEnsurer.(playback.CopySafetyScanner); ok && deps.FileRepo != nil { + copySafetyRace := playback.NewCopySafetyRace( + copySafetyScanner, + deps.FileRepo, + playback.NewCopySafetyNotifier( + deps.SessionMgr, + playbackHandler.PlanStoreV3, + playbackHandler.CommandDispatcher, + handlers.NewCopySafetyPlaybackControl(playbackHandler), + ), + ) + if copySafetyRace != nil { + playbackHandler.CopySafetyRacer = copySafetyRace + if detailSvc != nil { + detailSvc.SetCopySafetyRacer(copySafetyRace) + } + } + } // A resolver lets subtitle realtime events carry the combined ordinal // the new track will hold in the next plan. Without a file repository // the notifier still fires; its events just omit the track block. diff --git a/internal/catalog/detail.go b/internal/catalog/detail.go index a4471ccb0..380111ed9 100644 --- a/internal/catalog/detail.go +++ b/internal/catalog/detail.go @@ -43,12 +43,27 @@ type batchDurationFetcher interface { FirstDurationsByEpisodeIDs(ctx context.Context, ids []string) (map[string]int, error) } +// PlaybackProbeEnsurer repairs probe metadata for catalog responses. Neither +// half of it runs the H.264 bitstream scan: no catalog surface may block on it. type PlaybackProbeEnsurer interface { - // Ensure repairs probe metadata and resolves the H.264 copy-safety - // verdict; EnsureProbeOnly does the repair alone. Browse surfaces use the - // latter — see prepareBrowseFiles. - Ensure(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + // EnsureProbeOnly does the probe repair alone. Browse surfaces use it — see + // prepareBrowseFiles. EnsureProbeOnly(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) + // EnsureCopySafetyCached adds the copy-safety verdict when it is already + // known, and never execs ffmpeg. Watch surfaces use it — see + // preparePlaybackFiles. + EnsureCopySafetyCached(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) +} + +// CopySafetyRacer resolves an unknown H.264 copy-safety verdict out of band. +// The watch page asks for it and never waits: the verdict is not part of the +// response, and any session that later ends up on a stream-copy route for the +// file is switched off it by the playback-side notifier when the scan lands. +// +// It is a narrow injected interface so the catalog keeps no dependency on the +// playback session machinery that implements it. +type CopySafetyRacer interface { + RaceScan(fileID int) } type ChapterThumbnailQueuer interface { @@ -662,6 +677,7 @@ type DetailService struct { workSummary WorkSummaryProvider originalLangFn func(context.Context, string) string probeEnsurer PlaybackProbeEnsurer + copySafetyRacer CopySafetyRacer chapterThumbs ChapterThumbnailQueuer // resolver is built once on first use; see settingsResolver. @@ -712,6 +728,15 @@ func (s *DetailService) SetProbeEnsurer(ensurer PlaybackProbeEnsurer) { s.probeEnsurer = ensurer } +// SetCopySafetyRacer wires the out-of-band H.264 copy-safety scan the watch +// surfaces trigger. Optional: without it an unknown verdict is simply left +// unknown until a play resolves it. +func (s *DetailService) SetCopySafetyRacer(racer CopySafetyRacer) { + if s != nil { + s.copySafetyRacer = racer + } +} + func (s *DetailService) SetChapterThumbnailQueuer(queuer ChapterThumbnailQueuer) { s.chapterThumbs = queuer } @@ -3698,9 +3723,15 @@ func fileIDOrZero(version *FileVersion) int { return version.FileID } -// preparePlaybackFiles repairs probe metadata and resolves the H.264 -// copy-safety verdict. Used by the watch surfaces, where a play is being -// prepared and the verdict is about to matter. +// preparePlaybackFiles repairs probe metadata and stamps the H.264 copy-safety +// verdict when it is already known. Used by the watch surfaces, where a play is +// about to be prepared and the verdict is about to matter. +// +// It deliberately does not wait for an unknown verdict: the bitstream scan is +// started in the background instead, so opening the watch page costs nothing +// even for a file nobody has played yet. No session exists at this point — if +// one appears and lands on a stream-copy route before the scan finishes, the +// playback-side notifier switches it off that route when the verdict lands. func (s *DetailService) preparePlaybackFiles(ctx context.Context, files []*models.MediaFile) []*models.MediaFile { return s.prepareFiles(ctx, files, true) } @@ -3727,7 +3758,7 @@ func (s *DetailService) prepareFiles(ctx context.Context, files []*models.MediaF var ensured *models.MediaFile var err error if withCopySafety { - ensured, err = s.probeEnsurer.Ensure(ctx, file) + ensured, err = s.probeEnsurer.EnsureCopySafetyCached(ctx, file) } else { ensured, err = s.probeEnsurer.EnsureProbeOnly(ctx, file) } @@ -3735,6 +3766,9 @@ func (s *DetailService) prepareFiles(ctx context.Context, files []*models.MediaF file = ensured } } + if withCopySafety && s.copySafetyRacer != nil && file.ID > 0 && file.VideoCopySafetyUnknown() { + s.copySafetyRacer.RaceScan(file.ID) + } prepared = append(prepared, file) } diff --git a/internal/catalog/detail_prepare_files_test.go b/internal/catalog/detail_prepare_files_test.go index f667ce414..84d516a42 100644 --- a/internal/catalog/detail_prepare_files_test.go +++ b/internal/catalog/detail_prepare_files_test.go @@ -10,58 +10,102 @@ import ( // recordingProbeEnsurer records which half of the ensurer contract each // prepare path asks for. type recordingProbeEnsurer struct { - fullCalls []int - probeCalls []int + probeCalls []int + cachedCalls []int } -func (e *recordingProbeEnsurer) Ensure(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { - e.fullCalls = append(e.fullCalls, file.ID) +func (e *recordingProbeEnsurer) EnsureProbeOnly(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.probeCalls = append(e.probeCalls, file.ID) return file, nil } -func (e *recordingProbeEnsurer) EnsureProbeOnly(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { - e.probeCalls = append(e.probeCalls, file.ID) +func (e *recordingProbeEnsurer) EnsureCopySafetyCached(_ context.Context, file *models.MediaFile) (*models.MediaFile, error) { + e.cachedCalls = append(e.cachedCalls, file.ID) return file, nil } +type recordingCopySafetyRacer struct { + raced []int +} + +func (r *recordingCopySafetyRacer) RaceScan(fileID int) { + r.raced = append(r.raced, fileID) +} + +func h264File(id int, multiplePPS *bool) *models.MediaFile { + return &models.MediaFile{ + ID: id, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{ + Codec: "h264", + MultiplePPS: multiplePPS, + }}, + } +} + // Browse detail must never trigger the H.264 copy-safety scan: the verdict is // not serialized into those responses, so the scan is pure warm-up and its // read is what made first-time browsing slow on remote storage. func TestPrepareBrowseFilesSkipsCopySafety(t *testing.T) { ensurer := &recordingProbeEnsurer{} - svc := &DetailService{probeEnsurer: ensurer} - files := []*models.MediaFile{{ID: 1}, {ID: 2}} + racer := &recordingCopySafetyRacer{} + svc := &DetailService{probeEnsurer: ensurer, copySafetyRacer: racer} + files := []*models.MediaFile{h264File(1, nil), h264File(2, nil)} prepared := svc.prepareBrowseFiles(context.Background(), files) if len(prepared) != 2 { t.Fatalf("prepareBrowseFiles() returned %d files, want 2", len(prepared)) } - if len(ensurer.fullCalls) != 0 { - t.Fatalf("browse path called Ensure for %v, want no copy-safety scans", ensurer.fullCalls) + if len(ensurer.cachedCalls) != 0 { + t.Fatalf("browse path resolved copy safety for %v, want probe repair only", ensurer.cachedCalls) } if len(ensurer.probeCalls) != 2 { t.Fatalf("browse path called EnsureProbeOnly %d times, want 2 — probe repair must still run", len(ensurer.probeCalls)) } + if len(racer.raced) != 0 { + t.Fatalf("browse path raced scans for %v, want none", racer.raced) + } } -// The watch surfaces are where a play is being prepared, so they keep the -// full ensure and warm the verdict while the user looks at the Play button. -func TestPreparePlaybackFilesKeepsCopySafety(t *testing.T) { +// The watch surfaces prepare a play, but must not block on the bitstream scan: +// they take the cached-only ensure and start the scan in the background. +func TestPreparePlaybackFilesUsesCachedEnsureAndRacesScan(t *testing.T) { ensurer := &recordingProbeEnsurer{} - svc := &DetailService{probeEnsurer: ensurer} - files := []*models.MediaFile{{ID: 1}, {ID: 2}} + racer := &recordingCopySafetyRacer{} + svc := &DetailService{probeEnsurer: ensurer, copySafetyRacer: racer} + files := []*models.MediaFile{h264File(1, nil), h264File(2, nil)} prepared := svc.preparePlaybackFiles(context.Background(), files) if len(prepared) != 2 { t.Fatalf("preparePlaybackFiles() returned %d files, want 2", len(prepared)) } - if len(ensurer.fullCalls) != 2 { - t.Fatalf("watch path called Ensure %d times, want 2", len(ensurer.fullCalls)) + if len(ensurer.cachedCalls) != 2 { + t.Fatalf("watch path called EnsureCopySafetyCached %d times, want 2", len(ensurer.cachedCalls)) + } + if len(racer.raced) != 2 || racer.raced[0] != 1 || racer.raced[1] != 2 { + t.Fatalf("watch path raced %v, want scans for files 1 and 2", racer.raced) } - if len(ensurer.probeCalls) != 0 { - t.Fatalf("watch path called EnsureProbeOnly for %v, want the full ensure", ensurer.probeCalls) +} + +// A file whose verdict is already known, or that is not H.264, has nothing to +// resolve: no background scan may be started for it. +func TestPreparePlaybackFilesSkipsRaceWhenNothingToScan(t *testing.T) { + known := false + ensurer := &recordingProbeEnsurer{} + racer := &recordingCopySafetyRacer{} + svc := &DetailService{probeEnsurer: ensurer, copySafetyRacer: racer} + files := []*models.MediaFile{ + h264File(1, &known), + {ID: 2, CodecVideo: "hevc", VideoTracks: []models.VideoTrack{{Codec: "hevc"}}}, + {ID: 3}, + } + + svc.preparePlaybackFiles(context.Background(), files) + + if len(racer.raced) != 0 { + t.Fatalf("watch path raced %v, want no scans for known or non-H.264 files", racer.raced) } } diff --git a/internal/models/media.go b/internal/models/media.go index 8e7175a61..e05b11434 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -9,6 +9,7 @@ const ( mediaBaseTypeAudiobook = "audiobook" mediaBaseTypePodcast = "podcast" mediaCodecMJPEG = "mjpeg" + mediaCodecH264 = "h264" ) // MediaFolder represents a row in the media_folders table. @@ -170,6 +171,68 @@ func (f *MediaFile) PrimaryDVProfile() int { return f.VideoTracks[0].DVProfile } +// VideoCopySafetyUnknown reports whether this file is an H.264 video whose +// multi-PPS copy-safety verdict is not stamped on the in-memory track. Only +// H.264 can carry the conflicting in-band parameter sets that make a video +// stream-copy unsafe, so every other codec is trivially known-safe. +// +// This is the single definition of "the verdict is still open", shared by the +// scanner that resolves it, the catalog surfaces that trigger the resolution, +// and playback. +func (f *MediaFile) VideoCopySafetyUnknown() bool { + if f == nil || len(f.VideoTracks) == 0 { + return false + } + if f.VideoTracks[0].MultiplePPS != nil { + return false + } + codec := strings.ToLower(strings.TrimSpace(f.VideoTracks[0].Codec)) + if codec == "" { + codec = strings.ToLower(strings.TrimSpace(f.CodecVideo)) + } + return codec == mediaCodecH264 || codec == "avc" || codec == "avc1" +} + +// PersistedVideoCopyVerdict returns the H.264 multi-PPS verdict recorded on the +// media_files row and whether it still describes the file as it stands. +// +// The verdict is self-validating: it is only honored while the size and mtime +// it was computed from still match the row, so a rewrite in place falls through +// to a rescan without any writer having to clear it. A verdict recorded for a +// row that carries no mtime is trusted on size alone — that is the only signal +// such a row has, and it is the same rule the scanner's in-process memo +// applies. +// +// This lives on the model because the row columns are loaded by every media +// file read, while the VideoTrack copy-safety flags are runtime-only and are +// stamped by the probe ensurer, which not every path that loads a file runs. +func (f *MediaFile) PersistedVideoCopyVerdict() (bool, bool) { + if f == nil || f.MultiplePPS == nil || f.MultiplePPSScanSize == nil { + return false, false + } + if *f.MultiplePPSScanSize != f.FileSize { + return false, false + } + if f.MultiplePPSScanMtime == nil || f.FileModifiedAt == nil { + if f.MultiplePPSScanMtime != nil || f.FileModifiedAt != nil { + return false, false + } + return *f.MultiplePPS, true + } + if !NormalizeFileModifiedAt(*f.MultiplePPSScanMtime).Equal(NormalizeFileModifiedAt(*f.FileModifiedAt)) { + return false, false + } + return *f.MultiplePPS, true +} + +// NormalizeFileModifiedAt puts a filesystem mtime in the one shape every +// comparison uses. Postgres stores microseconds and local filesystems report +// nanoseconds, so a round trip through the database is only equal to the value +// that was written after truncation. +func NormalizeFileModifiedAt(ts time.Time) time.Time { + return ts.UTC().Truncate(time.Microsecond) +} + // AudioOnlyProbeFacts is the compact probe shape needed to distinguish known // audio media from incomplete video probes and legacy attached cover art. type AudioOnlyProbeFacts struct { diff --git a/internal/playback/command_dispatcher_test.go b/internal/playback/command_dispatcher_test.go index 4872a28d7..33ddfe07d 100644 --- a/internal/playback/command_dispatcher_test.go +++ b/internal/playback/command_dispatcher_test.go @@ -2,19 +2,31 @@ package playback import ( "encoding/json" + "sync" "testing" "time" ) type dispatchTestConn struct { + mu sync.Mutex messages []any } func (c *dispatchTestConn) WriteJSON(v any) error { + c.mu.Lock() + defer c.mu.Unlock() c.messages = append(c.messages, v) return nil } +// sent is the safe reader for tests where a command is dispatched from a +// background goroutine rather than inline. +func (c *dispatchTestConn) sent() []any { + c.mu.Lock() + defer c.mu.Unlock() + return append([]any(nil), c.messages...) +} + func TestCommandDispatcherDispatchToSession(t *testing.T) { sessions := NewSessionManager(0, 0) session, err := sessions.StartSession(1, "profile-1", 100, PlayDirect, false) diff --git a/internal/playback/copy_safety_notifier.go b/internal/playback/copy_safety_notifier.go new file mode 100644 index 000000000..d13efc7bb --- /dev/null +++ b/internal/playback/copy_safety_notifier.go @@ -0,0 +1,364 @@ +package playback + +import ( + "context" + "errors" + "log/slog" + "time" + + "github.com/google/uuid" +) + +// CopySafetyInvalidationDeadline bounds how long a client has to ack a +// plan_invalidated command and report the replan it triggered. It is long +// enough for a client to run a full replan round trip and short enough that a +// wedged player is not left decoding a stream its decoder will desync on. When +// it expires the session is stopped, which is the same recovery an +// unnegotiated client gets. +const CopySafetyInvalidationDeadline = 8 * time.Second + +// CopySafetySessionSettleWindow is how long a session is treated as still being +// established. A session is registered with the manager well before its start +// handler has written the v3 attempt record, and the client cannot open its +// realtime channel until it has the response, so a verdict landing inside that +// window would see a session that looks unreachable and stop one that is still +// being built — a hard failure exactly where the graceful path was meant to +// apply. Sessions that are already reachable are acted on immediately; only the +// ones that would otherwise be stopped wait out the remainder of this window +// and are then re-examined once. +const CopySafetySessionSettleWindow = 5 * time.Second + +// copySafetyReconsiderTimeout bounds the work a deferred second look does. The +// scan context it inherited is already gone by then, so it carries no deadline +// of its own. +const copySafetyReconsiderTimeout = 30 * time.Second + +// CommandIssuedByServer marks a command the server originated on its own, +// distinct from the "admin" commands an operator sends from the admin surface. +const CommandIssuedByServer = "server" + +type copySafetySessionLookup interface { + GetSessionsByMediaFileID(fileID int) []*Session +} + +// copySafetyAttemptLookup reads the durable attempt for a live session: the +// plan currently issued for it and the features its client negotiated. +type copySafetyAttemptLookup interface { + GetAttempt(ctx context.Context, sessionID string) (*AttemptRecordV3, error) +} + +// CopySafetySessionControl is the small slice of playback-session lifecycle the +// notifier needs and does not own. The API handler implements it: it is the +// component that tracks realtime commands by ID and knows how to tear a +// playback session down completely. +type CopySafetySessionControl interface { + // RememberRealtimeCommand records a dispatched command so the realtime + // result handler can attribute it back to this notifier. + RememberRealtimeCommand(commandID, sessionID string, name CommandName) + // ForgetRealtimeCommand drops a command that was never delivered. + ForgetRealtimeCommand(commandID string) + // StopSession ends a playback session, which is the fallback whenever the + // command cannot be delivered or is not honored. + StopSession(ctx context.Context, sessionID string) error +} + +// CopySafetyNotifier switches live sessions off a video stream-copy route after +// the asynchronous H.264 copy-safety scan reports the source is unsafe to copy. +// +// Playback no longer waits for that scan, so a session can already be running a +// remux when the verdict lands. For a client that negotiated +// FeaturePlanInvalidatedV3 and is connected, the notifier pushes a +// plan_invalidated command and lets the client replan itself. Every other v3 +// session — no feature, no realtime connection, no ack, or a rejected result — +// is stopped, and the client's ordinary recovery mints a fresh attempt that +// plans against the now-persisted verdict and lands on a transcode. +// Jellyfin-compatibility sessions are exempt from the stop because their route +// decision never consults the verdict; see consider. +// +// Delivery is in-process only, matching the other realtime notifiers: it acts +// on the sessions this replica owns, which are exactly the ones whose realtime +// connections it holds. +type CopySafetyNotifier struct { + sessions copySafetySessionLookup + attempts copySafetyAttemptLookup + dispatcher *CommandDispatcher + control CopySafetySessionControl + deadline time.Duration + settle time.Duration +} + +// NewCopySafetyNotifier returns a notifier, or nil when a dependency it cannot +// work without is missing. A nil notifier is safe to call. +func NewCopySafetyNotifier( + sessions copySafetySessionLookup, + attempts copySafetyAttemptLookup, + dispatcher *CommandDispatcher, + control CopySafetySessionControl, +) *CopySafetyNotifier { + if sessions == nil || control == nil { + return nil + } + return &CopySafetyNotifier{ + sessions: sessions, + attempts: attempts, + dispatcher: dispatcher, + control: control, + deadline: CopySafetyInvalidationDeadline, + settle: CopySafetySessionSettleWindow, + } +} + +// VideoCopyUnsafe reports that fileID cannot be video stream-copied after all. +// Sessions that are not on a copy route for that file are left alone. +func (n *CopySafetyNotifier) VideoCopyUnsafe(ctx context.Context, fileID int) { + if n == nil || fileID <= 0 { + return + } + + for _, session := range n.sessions.GetSessionsByMediaFileID(fileID) { + n.consider(ctx, session, fileID, true) + } +} + +// consider decides what to do with one session the file lookup returned. +// maySettle is false on the deferred second look, so a session can never be +// postponed twice. +func (n *CopySafetyNotifier) consider(ctx context.Context, session *Session, fileID int, maySettle bool) { + if session == nil || session.ID == "" { + return + } + record := n.attempt(ctx, session.ID) + if !sessionServesFileV3(session, record, fileID) { + return + } + if !sessionOnVideoCopyRouteV3(session, record) { + return + } + if session.IsJellyfinCompat { + // Stopping only helps a client whose recovery re-decides the route + // against the verdict. The Jellyfin-protocol surface picks direct + // stream from the device profile and the catalog version alone + // (DeviceProfile.SupportsDirectStream) and never reads copy safety, so + // a compat client would reconnect straight back onto the same remux — + // the kill would be a pure mid-stream interruption with no remedy. + // Teaching that surface to read the verdict is the fix; until it does, + // leave these sessions playing. + slog.InfoContext(ctx, "leaving a Jellyfin-compatibility session on a copy-unsafe route", + "component", "playback", + "session_id", session.ID, + "file_id", fileID, + "reason", PlanInvalidatedVideoCopyUnsafe, + ) + return + } + if maySettle && !n.canTellClient(session, record) { + if wait := n.settleRemaining(session); wait > 0 { + n.reconsiderAfter(ctx, session.ID, fileID, wait) + return + } + } + n.invalidate(ctx, session, record, fileID) +} + +// reconsiderAfter re-examines one session once the settle window has passed. +// The scan context dies as soon as the caller returns, so the deferred look is +// deliberately not bound to its cancellation. +func (n *CopySafetyNotifier) reconsiderAfter(ctx context.Context, sessionID string, fileID int, wait time.Duration) { + parent := context.WithoutCancel(ctx) + go func() { + timer := time.NewTimer(wait) + defer timer.Stop() + <-timer.C + // The second look does its own plan-store read, which needs a bound of + // its own now that the scan's is gone. + ctx, cancel := context.WithTimeout(parent, copySafetyReconsiderTimeout) + defer cancel() + for _, session := range n.sessions.GetSessionsByMediaFileID(fileID) { + if session == nil || session.ID != sessionID { + continue + } + n.consider(ctx, session, fileID, false) + return + } + }() +} + +// settleRemaining reports how much of the settle window a session still has +// left. A session with no recorded start is treated as settled. +func (n *CopySafetyNotifier) settleRemaining(session *Session) time.Duration { + if n.settle <= 0 || session.StartedAt.IsZero() { + return 0 + } + remaining := n.settle - time.Since(session.StartedAt) + if remaining <= 0 { + return 0 + } + return remaining +} + +// canTellClient reports whether the session can be handed a plan_invalidated +// command instead of being stopped. +func (n *CopySafetyNotifier) canTellClient(session *Session, record *AttemptRecordV3) bool { + _, ok := n.tellablePlanID(session, record) + return ok +} + +// tellablePlanID returns the plan a plan_invalidated command would withdraw, +// and whether the session can be told about it at all. +func (n *CopySafetyNotifier) tellablePlanID(session *Session, record *AttemptRecordV3) (string, bool) { + if record == nil || n.dispatcher == nil || !session.HasRealtimeConnection { + return "", false + } + if !HasFeatureV3(record.NormalizedRequest.ClientFeatures, FeaturePlanInvalidatedV3) { + return "", false + } + planID := record.CurrentPlan.PlanID + if planID == "" { + planID = record.CurrentPlanID + } + if planID == "" { + return "", false + } + return planID, true +} + +func (n *CopySafetyNotifier) attempt(ctx context.Context, sessionID string) *AttemptRecordV3 { + if n.attempts == nil { + return nil + } + record, err := n.attempts.GetAttempt(ctx, sessionID) + if err != nil { + if !errors.Is(err, ErrSessionNotFound) { + // A session with no attempt and a session whose attempt could not be + // read lead to the same fallback, so the failure has to be visible: + // otherwise a plan-store outage looks exactly like a fleet of + // unnegotiated clients in the logs. + slog.WarnContext(ctx, "could not read the playback attempt for a copy-unsafe session", + "component", "playback", "session_id", sessionID, "error", err) + } + return nil + } + return record +} + +func (n *CopySafetyNotifier) invalidate(ctx context.Context, session *Session, record *AttemptRecordV3, fileID int) { + negotiated := record != nil && HasFeatureV3(record.NormalizedRequest.ClientFeatures, FeaturePlanInvalidatedV3) + planID, tellable := n.tellablePlanID(session, record) + + if !tellable { + slog.InfoContext(ctx, "stopping playback session on a copy-unsafe route", + "component", "playback", + "session_id", session.ID, + "file_id", fileID, + "reason", PlanInvalidatedVideoCopyUnsafe, + "negotiated_plan_invalidated", negotiated, + "realtime_connected", session.HasRealtimeConnection, + ) + n.stop(ctx, session.ID, fileID) + return + } + + commandID := uuid.NewString() + command, err := NewPlanInvalidatedCommand(session.ID, commandID, planID, PlanInvalidatedVideoCopyUnsafe) + if err != nil { + slog.WarnContext(ctx, "failed to encode plan invalidated command", + "component", "playback", "session_id", session.ID, "file_id", fileID, "error", err) + n.stop(ctx, session.ID, fileID) + return + } + command.Reason = PlanInvalidatedVideoCopyUnsafe + command.IssuedBy = &CommandIssuedBy{Kind: CommandIssuedByServer} + command.DeadlineMS = int(n.commandDeadline() / time.Millisecond) + + sessionID := session.ID + fallback := func() { + n.control.ForgetRealtimeCommand(commandID) + n.stop(context.WithoutCancel(ctx), sessionID, fileID) + } + + n.control.RememberRealtimeCommand(commandID, sessionID, CommandPlanInvalidated) + result := n.dispatcher.DispatchToSession(command, n.commandDeadline(), fallback) + if result.DispatchErr != nil { + // The command never reached the client, so nothing will ack it and the + // tracker has already dropped it. Stop the session directly rather than + // waiting out a deadline that will not fire. + n.control.ForgetRealtimeCommand(commandID) + slog.InfoContext(ctx, "plan invalidated command undeliverable; stopping session", + "component", "playback", "session_id", sessionID, "file_id", fileID, "error", result.DispatchErr) + n.stop(ctx, sessionID, fileID) + return + } + slog.InfoContext(ctx, "playback plan invalidated", + "component", "playback", + "session_id", sessionID, + "file_id", fileID, + "plan_id", planID, + "reason", PlanInvalidatedVideoCopyUnsafe, + "command_id", commandID, + ) +} + +func (n *CopySafetyNotifier) commandDeadline() time.Duration { + if n.deadline <= 0 { + return CopySafetyInvalidationDeadline + } + return n.deadline +} + +func (n *CopySafetyNotifier) stop(ctx context.Context, sessionID string, fileID int) { + if err := n.control.StopSession(ctx, sessionID); err != nil && !isSessionGoneV3(err) { + slog.WarnContext(ctx, "failed to stop playback session on a copy-unsafe route", + "component", "playback", "session_id", sessionID, "file_id", fileID, "error", err) + } +} + +func isSessionGoneV3(err error) bool { + return errors.Is(err, ErrSessionNotFound) +} + +// sessionServesFileV3 reports whether fileID is the source the session is +// actually delivering. +// +// The session lookup matches the requested file as well as the effective one, +// which is right for the additive notifiers that share it but wrong here: after +// the 4K guard or a version replan picks another edition, a session's requested +// file is an identity the session no longer streams a byte of. Withdrawing its +// route because a different edition turned out to be copy-unsafe would cost it +// a perfectly valid remux. +func sessionServesFileV3(session *Session, record *AttemptRecordV3, fileID int) bool { + if session == nil { + return false + } + if record != nil && record.EffectiveMediaFileID > 0 { + return record.EffectiveMediaFileID == fileID + } + return session.MediaFileID == fileID +} + +// sessionOnVideoCopyRouteV3 reports whether the session is currently serving a +// video stream-copy. The durable plan wins when there is one — it advances +// atomically with each completed replan, while the live play method can lag — +// and the session's own play method covers sessions with no v3 attempt at all +// (reconstructed sessions, and sessions whose attempt has expired). +// +// Direct play is deliberately not a copy route here: multi-PPS only breaks the +// avc1/fMP4 repackaging a remux performs, and the planner gates only the remux +// branches on it. +func sessionOnVideoCopyRouteV3(session *Session, record *AttemptRecordV3) bool { + if session == nil { + return false + } + if record != nil && record.CurrentPlan.Delivery != "" { + switch record.CurrentPlan.Delivery { + case DeliveryRemuxHLSV3, DeliveryRemuxProgressiveV3: + return true + default: + return false + } + } + method := session.BasePlayMethod + if method == "" { + method = session.PlayMethod + } + return method == PlayRemux +} diff --git a/internal/playback/copy_safety_notifier_test.go b/internal/playback/copy_safety_notifier_test.go new file mode 100644 index 000000000..ef6d48f85 --- /dev/null +++ b/internal/playback/copy_safety_notifier_test.go @@ -0,0 +1,466 @@ +package playback + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" +) + +type fakeCopySafetyControl struct { + mu sync.Mutex + remembered []playbackCommandNote + forgotten []string + stopped []string +} + +type playbackCommandNote struct { + commandID string + sessionID string + name CommandName +} + +func (c *fakeCopySafetyControl) RememberRealtimeCommand(commandID, sessionID string, name CommandName) { + c.mu.Lock() + defer c.mu.Unlock() + c.remembered = append(c.remembered, playbackCommandNote{commandID: commandID, sessionID: sessionID, name: name}) +} + +func (c *fakeCopySafetyControl) ForgetRealtimeCommand(commandID string) { + c.mu.Lock() + defer c.mu.Unlock() + c.forgotten = append(c.forgotten, commandID) +} + +func (c *fakeCopySafetyControl) StopSession(_ context.Context, sessionID string) error { + c.mu.Lock() + defer c.mu.Unlock() + c.stopped = append(c.stopped, sessionID) + return nil +} + +func (c *fakeCopySafetyControl) stoppedSessions() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.stopped...) +} + +func (c *fakeCopySafetyControl) trackedCommands() []playbackCommandNote { + c.mu.Lock() + defer c.mu.Unlock() + return append([]playbackCommandNote(nil), c.remembered...) +} + +type fakeAttemptLookup struct { + records map[string]*AttemptRecordV3 +} + +func (l *fakeAttemptLookup) GetAttempt(_ context.Context, sessionID string) (*AttemptRecordV3, error) { + record, ok := l.records[sessionID] + if !ok { + return nil, ErrSessionNotFound + } + return record, nil +} + +func remuxAttempt(sessionID, planID string, features ...string) *AttemptRecordV3 { + return &AttemptRecordV3{ + SessionID: sessionID, + CurrentPlanID: planID, + CurrentPlan: PlanV3{ + PlanID: planID, + Delivery: DeliveryRemuxHLSV3, + }, + NormalizedRequest: StartRequestV3{ClientFeatures: features}, + } +} + +func newCopySafetyFixture(t *testing.T) (*SessionManager, *RealtimeHub, *CommandTracker, *fakeCopySafetyControl) { + t.Helper() + sessions := NewSessionManager(0, 0) + hub := NewRealtimeHub() + tracker := NewCommandTracker() + t.Cleanup(tracker.Close) + return sessions, hub, tracker, &fakeCopySafetyControl{} +} + +// A negotiated, connected client is told to replan; the session keeps playing +// until it reports back. +func TestCopySafetyNotifierPushesPlanInvalidated(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{ + session.ID: remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3), + }} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if len(conn.messages) != 1 { + t.Fatalf("messages = %d, want 1 plan_invalidated command", len(conn.messages)) + } + command, ok := conn.messages[0].(CommandEnvelope) + if !ok { + t.Fatalf("message type = %T, want CommandEnvelope", conn.messages[0]) + } + if command.Type != RealtimeMessageTypeCommand || command.Name != CommandPlanInvalidated { + t.Fatalf("command = %#v, want a plan_invalidated command", command) + } + if command.DeadlineMS != int(CopySafetyInvalidationDeadline/time.Millisecond) { + t.Fatalf("deadline_ms = %d, want %d", command.DeadlineMS, int(CopySafetyInvalidationDeadline/time.Millisecond)) + } + var payload PlanInvalidatedPayload + if err := json.Unmarshal(command.Payload, &payload); err != nil { + t.Fatalf("json.Unmarshal(payload): %v", err) + } + if payload.Reason != PlanInvalidatedVideoCopyUnsafe || payload.PlanID != "plan-abc" { + t.Fatalf("payload = %#v, want the invalidated plan and the copy-unsafe reason", payload) + } + tracked := control.trackedCommands() + if len(tracked) != 1 || tracked[0].sessionID != session.ID || tracked[0].name != CommandPlanInvalidated { + t.Fatalf("tracked commands = %#v, want one plan_invalidated for the session", tracked) + } + if tracked[0].commandID != command.CommandID { + t.Fatalf("tracked command id = %q, want the dispatched %q", tracked[0].commandID, command.CommandID) + } + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped %v, want the session left running until the client reports back", stopped) + } +} + +// Everything that cannot be told to replan is stopped instead: that fallback is +// the whole backwards-compatibility story for clients shipped before the token. +func TestCopySafetyNotifierStopsSessionsItCannotTell(t *testing.T) { + tests := []struct { + name string + features []string + connected bool + attempt bool + }{ + {name: "feature not negotiated", features: []string{FeatureSeekReanchorV3}, connected: true, attempt: true}, + {name: "no realtime connection", features: []string{FeaturePlanInvalidatedV3}, connected: false, attempt: true}, + {name: "no durable attempt", connected: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if tc.connected { + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{}} + if tc.attempt { + attempts.records[session.ID] = remuxAttempt(session.ID, "plan-abc", tc.features...) + } + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 0 + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if len(conn.messages) != 0 { + t.Fatalf("messages = %d, want no command pushed", len(conn.messages)) + } + if stopped := control.stoppedSessions(); len(stopped) != 1 || stopped[0] != session.ID { + t.Fatalf("stopped = %v, want the session terminated", stopped) + } + }) + } +} + +// Only sessions actually stream-copying video for the file are touched: a +// transcode is already safe, and a direct play never repackages into fMP4. +func TestCopySafetyNotifierIgnoresNonCopyRoutes(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + transcoding, err := sessions.StartSession(1, "profile-1", 100, PlayTranscode, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + direct, err := sessions.StartSession(2, "profile-2", 100, PlayDirect, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + otherFile, err := sessions.StartSession(3, "profile-3", 101, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + transcodeAttempt := remuxAttempt(transcoding.ID, "plan-transcode", FeaturePlanInvalidatedV3) + transcodeAttempt.CurrentPlan.Delivery = DeliveryTranscodeHLSV3 + directAttempt := remuxAttempt(direct.ID, "plan-direct", FeaturePlanInvalidatedV3) + directAttempt.CurrentPlan.Delivery = DeliveryOriginalHTTPV3 + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{ + transcoding.ID: transcodeAttempt, + direct.ID: directAttempt, + otherFile.ID: remuxAttempt(otherFile.ID, "plan-other", FeaturePlanInvalidatedV3), + }} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want no session touched", stopped) + } + if tracked := control.trackedCommands(); len(tracked) != 0 { + t.Fatalf("tracked = %#v, want no command pushed", tracked) + } +} + +// A session with no v3 attempt at all (a reconstructed session, or one whose +// attempt expired) is classified by its live play method, and stopped because +// it can be told nothing. +func TestCopySafetyNotifierUsesSessionPlayMethodWithoutAnAttempt(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + remuxing, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + transcoding, err := sessions.StartSession(2, "profile-2", 100, PlayTranscode, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + notifier := NewCopySafetyNotifier(sessions, nil, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 0 + notifier.VideoCopyUnsafe(context.Background(), 100) + + stopped := control.stoppedSessions() + if len(stopped) != 1 || stopped[0] != remuxing.ID { + t.Fatalf("stopped = %v, want only the remuxing session %q (not %q)", stopped, remuxing.ID, transcoding.ID) + } +} + +// The Jellyfin-protocol surface picks direct stream from the device profile +// alone, so a compat client reconnects onto the identical remux. Stopping it +// would be a mid-stream interruption that buys nothing. +func TestCopySafetyNotifierLeavesJellyfinCompatSessionsAlone(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + ctx := WithClientInfo(context.Background(), ClientInfo{Name: "Infuse", IsCompat: true}) + compat, err := sessions.StartSessionWithFilesContext(ctx, 1, "profile-1", 100, 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSessionWithFilesContext: %v", err) + } + if !compat.IsJellyfinCompat { + t.Fatal("fixture session is not a compat session; the test would prove nothing") + } + native, err := sessions.StartSession(2, "profile-2", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + notifier := NewCopySafetyNotifier(sessions, nil, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 0 + notifier.VideoCopyUnsafe(context.Background(), 100) + + stopped := control.stoppedSessions() + if len(stopped) != 1 || stopped[0] != native.ID { + t.Fatalf("stopped = %v, want only the native session %q (not the compat session %q)", stopped, native.ID, compat.ID) + } +} + +// A session is registered with the manager before its start handler has written +// the attempt record and long before the client has opened a realtime channel. +// A verdict landing in that window must not stop a session that is still being +// built: the notifier waits out the settle window and looks again. +func TestCopySafetyNotifierWaitsOutTheSettleWindowBeforeStopping(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{}} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 60 * time.Millisecond + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want the still-establishing session left alone", stopped) + } + + // The start finishes inside the window: the attempt lands and the client + // connects, so the second look finds a session it can tell instead of kill. + attempts.records[session.ID] = remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3) + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for len(conn.sent()) == 0 { + if time.Now().After(deadline) { + t.Fatalf("no command pushed after the settle window; stopped = %v", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } + sent := conn.sent() + if command, ok := sent[0].(CommandEnvelope); !ok || command.Name != CommandPlanInvalidated { + t.Fatalf("message = %#v, want a plan_invalidated command", sent[0]) + } + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want the settled session told rather than terminated", stopped) + } +} + +// A session that never becomes reachable is still stopped, just one settle +// window later than an already-settled one. +func TestCopySafetyNotifierStopsAfterTheSettleWindow(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + notifier := NewCopySafetyNotifier(sessions, nil, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 20 * time.Millisecond + + notifier.VideoCopyUnsafe(context.Background(), 100) + + deadline := time.Now().Add(2 * time.Second) + for { + if stopped := control.stoppedSessions(); len(stopped) == 1 && stopped[0] == session.ID { + return + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the unreachable session terminated after the settle window", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } +} + +// The session lookup also matches the requested file, which after a 4K guard or +// a version replan is not the file being streamed. A verdict about that +// inactive edition must not touch a session remuxing a different, copy-safe one. +func TestCopySafetyNotifierIgnoresSessionsServingAnotherFile(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSessionWithFiles(1, "profile-1", 11, 10, PlayRemux, false) + if err != nil { + t.Fatalf("StartSessionWithFiles: %v", err) + } + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + record := remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3) + record.RequestedMediaFileID = 10 + record.EffectiveMediaFileID = 11 + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{session.ID: record}} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 0 + + // File 10 is the requested edition the session abandoned; file 11 is playing. + notifier.VideoCopyUnsafe(context.Background(), 10) + + if len(conn.messages) != 0 { + t.Fatalf("messages = %d, want nothing pushed for an edition the session is not streaming", len(conn.messages)) + } + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want the session on the other edition left alone", stopped) + } + + notifier.VideoCopyUnsafe(context.Background(), 11) + if len(conn.messages) != 1 { + t.Fatalf("messages = %d, want the effective file's verdict to reach the session", len(conn.messages)) + } +} + +// The command carries a deadline: a client that acks and then goes quiet loses +// its session, so it cannot keep decoding a stream the server withdrew. +func TestCopySafetyNotifierDeadlineStopsUnansweredSession(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{ + session.ID: remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3), + }} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.deadline = 10 * time.Millisecond + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if len(conn.messages) != 1 { + t.Fatalf("messages = %d, want the command delivered first", len(conn.messages)) + } + deadline := time.Now().Add(2 * time.Second) + for { + if stopped := control.stoppedSessions(); len(stopped) == 1 && stopped[0] == session.ID { + break + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the unanswered session terminated by the deadline", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } + // An acked-then-completed command cancels the deadline; here nothing + // answered, so the tracker must have released the command as well. + if _, tracked := tracker.Status("unused"); tracked { + t.Fatal("tracker reported an unknown command as tracked") + } +} + +// An acked command whose result completes leaves the session alone: the client +// replanned itself. +func TestCopySafetyNotifierCompletedResultKeepsSession(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { + t.Fatalf("SetRealtimeConnection: %v", err) + } + conn := &dispatchTestConn{} + reg := hub.Register(session.ID, conn) + defer hub.Unregister(reg) + + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{ + session.ID: remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3), + }} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.deadline = 50 * time.Millisecond + + notifier.VideoCopyUnsafe(context.Background(), 100) + command := conn.messages[0].(CommandEnvelope) + tracker.Ack(command.CommandID) + tracker.Result(command.CommandID) + + time.Sleep(150 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want the session kept after a completed replan", stopped) + } +} diff --git a/internal/playback/copy_safety_race.go b/internal/playback/copy_safety_race.go new file mode 100644 index 000000000..96843f31f --- /dev/null +++ b/internal/playback/copy_safety_race.go @@ -0,0 +1,136 @@ +package playback + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// copySafetyScanTimeout bounds one asynchronous multi-PPS scan. The scan reads +// the opening seconds of the file, which on cold remote storage is dominated by +// the read rather than the demux; a minute is generous for that and still +// guarantees the goroutine cannot outlive the session it was started for by +// much. It is deliberately not the request timeout: nothing about this work +// belongs to the HTTP request that triggered it. +const copySafetyScanTimeout = time.Minute + +// CopySafetyScanner is the scanner-side half of the race: it decides whether a +// file still needs the H.264 multi-PPS scan and runs it. *scanner.PlaybackProbeEnsurer +// implements it. +type CopySafetyScanner interface { + NeedsCopySafetyScan(file *models.MediaFile) bool + ScanCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) +} + +// CopySafetyFileLoader loads the media file a race was requested for. +type CopySafetyFileLoader interface { + GetByID(ctx context.Context, id int) (*models.MediaFile, error) +} + +// CopySafetyRace runs the H.264 copy-safety scan out of band, after a plan that +// stream-copies video has already been handed to a client (or a watch page has +// been rendered for a file no session exists for yet). +// +// This is the asynchronous half of optimistic remuxing: an unknown verdict no +// longer blocks a play, so it has to be resolved behind the play instead. A +// multi-PPS verdict is both persisted — every later plan for the file excludes +// the copy route deterministically — and pushed at whatever sessions are live +// on a copy route by the time it lands. +type CopySafetyRace struct { + scanner CopySafetyScanner + files CopySafetyFileLoader + notifier *CopySafetyNotifier + // inFlight keeps one goroutine per file. The scanner's own singleflight + // already collapses concurrent scans, but every start, replan and watch-page + // load for a popular file would otherwise stack a goroutine that does + // nothing but wait on it. + inFlight sync.Map // file ID -> struct{} + timeout time.Duration +} + +// NewCopySafetyRace returns a racer, or nil when it has nothing to scan with. A +// nil racer is safe to call. +func NewCopySafetyRace(scanner CopySafetyScanner, files CopySafetyFileLoader, notifier *CopySafetyNotifier) *CopySafetyRace { + if scanner == nil || files == nil { + return nil + } + return &CopySafetyRace{scanner: scanner, files: files, notifier: notifier, timeout: copySafetyScanTimeout} +} + +// RaceScan resolves the copy-safety verdict for fileID in the background. It +// returns immediately, and does nothing when the verdict is already known, the +// file is not H.264, or a scan for the file is already running. +// +// The caller's request context is deliberately not used: the scan outlives the +// request that noticed the verdict was missing, and the whole point is that no +// client ever waits on it. +func (r *CopySafetyRace) RaceScan(fileID int) { + if r == nil || fileID <= 0 { + return + } + if _, running := r.inFlight.LoadOrStore(fileID, struct{}{}); running { + return + } + go func() { + defer r.inFlight.Delete(fileID) + r.scan(fileID) + }() +} + +func (r *CopySafetyRace) scan(fileID int) { + timeout := r.timeout + if timeout <= 0 { + timeout = copySafetyScanTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + file, err := r.files.GetByID(ctx, fileID) + if err != nil || file == nil { + if err != nil { + slog.WarnContext(ctx, "video copy-safety race could not load the file", + "component", "playback", "file_id", fileID, "error", err) + } + return + } + if !r.scanner.NeedsCopySafetyScan(file) { + return + } + + multi, err := r.scanner.ScanCopySafety(ctx, file) + if err != nil { + // An inconclusive scan is not evidence of anything. Nothing is persisted + // (the scanner only records a verdict it reached), live sessions keep + // playing the route they were given, and a later request retries. The + // old behavior — treating a failed scan as copy-unsafe — belonged to a + // world where the scan ran before playback started. + slog.WarnContext(ctx, "video copy-safety scan failed", + "component", "playback", "file_id", fileID, "error", err) + return + } + if !multi { + return + } + + slog.InfoContext(ctx, "video copy-safety scan disqualified the stream-copy route", + "component", "playback", "file_id", fileID) + r.notifier.VideoCopyUnsafe(ctx, fileID) +} + +// RaceScanForPlan starts a race only when the plan actually stream-copies video +// for this file. Callers on the playback start and replan paths use it so the +// route test lives in one place. +func (r *CopySafetyRace) RaceScanForPlan(fileID int, plan *PlanV3) { + if r == nil || plan == nil { + return + } + switch plan.Delivery { + case DeliveryRemuxHLSV3, DeliveryRemuxProgressiveV3: + default: + return + } + r.RaceScan(fileID) +} diff --git a/internal/playback/copy_safety_race_test.go b/internal/playback/copy_safety_race_test.go new file mode 100644 index 000000000..7ea0c7733 --- /dev/null +++ b/internal/playback/copy_safety_race_test.go @@ -0,0 +1,234 @@ +package playback + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type fakeCopySafetyScanner struct { + mu sync.Mutex + needs bool + multi bool + err error + scans int + release chan struct{} + scanning chan struct{} +} + +func (s *fakeCopySafetyScanner) NeedsCopySafetyScan(*models.MediaFile) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.needs +} + +func (s *fakeCopySafetyScanner) ScanCopySafety(context.Context, *models.MediaFile) (bool, error) { + s.mu.Lock() + s.scans++ + s.mu.Unlock() + if s.scanning != nil { + s.scanning <- struct{}{} + } + if s.release != nil { + <-s.release + } + return s.multi, s.err +} + +func (s *fakeCopySafetyScanner) scanCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.scans +} + +type fakeFileLoader struct { + mu sync.Mutex + file *models.MediaFile + err error + loads int +} + +func (l *fakeFileLoader) GetByID(context.Context, int) (*models.MediaFile, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.loads++ + return l.file, l.err +} + +// raceFixture wires a racer whose notifier reports into a fake control, so a +// multi-PPS verdict is observable as a session stop. +func raceFixture(t *testing.T, scanner *fakeCopySafetyScanner) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { + t.Helper() + sessions := NewSessionManager(0, 0) + hub := NewRealtimeHub() + tracker := NewCommandTracker() + t.Cleanup(tracker.Close) + control := &fakeCopySafetyControl{} + notifier := NewCopySafetyNotifier(sessions, nil, NewCommandDispatcher(sessions, hub, tracker), control) + // These tests are about the race, not about waiting out the window a + // just-started session gets before it can be stopped. + notifier.settle = 0 + loader := &fakeFileLoader{file: &models.MediaFile{ID: 100, CodecVideo: "h264", VideoTracks: []models.VideoTrack{{Codec: "h264"}}}} + return NewCopySafetyRace(scanner, loader, notifier), sessions, control +} + +func waitForStop(t *testing.T, control *fakeCopySafetyControl, sessionID string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + stopped := control.stoppedSessions() + if len(stopped) == 1 && stopped[0] == sessionID { + return + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the copy-routed session %q", stopped, sessionID) + } + time.Sleep(time.Millisecond) + } +} + +func TestCopySafetyRaceNotifiesOnMultiPPS(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, multi: true} + race, sessions, control := raceFixture(t, scanner) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForStop(t, control, session.ID) +} + +// A copy-safe verdict is the common case and must be silent: the plan the +// client is already running stays valid. +func TestCopySafetyRaceKeepsSessionsWhenCopySafe(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, multi: false} + race, sessions, control := raceFixture(t, scanner) + if _, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false); err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForScans(t, scanner, 1) + time.Sleep(20 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want no session touched by a copy-safe verdict", stopped) + } +} + +// An inconclusive scan proves nothing, so sessions keep playing. Failing closed +// here would kill a live playback over a transient ffmpeg or storage error. +func TestCopySafetyRaceLeavesSessionsAloneOnScanError(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, err: errors.New("ffmpeg exploded")} + race, sessions, control := raceFixture(t, scanner) + if _, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false); err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForScans(t, scanner, 1) + time.Sleep(20 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want live sessions untouched after an inconclusive scan", stopped) + } +} + +func TestCopySafetyRaceSkipsFilesWithNothingToScan(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: false, multi: true} + race, _, control := raceFixture(t, scanner) + + race.RaceScan(100) + + time.Sleep(20 * time.Millisecond) + if got := scanner.scanCount(); got != 0 { + t.Fatalf("scans = %d, want 0 when the verdict is already known", got) + } + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want none", stopped) + } +} + +// Every start, replan and watch-page load for a popular file asks for the same +// race; only one goroutine may be in flight for it. +func TestCopySafetyRaceDedupesInFlightScans(t *testing.T) { + scanner := &fakeCopySafetyScanner{ + needs: true, + release: make(chan struct{}), + scanning: make(chan struct{}, 1), + } + race, _, _ := raceFixture(t, scanner) + + race.RaceScan(100) + <-scanner.scanning + for i := 0; i < 5; i++ { + race.RaceScan(100) + } + close(scanner.release) + + time.Sleep(50 * time.Millisecond) + if got := scanner.scanCount(); got != 1 { + t.Fatalf("scans = %d, want 1 while a scan for the file is already running", got) + } +} + +// The route test lives with the racer so start and replan cannot disagree: only +// a plan that actually stream-copies video is worth chasing. +func TestCopySafetyRaceForPlanOnlyChasesCopyRoutes(t *testing.T) { + tests := []struct { + delivery DeliveryV3 + wantScans int + }{ + {delivery: DeliveryRemuxHLSV3, wantScans: 1}, + {delivery: DeliveryRemuxProgressiveV3, wantScans: 1}, + {delivery: DeliveryTranscodeHLSV3, wantScans: 0}, + {delivery: DeliveryOriginalHTTPV3, wantScans: 0}, + } + + for _, tc := range tests { + t.Run(string(tc.delivery), func(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true} + race, _, _ := raceFixture(t, scanner) + + race.RaceScanForPlan(100, &PlanV3{PlanID: "plan-1", Delivery: tc.delivery}) + + if tc.wantScans > 0 { + waitForScans(t, scanner, tc.wantScans) + return + } + time.Sleep(20 * time.Millisecond) + if got := scanner.scanCount(); got != 0 { + t.Fatalf("scans = %d, want 0 for delivery %q", got, tc.delivery) + } + }) + } +} + +func TestCopySafetyRaceNilIsSafe(t *testing.T) { + var race *CopySafetyRace + race.RaceScan(100) + race.RaceScanForPlan(100, &PlanV3{Delivery: DeliveryRemuxHLSV3}) + if NewCopySafetyRace(nil, nil, nil) != nil { + t.Fatal("NewCopySafetyRace() with no dependencies = non-nil, want nil") + } +} + +func waitForScans(t *testing.T, scanner *fakeCopySafetyScanner, want int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if scanner.scanCount() >= want { + return + } + if time.Now().After(deadline) { + t.Fatalf("scans = %d, want %d", scanner.scanCount(), want) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index b9c7bd456..f162fdfd9 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -46,14 +46,26 @@ const ( // origin, so every byte egresses from the API server; with it the plan may // hand out credential-free proxy origins and distributed egress is restored. FeatureAuthorizedMediaOriginsV3 = "authorized_media_origins_v1" - PlanRecipeVersionV3 = "v3.4" - ClientDV7ToDV81V3 = "client_dv7_to_dv81" - ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" - ClientDVTransformVersionV3 = "1" - ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" - ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" - ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" - DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" + // FeaturePlanInvalidatedV3 is the client's promise to handle the realtime + // plan_invalidated command: ack it, replan off the named plan with operation + // failure_recovery and the invalidated plan's attempt key in + // attempted_plan_keys, then report the result. + // + // It exists because the server may only learn a route is wrong after the + // plan is already playing — the H.264 copy-safety scan now runs + // asynchronously so playback never waits on it. A session that did not + // negotiate the token, or has no realtime connection, is stopped instead; + // the client's ordinary recovery then mints a fresh attempt that plans + // against the now-persisted verdict. + FeaturePlanInvalidatedV3 = "plan_invalidated_v1" + PlanRecipeVersionV3 = "v3.4" + ClientDV7ToDV81V3 = "client_dv7_to_dv81" + ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" + ClientDVTransformVersionV3 = "1" + ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" + ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" + ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" + DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" ) // ServerFeaturesV3 returns the complete feature set advertised by protocol-v3 @@ -72,6 +84,7 @@ func ServerFeaturesV3() []string { FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3, + FeaturePlanInvalidatedV3, // Advertised so a client can tell "this server does not populate // source.duration_seconds" apart from "this server knows the runtime // is genuinely unknown". Without the distinction both look like an diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index 8f2520697..ff63f3af5 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -5,6 +5,7 @@ import ( "os" "strings" "testing" + "time" "github.com/Silo-Server/silo-server/internal/models" ) @@ -33,6 +34,7 @@ func TestServerFeaturesV3ReturnsCompleteIndependentSlices(t *testing.T) { FeatureHeaderAuthenticatedMediaV3: {}, FeatureAuthorizedMediaOriginsV3: {}, FeatureSoftwareVideoDecodeV3: {}, + FeaturePlanInvalidatedV3: {}, FeaturePlanSourceDurationV3: {}, } if len(first) != len(expected) { @@ -1104,6 +1106,23 @@ func TestPlanPlaybackV3CopySafeSourceStillCopies(t *testing.T) { } } +// An unresolved verdict plans optimistically. Playback no longer waits on the +// multi-second bitstream scan, so "not scanned yet" must read as "copy is +// allowed"; the scan runs behind the issued plan and CopySafetyNotifier moves +// the session off this route if it comes back multi-PPS. +func TestPlanPlaybackV3UnknownCopySafetyStillCopies(t *testing.T) { + file, req := copyUnsafeFixtureV3(false) + file.VideoTracks[0].MultiplePPS = nil + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if result.Plan.Source.VideoCopyUnsafe { + t.Fatal("source.video_copy_unsafe = true for an unresolved verdict, want false") + } +} + func TestPlanPlaybackV3FallsBackFromProgressiveToHLSWithoutRepeatingKey(t *testing.T) { file := detailedFixtureFileV3() file.VideoTracks[0].VideoRange = "SDR" @@ -1129,6 +1148,52 @@ func TestPlanPlaybackV3FallsBackFromProgressiveToHLSWithoutRepeatingKey(t *testi } } +// The copy-safety verdict has to be readable from the row alone. The track +// flags are stamped by the probe ensurer, which the replan path and the +// Jellyfin-protocol route decision never run; without this the replan a +// plan_invalidated command triggers would just walk to the sibling stream-copy +// delivery, which is broken for exactly the same reason. +func TestPlanPlaybackV3HonorsThePersistedCopySafetyVerdict(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + req.Capabilities.Containers = []string{"mp4"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + + // Only the persisted columns carry the verdict, exactly as a raw repository + // read delivers them. + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + multi := true + scanSize := file.FileSize + file.FileModifiedAt = &mtime + file.MultiplePPS = &multi + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &mtime + if file.VideoTracks[0].MultiplePPS != nil || file.VideoTracks[0].VideoCopyUnsafe { + t.Fatal("fixture already carries the runtime copy-safety flags; the test would prove nothing") + } + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + switch result.Plan.Delivery { + case DeliveryRemuxProgressiveV3, DeliveryRemuxHLSV3: + t.Fatalf("delivery = %q, want a route that does not stream-copy a copy-unsafe source", result.Plan.Delivery) + } + + // A stale verdict (the file was rewritten) must not be honored. + staleSize := file.FileSize + 1 + file.MultiplePPSScanSize = &staleSize + stale := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if stale.Plan == nil || stale.Plan.Delivery != DeliveryRemuxProgressiveV3 { + t.Fatalf("stale verdict = %s, want the ordinary remux back", ExplainPlannerResultV3(stale)) + } +} + // pgsBurnRequestV3 selects the single embedded PGS track on a client that // cannot render bitmap subtitles anywhere, so every route but a burn-in // transcode is closed by the subtitle alone. diff --git a/internal/playback/realtime.go b/internal/playback/realtime.go index 78fccbd5e..54254319a 100644 --- a/internal/playback/realtime.go +++ b/internal/playback/realtime.go @@ -57,6 +57,13 @@ const ( CommandPlayMedia CommandName = "play_media" CommandSetAudioTrack CommandName = "set_audio_track" CommandSetSubtitleTrack CommandName = "set_subtitle_track" + // CommandPlanInvalidated tells a playing client that the plan it is running + // can no longer serve this source, so it must replan off that plan. It is + // the first server-initiated control push in the v3 protocol, and it is only + // sent to a session whose attempt negotiated FeaturePlanInvalidatedV3 and + // which has a live realtime connection; every other session is stopped + // instead. See CopySafetyNotifier. + CommandPlanInvalidated CommandName = "plan_invalidated" ) var supportedCommandNames = []CommandName{ @@ -73,6 +80,7 @@ var supportedCommandNames = []CommandName{ CommandPlayMedia, CommandSetAudioTrack, CommandSetSubtitleTrack, + CommandPlanInvalidated, } var supportedCommandNameSet = func() map[CommandName]struct{} { @@ -399,6 +407,36 @@ type CommandIssuedBy struct { Kind string `json:"kind"` } +// PlanInvalidationReason values a plan_invalidated command can carry. +const ( + // PlanInvalidatedVideoCopyUnsafe means the asynchronous H.264 copy-safety + // scan came back multi-PPS after the plan was already playing, so the video + // stream-copy route it named cannot serve this source. + PlanInvalidatedVideoCopyUnsafe = "video_copy_unsafe" +) + +// PlanInvalidatedPayload names the plan the server withdrew and why. +// +// PlanID is required: the client compares it against the plan it is running and +// does nothing when it has already replanned past it, so a late command can +// never evict a route the server never complained about. +type PlanInvalidatedPayload struct { + Reason string `json:"reason"` + PlanID string `json:"plan_id"` +} + +// NewPlanInvalidatedCommand builds a validated plan_invalidated command. +func NewPlanInvalidatedCommand(sessionID, commandID, planID, reason string) (CommandEnvelope, error) { + if planID == "" || reason == "" { + return CommandEnvelope{}, ErrInvalidRealtimePayload + } + payload, err := json.Marshal(PlanInvalidatedPayload{Reason: reason, PlanID: planID}) + if err != nil { + return CommandEnvelope{}, err + } + return NewCommandEnvelope(sessionID, commandID, CommandPlanInvalidated, payload) +} + // NewCommandEnvelope creates a validated command envelope. func NewCommandEnvelope(sessionID, commandID string, name CommandName, payload json.RawMessage) (CommandEnvelope, error) { normalizedPayload, err := normalizeJSONPayload(payload) diff --git a/internal/playback/realtime_test.go b/internal/playback/realtime_test.go index f1f28b4b1..7038cf7f5 100644 --- a/internal/playback/realtime_test.go +++ b/internal/playback/realtime_test.go @@ -85,3 +85,45 @@ func TestParseCommandEnvelopeStillWorks(t *testing.T) { t.Fatalf("command.Name = %q, want %q", command.Name, CommandPause) } } + +func TestNewPlanInvalidatedCommand(t *testing.T) { + command, err := NewPlanInvalidatedCommand("session-1", "cmd-1", "plan-1", PlanInvalidatedVideoCopyUnsafe) + if err != nil { + t.Fatalf("NewPlanInvalidatedCommand() error = %v", err) + } + if command.Type != RealtimeMessageTypeCommand || command.Name != CommandPlanInvalidated { + t.Fatalf("command = %#v, want a plan_invalidated command envelope", command) + } + var payload PlanInvalidatedPayload + if err := json.Unmarshal(command.Payload, &payload); err != nil { + t.Fatalf("json.Unmarshal(payload): %v", err) + } + if payload.PlanID != "plan-1" || payload.Reason != PlanInvalidatedVideoCopyUnsafe { + t.Fatalf("payload = %#v, want the invalidated plan and reason", payload) + } +} + +// The plan id is what lets a client ignore a command for a plan it has already +// replanned past, so an envelope without one must never be built. +func TestNewPlanInvalidatedCommandRequiresPlanAndReason(t *testing.T) { + if _, err := NewPlanInvalidatedCommand("session-1", "cmd-1", "", PlanInvalidatedVideoCopyUnsafe); err == nil { + t.Fatal("NewPlanInvalidatedCommand() with no plan id = nil error, want a rejection") + } + if _, err := NewPlanInvalidatedCommand("session-1", "cmd-1", "plan-1", ""); err == nil { + t.Fatal("NewPlanInvalidatedCommand() with no reason = nil error, want a rejection") + } +} + +// A client advertising the command in its hello must validate: the closed +// command enum is the negotiation surface for the realtime channel. +func TestHelloAcceptsPlanInvalidatedCapability(t *testing.T) { + hello := HelloEnvelope{ + Type: RealtimeMessageTypeHello, + SessionID: "session-1", + Client: HelloClientInfo{Name: "silo-web", Version: "1.0.0"}, + Capabilities: HelloCapabilities{Commands: []CommandName{CommandPause, CommandPlanInvalidated}}, + } + if err := hello.Validate(); err != nil { + t.Fatalf("hello.Validate() = %v, want the plan_invalidated capability accepted", err) + } +} diff --git a/internal/playback/resolver.go b/internal/playback/resolver.go index 688115f92..c704cdbc0 100644 --- a/internal/playback/resolver.go +++ b/internal/playback/resolver.go @@ -246,10 +246,21 @@ func containsStr(slice []string, s string) bool { // bitstream scan (H.264 sources that redefine a pic_parameter_set_id in-band // with conflicting content). Scan failures also disable copy for the current // decision while remaining eligible for retry on a later request. +// +// The track flags are runtime-only: they are stamped by the probe ensurer, +// which only the playback start path and the watch surfaces run. The verdict +// persisted on the media_files row carries the same answer and is present on +// every repository read, so it is honored directly — a replan, a +// Jellyfin-protocol route decision, and a fresh start must all reach the same +// conclusion about the same file. func videoCopyUnsafeFile(file *models.MediaFile) bool { if file == nil || len(file.VideoTracks) == 0 { return false } track := file.VideoTracks[0] - return track.VideoCopyUnsafe || (track.MultiplePPS != nil && *track.MultiplePPS) + if track.VideoCopyUnsafe || (track.MultiplePPS != nil && *track.MultiplePPS) { + return true + } + multi, known := file.PersistedVideoCopyVerdict() + return known && multi } diff --git a/internal/playback/testdata/protocol_v3/capability_response.json b/internal/playback/testdata/protocol_v3/capability_response.json index 230cd0820..58fa7acee 100644 --- a/internal/playback/testdata/protocol_v3/capability_response.json +++ b/internal/playback/testdata/protocol_v3/capability_response.json @@ -15,6 +15,7 @@ "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_invalidated_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index e3b90a10f..d0b9b3943 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -5411,6 +5411,7 @@ "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_invalidated_v1", "plan_source_duration_v1" ], "outcome": "adaptation_unavailable", diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json index b194163f6..3e57dcb70 100644 --- a/internal/playback/testdata/protocol_v3/decision_response.json +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -12,6 +12,7 @@ "header_authenticated_media_v1", "authorized_media_origins_v1", "software_video_decode_v1", + "plan_invalidated_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 10c72c152..04576f229 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -1182,12 +1182,13 @@ func (r *FileRepository) SetChapterThumbnailFailure( // UpdateMultiplePPS records the H.264 multi-PPS copy-safety verdict together // with the size and mtime it was computed from, so a later read can tell -// whether the file has been rewritten since. +// whether the file has been rewritten since. A nil scanMtime records a verdict +// for a row that has no file mtime; reading it back validates on size alone. // // It deliberately does not go through Upsert: that path also clears // match_suppressed_at and missing_since, which a copy-safety scan has no // business touching. -func (r *FileRepository) UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error { +func (r *FileRepository) UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime *time.Time) error { tag, err := r.pool.Exec(ctx, ` UPDATE media_files SET multiple_pps = $2, diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index 6b11ec79b..4ada4adea 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -2,6 +2,7 @@ package scanner import ( "context" + "errors" "log/slog" "strconv" "strings" @@ -88,8 +89,13 @@ func videoTracksMissingColorRange(tracks []models.VideoTrack) bool { // copySafetyWriter persists a multi-PPS verdict. *FileRepository satisfies it; // the indirection keeps the ensurer testable without a database. +// +// scanMtime is nil for a row that carries no file mtime: such a verdict is +// still recorded, and validated on size alone when it is read back. Refusing to +// write it would leave those rows permanently unverdicted, so every replica +// would rescan and re-invalidate the same sessions forever. type copySafetyWriter interface { - UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error + UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime *time.Time) error } // PlaybackProbeEnsurer repairs missing playback-critical probe metadata on @@ -168,6 +174,78 @@ func (e *PlaybackProbeEnsurer) EnsureProbeOnly(ctx context.Context, file *models return e.ensureProbeRepair(ctx, file) } +// EnsureCopySafetyCached repairs playback-critical probe metadata and stamps +// the copy-safety verdict only when it is already known — from the process +// cache or from the verdict persisted on the media_files row. It never execs +// ffmpeg, so it never blocks a play or a watch page on a bitstream scan. +// +// An unknown verdict is left unknown: VideoTrack.MultiplePPS stays nil and +// VideoCopyUnsafe stays false, which the planner reads as "stream copy is +// allowed". That is the optimistic half of the race — the caller is expected to +// kick off ScanCopySafety asynchronously and switch live sessions off the copy +// route if the scan comes back multi-PPS. +func (e *PlaybackProbeEnsurer) EnsureCopySafetyCached(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { + current, err := e.ensureProbeRepair(ctx, file) + if err != nil || current == nil || e == nil { + return current, err + } + if !needsCopySafetyProbe(current) { + return current, nil + } + if multi, ok := e.knownCopySafetyVerdict(current); ok { + return fileWithMultiplePPS(current, multi), nil + } + return current, nil +} + +// NeedsCopySafetyScan reports whether an asynchronous ScanCopySafety would do +// real work for this file: an H.264 video whose verdict is neither cached nor +// persisted, on a server that has an ffmpeg to scan with. +func (e *PlaybackProbeEnsurer) NeedsCopySafetyScan(file *models.MediaFile) bool { + if e == nil || strings.TrimSpace(e.ffmpegPath) == "" || !needsCopySafetyProbe(file) { + return false + } + _, known := e.knownCopySafetyVerdict(file) + return !known +} + +// ScanCopySafety runs the multi-PPS bitstream scan for a file whose verdict is +// unknown, persisting and memoizing the result. Concurrent callers for one file +// share a single scan, so a start, a replan and a watch-page load racing on the +// same file spawn one ffmpeg between them. +func (e *PlaybackProbeEnsurer) ScanCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { + if e == nil || file == nil { + return false, nil + } + if strings.TrimSpace(e.ffmpegPath) == "" { + return false, errCopySafetyScanUnavailable + } + return e.scanAndPersistCopySafety(ctx, file) +} + +var errCopySafetyScanUnavailable = errors.New("ffmpeg path not configured") + +// knownCopySafetyVerdict answers the copy-safety question from memory or from +// the persisted row, never from ffmpeg. A persisted verdict is self-validating: +// it is only honored while the recorded size and mtime still describe the file, +// so a rewrite in place falls through to a rescan without any writer having to +// clear it. Promoting it into the process cache keeps later calls off the row. +func (e *PlaybackProbeEnsurer) knownCopySafetyVerdict(file *models.MediaFile) (bool, bool) { + if e == nil || file == nil { + return false, false + } + if cached, ok := e.copySafety.Load(file.ID); ok { + if result, ok := cached.(copySafetyResult); ok && result.matches(file) { + return result.multi, true + } + } + if multi, ok := persistedCopySafetyVerdict(file); ok { + e.storeCopySafety(file, multi) + return multi, true + } + return false, false +} + func (e *PlaybackProbeEnsurer) ensureProbeRepair(ctx context.Context, file *models.MediaFile) (*models.MediaFile, error) { if file == nil || e == nil || e.fileRepo == nil { return file, nil @@ -210,17 +288,7 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model return file, nil } - if cached, ok := e.copySafety.Load(file.ID); ok { - if result, ok := cached.(copySafetyResult); ok && result.matches(file) { - return fileWithMultiplePPS(file, result.multi), nil - } - } - - // A persisted verdict is self-validating: it is only honored while the - // recorded size and mtime still describe the file, so a rewrite in place - // falls through to a rescan without any writer having to clear it. - if multi, ok := persistedCopySafetyVerdict(file); ok { - e.storeCopySafety(file, multi) + if multi, ok := e.knownCopySafetyVerdict(file); ok { return fileWithMultiplePPS(file, multi), nil } @@ -263,8 +331,8 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil return false, err } - if e.copySafetyRepo != nil && fileModifiedAt != nil { - if writeErr := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, multi, fileSize, *fileModifiedAt); writeErr != nil { + if e.copySafetyRepo != nil { + if writeErr := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, multi, fileSize, fileModifiedAt); writeErr != nil { slog.WarnContext(ctx, "persisting video copy-safety verdict failed", "component", "scanner", "file_id", fileID, @@ -292,20 +360,11 @@ func (e *PlaybackProbeEnsurer) storeCopySafety(file *models.MediaFile, multi boo } // persistedCopySafetyVerdict returns the multi-PPS verdict stored on the -// media_files row, and whether it is still valid for the file as it stands. A -// verdict is valid only when it was computed from the same size and mtime the -// row now reports. +// media_files row, and whether it is still valid for the file as it stands. +// The rule lives on the model because playback reads the same columns from +// files this package never touches. func persistedCopySafetyVerdict(file *models.MediaFile) (bool, bool) { - if file == nil || file.MultiplePPS == nil || file.MultiplePPSScanSize == nil || file.MultiplePPSScanMtime == nil { - return false, false - } - if *file.MultiplePPSScanSize != file.FileSize { - return false, false - } - if file.FileModifiedAt == nil || !sameFileModifiedAt(file.MultiplePPSScanMtime, *file.FileModifiedAt) { - return false, false - } - return *file.MultiplePPS, true + return file.PersistedVideoCopyVerdict() } // fileWithMultiplePPS returns a shallow copy of file with the (runtime-only) @@ -329,17 +388,7 @@ func fileWithCopySafety(file *models.MediaFile, multiplePPS *bool, copyUnsafe bo // needsCopySafetyProbe reports whether the file is an H.264 video whose // multi-PPS copy-safety flag has not yet been computed. func needsCopySafetyProbe(file *models.MediaFile) bool { - if file == nil || len(file.VideoTracks) == 0 { - return false - } - if file.VideoTracks[0].MultiplePPS != nil { - return false - } - codec := strings.ToLower(strings.TrimSpace(file.VideoTracks[0].Codec)) - if codec == "" { - codec = strings.ToLower(strings.TrimSpace(file.CodecVideo)) - } - return codec == "h264" || codec == "avc" || codec == "avc1" + return file.VideoCopySafetyUnknown() } // reprobeMayScanPackets reports whether reprobing this file is likely to hit diff --git a/internal/scanner/probe_repair_copy_safety_cached_test.go b/internal/scanner/probe_repair_copy_safety_cached_test.go new file mode 100644 index 000000000..11ce3c99d --- /dev/null +++ b/internal/scanner/probe_repair_copy_safety_cached_test.go @@ -0,0 +1,212 @@ +package scanner + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// EnsureCopySafetyCached is the optimistic half of the remux race: a play must +// never wait on the bitstream scan, so an unknown verdict is left unknown — +// which the planner reads as "stream copy is allowed" — and no ffmpeg runs. +func TestEnsureCopySafetyCachedNeverScans(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) + if err != nil { + t.Fatalf("EnsureCopySafetyCached() error = %v", err) + } + if runs() != 0 { + t.Fatalf("ffmpeg ran %d times, want 0 — the cached ensure must never exec", runs()) + } + track := got.VideoTracks[0] + if track.MultiplePPS != nil { + t.Fatalf("MultiplePPS = %v, want nil so the planner may still remux optimistically", *track.MultiplePPS) + } + if track.VideoCopyUnsafe { + t.Fatal("VideoCopyUnsafe = true, want false: an unresolved verdict must not disqualify the copy route") + } + if writes := writer.recorded(); len(writes) != 0 { + t.Fatalf("cached ensure recorded %d verdicts, want 0", len(writes)) + } +} + +// A verdict that is already known is stamped without a scan, so a known-unsafe +// file never gets an optimistic remux in the first place. +func TestEnsureCopySafetyCachedStampsKnownVerdicts(t *testing.T) { + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + + t.Run("persisted", func(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + + file := copySafetyTestFile(mtime) + verdict := true + scanSize := file.FileSize + scanMtime := mtime + file.MultiplePPS = &verdict + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &scanMtime + + got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) + if err != nil { + t.Fatalf("EnsureCopySafetyCached() error = %v", err) + } + if runs() != 0 { + t.Fatalf("ffmpeg ran %d times, want 0", runs()) + } + track := got.VideoTracks[0] + if track.MultiplePPS == nil || !*track.MultiplePPS || !track.VideoCopyUnsafe { + t.Fatalf("track = %+v, want the persisted multi-PPS verdict stamped copy-unsafe", track) + } + }) + + t.Run("memoized", func(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + + file := copySafetyTestFile(mtime) + ensurer.storeCopySafety(file, false) + + got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) + if err != nil { + t.Fatalf("EnsureCopySafetyCached() error = %v", err) + } + if runs() != 0 { + t.Fatalf("ffmpeg ran %d times, want 0", runs()) + } + track := got.VideoTracks[0] + if track.MultiplePPS == nil || *track.MultiplePPS || track.VideoCopyUnsafe { + t.Fatalf("track = %+v, want the memoized copy-safe verdict", track) + } + }) +} + +func TestNeedsCopySafetyScan(t *testing.T) { + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + ffmpegPath, _ := fakeFFmpeg(t, "", 0) + + t.Run("unknown h264 needs a scan", func(t *testing.T) { + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + if !ensurer.NeedsCopySafetyScan(copySafetyTestFile(mtime)) { + t.Fatal("NeedsCopySafetyScan() = false, want true for an unresolved H.264 file") + } + }) + + t.Run("known verdict needs none", func(t *testing.T) { + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + file := copySafetyTestFile(mtime) + ensurer.storeCopySafety(file, true) + if ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = true, want false once the verdict is known") + } + }) + + t.Run("non-h264 needs none", func(t *testing.T) { + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + file := copySafetyTestFile(mtime) + file.CodecVideo = "hevc" + file.VideoTracks[0].Codec = "hevc" + if ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = true, want false for a non-H.264 source") + } + }) + + t.Run("without ffmpeg needs none", func(t *testing.T) { + ensurer := &PlaybackProbeEnsurer{} + if ensurer.NeedsCopySafetyScan(copySafetyTestFile(mtime)) { + t.Fatal("NeedsCopySafetyScan() = true, want false without an ffmpeg to scan with") + } + }) +} + +// ScanCopySafety is the asynchronous half: it runs the scan, persists the +// verdict, and memoizes it so the next plan for the file excludes the copy +// route without touching the disk again. +func TestScanCopySafetyPersistsAndMemoizes(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + file := copySafetyTestFile(mtime) + + multi, err := ensurer.ScanCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ScanCopySafety() error = %v", err) + } + if !multi { + t.Fatal("ScanCopySafety() = false, want true for the conflicting-PPS stream") + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1", runs()) + } + want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234, scanMtime: mtime, scanMtimeSet: true} + if writes := writer.recorded(); len(writes) != 1 || writes[0] != want { + t.Fatalf("UpdateMultiplePPS writes = %+v, want exactly %+v", writes, want) + } + if ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = true after a completed scan, want false") + } + + // The memoized verdict is what a later start reads, without re-execing. + got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) + if err != nil { + t.Fatalf("EnsureCopySafetyCached() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times after the cached ensure, want 1", runs()) + } + if track := got.VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS || !track.VideoCopyUnsafe { + t.Fatalf("track = %+v, want the scanned multi-PPS verdict", track) + } +} + +// An inconclusive scan is not evidence: nothing is persisted, nothing is +// memoized, and the caller learns the scan failed rather than being handed a +// fabricated copy-unsafe verdict. +func TestScanCopySafetyErrorRecordsNothing(t *testing.T) { + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ + ffmpegPath: filepath.Join(t.TempDir(), "missing-ffmpeg"), + copySafetyRepo: writer, + } + + file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) + + multi, err := ensurer.ScanCopySafety(context.Background(), file) + if err == nil { + t.Fatal("ScanCopySafety() error = nil, want the scan failure surfaced to the caller") + } + if multi { + t.Fatal("ScanCopySafety() = true on error, want false") + } + if writes := writer.recorded(); len(writes) != 0 { + t.Fatalf("failed scan recorded %d verdicts, want 0", len(writes)) + } + if !ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = false after a failed scan, want true so a later request retries") + } + + got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) + if err != nil { + t.Fatalf("EnsureCopySafetyCached() error = %v", err) + } + if track := got.VideoTracks[0]; track.MultiplePPS != nil || track.VideoCopyUnsafe { + t.Fatalf("track = %+v, want the verdict left unknown after a failed scan", track) + } +} + +func TestVideoCopySafetyUnknownIgnoresAudioOnlyFiles(t *testing.T) { + file := &models.MediaFile{ID: 7, CodecAudio: "flac"} + if file.VideoCopySafetyUnknown() { + t.Fatal("VideoCopySafetyUnknown() = true for an audio-only file, want false") + } +} diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index 07a3064b7..b8e86ff2b 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -56,7 +56,10 @@ type recordedPPSWrite struct { fileID int multiplePPS bool scanSize int64 - scanMtime time.Time + // scanMtime is flattened to a comparable pair so a write for a row with no + // mtime is distinguishable from one that carries the zero time. + scanMtime time.Time + scanMtimeSet bool } type fakeCopySafetyWriter struct { @@ -65,15 +68,19 @@ type fakeCopySafetyWriter struct { err error } -func (w *fakeCopySafetyWriter) UpdateMultiplePPS(_ context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime time.Time) error { +func (w *fakeCopySafetyWriter) UpdateMultiplePPS(_ context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime *time.Time) error { w.mu.Lock() defer w.mu.Unlock() - w.writes = append(w.writes, recordedPPSWrite{ + write := recordedPPSWrite{ fileID: fileID, multiplePPS: multiplePPS, scanSize: scanSize, - scanMtime: scanMtime, - }) + } + if scanMtime != nil { + write.scanMtime = *scanMtime + write.scanMtimeSet = true + } + w.writes = append(w.writes, write) return w.err } @@ -186,7 +193,7 @@ func TestEnsureCopySafetyPersistsScanResult(t *testing.T) { if len(writes) != 1 { t.Fatalf("UpdateMultiplePPS called %d times, want 1", len(writes)) } - want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234, scanMtime: mtime} + want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234, scanMtime: mtime, scanMtimeSet: true} if writes[0] != want { t.Fatalf("UpdateMultiplePPS(%+v), want %+v", writes[0], want) } @@ -211,6 +218,51 @@ func TestEnsureCopySafetyScanSurvivesPersistFailure(t *testing.T) { } } +// Rows predating the file_modified_at column carry no mtime. Their verdict is +// still persisted and still honored on read — refusing to write it would leave +// them permanently unverdicted, so every replica would rescan the same file and +// tear down the same playback again. +func TestEnsureCopySafetyPersistsVerdictForRowWithoutMtime(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + file := copySafetyTestFile(time.Now()) + file.FileModifiedAt = nil + + if _, err := ensurer.ensureCopySafety(context.Background(), file); err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1", runs()) + } + writes := writer.recorded() + want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234} + if len(writes) != 1 || writes[0] != want { + t.Fatalf("UpdateMultiplePPS writes = %+v, want exactly [%+v]", writes, want) + } + + // A fresh process reading that row back must trust the verdict on size + // alone rather than rescanning. + reread := copySafetyTestFile(time.Now()) + reread.FileModifiedAt = nil + verdict := true + scanSize := reread.FileSize + reread.MultiplePPS = &verdict + reread.MultiplePPSScanSize = &scanSize + cold := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} + if cold.NeedsCopySafetyScan(reread) { + t.Fatal("NeedsCopySafetyScan() = true, want the persisted mtime-less verdict honored") + } + got, err := cold.ensureCopySafety(context.Background(), reread) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if track := got.VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS || !track.VideoCopyUnsafe { + t.Fatalf("track = %+v, want the persisted multi-PPS verdict stamped", track) + } +} + func TestEnsureCopySafetyWithoutRepoDoesNotPanic(t *testing.T) { ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 7a2d57da9..8ed0a328e 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -3184,7 +3184,7 @@ func sameFileModifiedAt(existing *time.Time, current time.Time) bool { } func normalizeFileModifiedAt(ts time.Time) time.Time { - return ts.UTC().Truncate(time.Microsecond) + return models.NormalizeFileModifiedAt(ts) } func needsCriticalProbeRepairScanState(file *scanStateFile) bool { diff --git a/web/src/player/components/VideoPlayer.test.tsx b/web/src/player/components/VideoPlayer.test.tsx index bde93ce00..76672a505 100644 --- a/web/src/player/components/VideoPlayer.test.tsx +++ b/web/src/player/components/VideoPlayer.test.tsx @@ -4,13 +4,19 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PlayerConfigProvider, type PlayerConfig } from "../context/PlayerConfigContext"; import { fixturePlanV3 } from "../protocol-v3.fixtures"; -import type { PlaybackRealtimeEventEnvelope } from "../realtime-protocol"; +import type { + PlaybackRealtimeCommandEnvelope, + PlaybackRealtimeEventEnvelope, +} from "../realtime-protocol"; import type { PlayerSubtitleInfo } from "../types"; import { HLS_STARTUP_TIMEOUT_MS } from "../utils/hlsStartupGuard"; import { VideoPlayer } from "./VideoPlayer"; const realtimeOptions = vi.hoisted(() => ({ - current: null as null | { onEvent?: (event: PlaybackRealtimeEventEnvelope) => void }, + current: null as null | { + onEvent?: (event: PlaybackRealtimeEventEnvelope) => void; + onCommand: (command: PlaybackRealtimeCommandEnvelope) => Promise | void; + }, })); const controls = vi.hoisted(() => ({ current: null as null | { @@ -121,6 +127,22 @@ function renderPlayer(overrides: Partial[0]> = {} }; } +function planInvalidatedCommand( + payload: Record = { + reason: "video_copy_unsafe", + plan_id: directPlan.plan_id, + }, +): PlaybackRealtimeCommandEnvelope { + return { + type: "command", + command_id: "cmd-invalidate-1", + session_id: "session-1", + name: "plan_invalidated", + deadline_ms: 8_000, + payload, + }; +} + function setMediaError(video: HTMLVideoElement, message: string) { Object.defineProperty(video, "error", { configurable: true, @@ -234,6 +256,45 @@ describe("VideoPlayer plan failure recovery", () => { expect(onPlanFailure).toHaveBeenCalledTimes(2); }); + it("replans off a plan the server invalidated", async () => { + const onPlanInvalidated = vi.fn().mockResolvedValue(true); + renderPlayer({ onPlanInvalidated }); + const onCommand = realtimeOptions.current?.onCommand; + if (!onCommand) throw new Error("expected the realtime command handler"); + + await act(async () => { + await onCommand(planInvalidatedCommand()); + }); + + expect(onPlanInvalidated).toHaveBeenCalledWith(directPlan.plan_id, "video_copy_unsafe", 0); + }); + + // A rejected result is the server's cue to stop the session, which is what + // lets the client's own recovery mint a fresh attempt against the persisted + // verdict. Swallowing the failure here would leave the copy route playing. + it("rejects the invalidation command when no replacement plan is adopted", async () => { + const onPlanInvalidated = vi.fn().mockResolvedValue(false); + renderPlayer({ onPlanInvalidated }); + const onCommand = realtimeOptions.current?.onCommand; + if (!onCommand) throw new Error("expected the realtime command handler"); + + await expect(onCommand(planInvalidatedCommand())).rejects.toThrow( + "plan_invalidation_replan_failed", + ); + }); + + it("rejects an invalidation command that names no plan", async () => { + const onPlanInvalidated = vi.fn().mockResolvedValue(true); + renderPlayer({ onPlanInvalidated }); + const onCommand = realtimeOptions.current?.onCommand; + if (!onCommand) throw new Error("expected the realtime command handler"); + + await expect( + onCommand(planInvalidatedCommand({ reason: "video_copy_unsafe" })), + ).rejects.toThrow("invalid_plan_invalidated_payload"); + expect(onPlanInvalidated).not.toHaveBeenCalled(); + }); + it("does not retry an auto-selected subtitle after its replan is refused", async () => { const onSubtitleTrackChange = vi.fn(); const sidecarTrack: PlayerSubtitleInfo = { diff --git a/web/src/player/components/VideoPlayer.tsx b/web/src/player/components/VideoPlayer.tsx index 3e2c24146..4f82a8a1f 100644 --- a/web/src/player/components/VideoPlayer.tsx +++ b/web/src/player/components/VideoPlayer.tsx @@ -27,6 +27,7 @@ import { usePlayerConfig } from "../context/PlayerConfigContext"; import { qualityOptionsFromPlanV3 } from "../playback-info"; import { preconnectToStreamOrigin } from "../stream-url"; import { WatchTogetherPanel } from "./WatchTogetherPanel"; +import { readPlanInvalidatedPayload, VIDEO_PLAYBACK_COMMANDS } from "../realtime-protocol"; import type { PlaybackRealtimeCommandEnvelope, PlaybackRealtimeEventEnvelope, @@ -109,6 +110,12 @@ interface VideoPlayerProps { onSubtitleTrackChange?: (combinedIndex: number | null, currentPosition: number) => void; /** `failure_recovery` replan after the client could not play the plan. */ onPlanFailure?: (failure: FailureV3, currentPosition: number) => void; + /** + * Replan for a plan the server invalidated over the realtime + * `plan_invalidated` command. Resolving false rejects the command, which is + * what tells the server to stop the session instead. + */ + onPlanInvalidated?: (planId: string, reason: string, currentPosition: number) => Promise; /** `seek_reanchor` replan when a seek target falls outside the seekable window. */ onReanchorSeek?: (positionSeconds: number) => void; preferredSubtitleLanguage?: string | null; @@ -216,6 +223,7 @@ export function VideoPlayer({ onQualitySelect, onSubtitleTrackChange, onPlanFailure, + onPlanInvalidated, onReanchorSeek, preferredSubtitleLanguage, preferredSubtitleTrackSignature, @@ -2416,6 +2424,28 @@ export function VideoPlayer({ tone: "warning", }); return; + case "plan_invalidated": { + // The server decided the route it planned cannot serve this source + // after all. Ack (already sent by the transport), replan off it, and + // report the outcome: a rejection is the server's cue to stop the + // session so the client's own recovery can mint a fresh attempt. + const invalidated = readPlanInvalidatedPayload(command.payload); + if (!invalidated) { + throw new Error("invalid_plan_invalidated_payload"); + } + if (!onPlanInvalidated) { + throw new Error("plan_invalidation_unsupported"); + } + const replaced = await onPlanInvalidated( + invalidated.plan_id, + invalidated.reason, + currentTimeRef.current, + ); + if (!replaced) { + throw new Error("plan_invalidation_replan_failed"); + } + return; + } case "stop": case "terminate": if (command.payload) { @@ -2436,13 +2466,14 @@ export function VideoPlayer({ throw new Error("unsupported"); } }, - [handleExit, performPlayerSeek], + [handleExit, onPlanInvalidated, performPlayerSeek], ); const realtime = usePlaybackRealtime({ sessionId, onCommand: executeRealtimeCommand, onEvent: handleRealtimeEvent, + supportedCommands: VIDEO_PLAYBACK_COMMANDS, }); useEffect(() => { diff --git a/web/src/player/components/WatchPage.test.ts b/web/src/player/components/WatchPage.test.ts index ea2cbb49c..c8faf6b72 100644 --- a/web/src/player/components/WatchPage.test.ts +++ b/web/src/player/components/WatchPage.test.ts @@ -87,6 +87,7 @@ function playbackSession( changeSubtitleTrack: vi.fn(), changeQuality: vi.fn(), recoverFromFailure: vi.fn(), + invalidatePlan: vi.fn().mockResolvedValue(true), reanchorSeek: vi.fn(), refreshSubtitles: vi.fn(), applySubtitleTrack: vi.fn(), diff --git a/web/src/player/components/WatchPage.tsx b/web/src/player/components/WatchPage.tsx index f3e1ce196..45ceddfa1 100644 --- a/web/src/player/components/WatchPage.tsx +++ b/web/src/player/components/WatchPage.tsx @@ -450,6 +450,7 @@ export function WatchPage({ onQualitySelect={session.changeQuality} onSubtitleTrackChange={session.changeSubtitleTrack} onPlanFailure={session.recoverFromFailure} + onPlanInvalidated={session.invalidatePlan} onReanchorSeek={session.reanchorSeek} onApplySubtitleTrack={session.applySubtitleTrack} preferredSubtitleLanguage={preferredSubtitleLanguage} diff --git a/web/src/player/hooks/usePlaybackRealtime.ts b/web/src/player/hooks/usePlaybackRealtime.ts index e5e8aeaa1..35f88b18d 100644 --- a/web/src/player/hooks/usePlaybackRealtime.ts +++ b/web/src/player/hooks/usePlaybackRealtime.ts @@ -5,6 +5,7 @@ import { buildPlaybackRealtimeHello, buildPlaybackRealtimeResult, parsePlaybackRealtimeMessage, + type PlaybackCommandName, type PlaybackRealtimeCommandEnvelope, type PlaybackRealtimeEventEnvelope, } from "../realtime-protocol"; @@ -15,6 +16,12 @@ interface UsePlaybackRealtimeOptions { sessionId: string | null; onCommand: (command: PlaybackRealtimeCommandEnvelope) => Promise | void; onEvent?: (event: PlaybackRealtimeEventEnvelope) => void; + /** + * The commands this surface can execute, announced in the hello. Defaults to + * the shared set; a surface that handles more names them so it does not + * announce a command it would only reject. + */ + supportedCommands?: PlaybackCommandName[]; } interface UsePlaybackRealtimeResult { @@ -39,17 +46,23 @@ export function usePlaybackRealtime({ sessionId, onCommand, onEvent, + supportedCommands, }: UsePlaybackRealtimeOptions): UsePlaybackRealtimeResult { const config = usePlayerConfig(); const [connectionState, setConnectionState] = useState("disconnected"); const onCommandRef = useRef(onCommand); const onEventRef = useRef(onEvent); + const supportedCommandsRef = useRef(supportedCommands); const seenCommandsRef = useRef>(new Set()); useEffect(() => { onCommandRef.current = onCommand; }, [onCommand]); + useEffect(() => { + supportedCommandsRef.current = supportedCommands; + }, [supportedCommands]); + useEffect(() => { onEventRef.current = onEvent; }, [onEvent]); @@ -94,7 +107,9 @@ export function usePlaybackRealtime({ attempt = 0; setConnectionState("connected"); seenCommandsRef.current.clear(); - socket.send(JSON.stringify(buildPlaybackRealtimeHello(sessionId))); + socket.send( + JSON.stringify(buildPlaybackRealtimeHello(sessionId, supportedCommandsRef.current)), + ); }); socket.addEventListener("message", (event) => { diff --git a/web/src/player/hooks/usePlaybackSession.test.ts b/web/src/player/hooks/usePlaybackSession.test.ts index 0c49dd86e..0bff22276 100644 --- a/web/src/player/hooks/usePlaybackSession.test.ts +++ b/web/src/player/hooks/usePlaybackSession.test.ts @@ -12,6 +12,7 @@ import { buildReplanRequestV3, buildStartRequestV3, routeEventPlanIdentityV3, + VIDEO_CLIENT_FEATURES_V3, } from "../playback-session-wire-v3"; import { usePlaybackSession } from "./usePlaybackSession"; @@ -67,6 +68,16 @@ const replanBase = { }; describe("buildStartRequestV3", () => { + // Feature tokens are promises the server enforces, so a surface advertises + // only what it implements: the base set alone unless the caller names more. + it("advertises only the surface's own features", () => { + expect( + buildStartRequestV3({ ...startBase, extraClientFeatures: VIDEO_CLIENT_FEATURES_V3 }) + .client_features, + ).toEqual(["playback_plan_v3", "plan_invalidated_v1"]); + expect(buildStartRequestV3(startBase).client_features).toEqual(["playback_plan_v3"]); + }); + it("declares the protocol version and the plan feature", () => { expect(buildStartRequestV3(startBase)).toMatchObject({ protocol_version: 3, @@ -145,6 +156,19 @@ describe("buildReplanRequestV3", () => { }); }); + // A replan that sends `client_features` replaces the negotiated list, so a + // replan which advertised less than the start did would silently withdraw the + // promise the server gates the invalidation command on. + it("re-advertises the same features a start negotiated", () => { + expect( + buildReplanRequestV3({ + ...replanBase, + operation: "failure_recovery", + extraClientFeatures: VIDEO_CLIENT_FEATURES_V3, + }).client_features, + ).toEqual(["playback_plan_v3", "plan_invalidated_v1"]); + }); + it("names a new audio track by index alone", () => { // An empty id makes the server resolve the ordinal against the *effective* // file, which the client cannot name: it changes on a version fallback. @@ -1395,3 +1419,152 @@ describe("usePlaybackSession replans", () => { unmount(); }); }); + +describe("usePlaybackSession server-invalidated plans", () => { + function invalidationFetchMock(replanBodies: Array>, replan: unknown) { + return vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/playback/start")) { + return jsonResponse( + { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3(), + }, + { status: 201 }, + ); + } + if (url.endsWith("/playback/session-1/replan")) { + replanBodies.push(JSON.parse(String(init?.body)) as Record); + return jsonResponse(replan); + } + if (url.endsWith("/playback/route-events")) { + return new Response(null, { status: 202 }); + } + if (init?.method === "DELETE") { + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected request: ${url}`); + }); + } + + it("recovers off the invalidated plan and excludes its attempt key", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ + plan_id: "plan:2222222222222222", + plan_attempt_key: "v3:2222222222222222", + delivery: "server_transcode_hls", + }), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan( + "plan:0123456789abcdef", + "video_copy_unsafe", + 450, + ); + }); + + // The server only pushes the command to a session that promised to handle + // it, so the promise has to be on the wire for any of this to be reachable. + const startCall = vi + .mocked(fetch) + .mock.calls.find(([url]) => String(url).endsWith("/playback/start")); + const startBody = JSON.parse(String(startCall?.[1]?.body)) as { client_features: string[] }; + expect(startBody.client_features).toContain("plan_invalidated_v1"); + + expect(outcome).toBe(true); + expect(replanBodies).toHaveLength(1); + expect(replanBodies[0]).toMatchObject({ + operation: "failure_recovery", + failed_plan_id: "plan:0123456789abcdef", + position_seconds: 450, + // The invalidated route is excluded by key, so the replacement plan + // cannot be the same copy route the server just disqualified. + attempted_plan_keys: ["v3:0123456789abcdef"], + failure: { classification: "video_copy_unsafe" }, + }); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:2222222222222222")); + + unmount(); + }); + + it("does nothing for a plan the session already moved past", async () => { + const replanBodies: Array> = []; + vi.stubGlobal("fetch", invalidationFetchMock(replanBodies, {})); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan("plan:superseded", "video_copy_unsafe", 12); + }); + + // Reported as handled: the invalidated route is already gone, and replanning + // would evict a plan the server never complained about. + expect(outcome).toBe(true); + expect(replanBodies).toHaveLength(0); + + unmount(); + }); + + it("reports failure when the replan produces no replacement plan", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "adaptation_unavailable", + terminal: { + reason: "video_conversion_unsupported", + message: "No executor can transcode this source.", + retryable: false, + }, + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan( + "plan:0123456789abcdef", + "video_copy_unsafe", + 30, + ); + }); + + // The caller rejects the realtime command on false, which is what makes the + // server stop the session instead of leaving the copy route playing. + expect(outcome).toBe(false); + expect(replanBodies).toHaveLength(1); + + unmount(); + }); +}); diff --git a/web/src/player/hooks/usePlaybackSession.ts b/web/src/player/hooks/usePlaybackSession.ts index 2dc59e70c..aab0d1286 100644 --- a/web/src/player/hooks/usePlaybackSession.ts +++ b/web/src/player/hooks/usePlaybackSession.ts @@ -28,6 +28,7 @@ import { buildReplanRequestV3, buildStartRequestV3, routeEventPlanIdentityV3, + VIDEO_CLIENT_FEATURES_V3, type ReplanOptions, } from "../playback-session-wire-v3"; import type { @@ -87,6 +88,12 @@ export interface UsePlaybackSessionResult extends PlaybackSessionState { changeQuality: (label: string, currentPosition: number) => void; /** `failure_recovery` replan after the client could not play the plan. */ recoverFromFailure: (failure: FailureV3, currentPosition: number) => void; + /** + * `failure_recovery` replan for a plan the *server* invalidated over the + * realtime `plan_invalidated` command. Resolves to whether a replacement plan + * is now playing; the caller reports that back as the command's result. + */ + invalidatePlan: (planId: string, reason: string, currentPosition: number) => Promise; /** `seek_reanchor` replan when the target lies outside the seekable window. */ reanchorSeek: (positionSeconds: number) => void; /** Re-reads the subtitle inventory by replanning with the selection unchanged. */ @@ -416,6 +423,7 @@ export function usePlaybackSession( playbackAttemptId: string, ): Promise => { const body = buildStartRequestV3({ + extraClientFeatures: VIDEO_CLIENT_FEATURES_V3, fileId: targetFileId, profileId: config.getProfileId() ?? "", playbackAttemptId, @@ -793,6 +801,7 @@ export function usePlaybackSession( const body = buildReplanRequestV3({ ...options, + extraClientFeatures: VIDEO_CLIENT_FEATURES_V3, plan, playbackAttemptId, replanRequestId: randomUUID(), @@ -1008,6 +1017,35 @@ export function usePlaybackSession( [replan, reportEvent], ); + /** + * Replans off a plan the server invalidated mid-playback. + * + * This is an ordinary `failure_recovery`, deliberately: that operation is + * what folds the current plan's attempt key into `attempted_plan_keys`, so + * the route the server just disqualified is excluded from the replacement + * plan without the client reasoning about deliveries at all. The plan + * revision the adopted plan bumps rebuilds the transport and restores the + * position, exactly as it does after a client-detected failure. + */ + const invalidatePlan = useCallback( + async (planId: string, reason: string, currentPosition: number): Promise => { + const plan = planRef.current; + if (!plan) return false; + // The command names the plan the server invalidated. Once the client has + // moved past it there is nothing to recover from, and replanning anyway + // would evict a route the server never complained about. + if (plan.plan_id !== planId) return true; + const classification = reason.trim().slice(0, 64) || "plan_invalidated"; + reportEvent("plan_invalidated", { fallbackReason: classification }); + return replan({ + operation: "failure_recovery", + positionSeconds: currentPosition, + failure: { classification, message: "The server invalidated this plan." }, + }); + }, + [replan, reportEvent], + ); + const reanchorSeek = useCallback( (positionSeconds: number) => { playbackPositionRef.current = positionSeconds; @@ -1111,6 +1149,7 @@ export function usePlaybackSession( changeSubtitleTrack, changeQuality, recoverFromFailure, + invalidatePlan, reanchorSeek, refreshSubtitles, applySubtitleTrack, diff --git a/web/src/player/playback-session-wire-v3.ts b/web/src/player/playback-session-wire-v3.ts index a3b819dbd..25419a0d9 100644 --- a/web/src/player/playback-session-wire-v3.ts +++ b/web/src/player/playback-session-wire-v3.ts @@ -1,4 +1,5 @@ import { + FEATURE_PLAN_INVALIDATED_V3, FEATURE_PLAYBACK_PLAN_V3, PROTOCOL_V3, type ClientCodecCapabilitiesV3, @@ -16,6 +17,34 @@ import { /** Contract bound on `position_seconds` and `start_position`. */ const MAX_POSITION_SECONDS = 31_536_000; +/** + * What every v3 surface in this app can do. + * + * Anything beyond it is named by the caller through `extraClientFeatures`, + * because a feature token is a promise the server holds the client to and then + * enforces destructively — a session that advertises the plan-invalidation + * command and then rejects it gets stopped. The video player and the audiobook + * player share these builders but do not execute the same realtime commands, so + * neither may inherit the other's promises. + */ +const BASE_CLIENT_FEATURES_V3 = [FEATURE_PLAYBACK_PLAN_V3]; + +/** + * The features the video watch page adds: it executes the realtime + * `plan_invalidated` command (see `VideoPlayer`) and replans off the plan the + * server names. + */ +export const VIDEO_CLIENT_FEATURES_V3 = [FEATURE_PLAN_INVALIDATED_V3]; + +/** + * `client_features` is the contract's single advertisement location, and a + * replan that sends it replaces the negotiated list — so start and replan build + * it identically rather than letting a replan silently withdraw a promise. + */ +function clientFeaturesV3(extra: string[] | undefined): string[] { + return [...BASE_CLIENT_FEATURES_V3, ...(extra ?? [])]; +} + function clampPosition(seconds: number): number { if (!Number.isFinite(seconds) || seconds < 0) return 0; return Math.min(seconds, MAX_POSITION_SECONDS); @@ -31,6 +60,8 @@ export interface ReplanOptions { } export interface StartRequestInput { + /** Features this surface implements beyond {@link BASE_CLIENT_FEATURES_V3}. */ + extraClientFeatures?: string[]; fileId: number; profileId: string; playbackAttemptId: string; @@ -60,7 +91,7 @@ export interface StartRequestInput { export function buildStartRequestV3(input: StartRequestInput): StartRequestV3 { return { protocol_version: PROTOCOL_V3, - client_features: [FEATURE_PLAYBACK_PLAN_V3], + client_features: clientFeaturesV3(input.extraClientFeatures), file_id: input.fileId, profile_id: input.profileId, playback_attempt_id: input.playbackAttemptId, @@ -86,6 +117,8 @@ export function buildStartRequestV3(input: StartRequestInput): StartRequestV3 { } export interface ReplanRequestInput extends ReplanOptions { + /** Features this surface implements beyond {@link BASE_CLIENT_FEATURES_V3}. */ + extraClientFeatures?: string[]; plan: PlanV3; playbackAttemptId: string; replanRequestId: string; @@ -123,7 +156,7 @@ export function buildReplanRequestV3(input: ReplanRequestInput): ReplanRequestV3 return { protocol_version: PROTOCOL_V3, - client_features: [FEATURE_PLAYBACK_PLAN_V3], + client_features: clientFeaturesV3(input.extraClientFeatures), operation: input.operation, playback_attempt_id: input.playbackAttemptId, replan_request_id: input.replanRequestId, diff --git a/web/src/player/protocol-v3.ts b/web/src/player/protocol-v3.ts index 84d439690..d1b143f48 100644 --- a/web/src/player/protocol-v3.ts +++ b/web/src/player/protocol-v3.ts @@ -126,6 +126,17 @@ export const FEATURE_NEUTRAL_PLAYBACK_V3_CONTRACT = "neutral_playback_v3_contrac /** Server accepts output-capability refreshes without treating the route as failed. */ export const FEATURE_OUTPUT_CHANGE_V3 = "output_change_v1"; +/** + * The client can be told mid-session that the plan it is playing is no longer + * valid, over the realtime `plan_invalidated` command. + * + * Advertising it is a promise: the client acks the command and replans off the + * named plan. A server that does not see the token, or has no realtime + * connection to the session, stops the session instead — so a client that sends + * this must actually implement the command. + */ +export const FEATURE_PLAN_INVALIDATED_V3 = "plan_invalidated_v1"; + /** The `original` rung label, which always preserves the source. */ export const QUALITY_ORIGINAL_V3 = "original"; diff --git a/web/src/player/realtime-protocol.test.ts b/web/src/player/realtime-protocol.test.ts index 70408666c..51d9c5a38 100644 --- a/web/src/player/realtime-protocol.test.ts +++ b/web/src/player/realtime-protocol.test.ts @@ -6,7 +6,9 @@ import { buildPlaybackRealtimeResult, parsePlaybackRealtimeMessage, parsePlaybackRealtimeCommand, + readPlanInvalidatedPayload, SUPPORTED_PLAYBACK_COMMANDS, + VIDEO_PLAYBACK_COMMANDS, } from "./realtime-protocol"; describe("realtime protocol", () => { @@ -33,6 +35,49 @@ describe("realtime protocol", () => { }); }); + it("parses a plan invalidation command and its payload", () => { + const command = parsePlaybackRealtimeCommand( + JSON.stringify({ + type: "command", + command_id: "cmd-9", + session_id: "session-1", + name: "plan_invalidated", + deadline_ms: 8_000, + payload: { reason: "video_copy_unsafe", plan_id: "plan:0123456789abcdef" }, + }), + ); + + expect(command).toMatchObject({ + type: "command", + command_id: "cmd-9", + name: "plan_invalidated", + deadline_ms: 8_000, + }); + expect(readPlanInvalidatedPayload(command?.payload)).toEqual({ + reason: "video_copy_unsafe", + plan_id: "plan:0123456789abcdef", + }); + }); + + it("rejects a plan invalidation payload missing the invalidated plan", () => { + // Without the plan id the client cannot tell whether the plan it is playing + // is the one that was invalidated, so acting on it is never correct. + expect(readPlanInvalidatedPayload({ reason: "video_copy_unsafe" })).toBeNull(); + expect(readPlanInvalidatedPayload({ reason: "", plan_id: "plan:1" })).toBeNull(); + expect(readPlanInvalidatedPayload({ reason: "video_copy_unsafe", plan_id: 42 })).toBeNull(); + expect(readPlanInvalidatedPayload(undefined)).toBeNull(); + }); + + // The audiobook surface shares this module and cannot replan off an + // invalidated plan, so only the video command set announces it. + it("announces plan invalidation only for the video surface", () => { + expect(SUPPORTED_PLAYBACK_COMMANDS).not.toContain("plan_invalidated"); + expect(VIDEO_PLAYBACK_COMMANDS).toContain("plan_invalidated"); + expect( + buildPlaybackRealtimeHello("session-1", VIDEO_PLAYBACK_COMMANDS).capabilities.commands, + ).toContain("plan_invalidated"); + }); + it("rejects unknown commands", () => { const command = parsePlaybackRealtimeCommand( JSON.stringify({ diff --git a/web/src/player/realtime-protocol.ts b/web/src/player/realtime-protocol.ts index 734d1e7a2..a9fa444b3 100644 --- a/web/src/player/realtime-protocol.ts +++ b/web/src/player/realtime-protocol.ts @@ -15,7 +15,8 @@ export type PlaybackCommandName = | "server_shutting_down" | "play_media" | "set_audio_track" - | "set_subtitle_track"; + | "set_subtitle_track" + | "plan_invalidated"; export type PlaybackRealtimeAckStatus = "accepted"; export type PlaybackRealtimeResultStatus = "completed" | "rejected"; @@ -41,6 +42,18 @@ export interface PlaybackRealtimeCommandEnvelope { payload?: Record; } +/** + * Payload of the `plan_invalidated` command: the server decided, after the plan + * was already playing, that the route it names cannot serve this source. + * + * `plan_id` is the invalidated plan, not necessarily the one on screen — a + * client that has already replanned past it has nothing left to do. + */ +export interface PlaybackPlanInvalidatedPayload { + reason: string; + plan_id: string; +} + export interface PlaybackRealtimeHelloEnvelope { type: "hello"; session_id: string; @@ -206,8 +219,10 @@ export const ALL_PLAYBACK_COMMANDS: PlaybackCommandName[] = [ "play_media", "set_audio_track", "set_subtitle_track", + "plan_invalidated", ]; +/** The commands every realtime surface in this app executes. */ export const SUPPORTED_PLAYBACK_COMMANDS: PlaybackCommandName[] = [ "pause", "unpause", @@ -221,6 +236,16 @@ export const SUPPORTED_PLAYBACK_COMMANDS: PlaybackCommandName[] = [ "server_shutting_down", ]; +/** + * What the video player executes on top of the shared set. `plan_invalidated` + * needs a replan the audiobook surface has no route ladder for, so the hello is + * per-surface rather than one list both over-claim. + */ +export const VIDEO_PLAYBACK_COMMANDS: PlaybackCommandName[] = [ + ...SUPPORTED_PLAYBACK_COMMANDS, + "plan_invalidated", +]; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -229,6 +254,23 @@ function isCommandName(value: unknown): value is PlaybackCommandName { return typeof value === "string" && ALL_PLAYBACK_COMMANDS.includes(value as PlaybackCommandName); } +/** + * Reads a `plan_invalidated` payload, or null when it is not well formed. + * + * Both fields are required: without `plan_id` the client cannot tell whether + * the invalidated plan is still the one playing, and acting anyway would evict + * a route the server never complained about. + */ +export function readPlanInvalidatedPayload( + payload: Record | undefined, +): PlaybackPlanInvalidatedPayload | null { + if (!isRecord(payload)) return null; + const { reason, plan_id: planId } = payload; + if (typeof reason !== "string" || reason.trim() === "") return null; + if (typeof planId !== "string" || planId.trim() === "") return null; + return { reason, plan_id: planId }; +} + function isChapterThumbnailReadyPayload( value: unknown, ): value is PlaybackChapterThumbnailReadyPayload { @@ -486,7 +528,10 @@ export function parsePlaybackRealtimeCommand(data: string): PlaybackRealtimeComm return message?.type === "command" ? message : null; } -export function buildPlaybackRealtimeHello(sessionId: string): PlaybackRealtimeHelloEnvelope { +export function buildPlaybackRealtimeHello( + sessionId: string, + commands: PlaybackCommandName[] = SUPPORTED_PLAYBACK_COMMANDS, +): PlaybackRealtimeHelloEnvelope { return { type: "hello", session_id: sessionId, @@ -495,7 +540,7 @@ export function buildPlaybackRealtimeHello(sessionId: string): PlaybackRealtimeH version: "1", }, capabilities: { - commands: [...SUPPORTED_PLAYBACK_COMMANDS], + commands: [...commands], }, }; } From 38ce1a3013003160b23a234b873e60ecec3cf18d Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:18:00 -0400 Subject: [PATCH 37/44] fix(playback): sweep sessions that register after a copy-unsafe verdict lands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The async scan can beat the start path by milliseconds: a plan is decided, the verdict persists before the session is registered, and the notifier's immediate pass finds nothing — leaving the session on a condemned remux route with no second look (observed live on dev: plan at t, verdict at t+4ms, playback restarting on corrupt output). VideoCopyUnsafe now schedules one file-wide sweep after the settle window that considers only sessions the immediate pass never saw. Co-Authored-By: Claude Fable 5 --- internal/playback/copy_safety_notifier.go | 49 +++++++++++++++++++ .../playback/copy_safety_notifier_test.go | 32 ++++++++++++ 2 files changed, 81 insertions(+) diff --git a/internal/playback/copy_safety_notifier.go b/internal/playback/copy_safety_notifier.go index d13efc7bb..0778727cb 100644 --- a/internal/playback/copy_safety_notifier.go +++ b/internal/playback/copy_safety_notifier.go @@ -110,14 +110,53 @@ func NewCopySafetyNotifier( // VideoCopyUnsafe reports that fileID cannot be video stream-copied after all. // Sessions that are not on a copy route for that file are left alone. +// +// The verdict can land in the gap between a plan being decided and its session +// becoming visible to the lookup — the scan and the start path race, and the +// scan can win by milliseconds. One immediate pass would miss such a session +// entirely and leave it playing a route the verdict just condemned, so a second +// file-wide look runs after the settle window for sessions that appeared late. +// Sessions the first pass saw are excluded: they were either acted on or have +// their own per-session deferred look. func (n *CopySafetyNotifier) VideoCopyUnsafe(ctx context.Context, fileID int) { if n == nil || fileID <= 0 { return } + seen := make(map[string]struct{}) for _, session := range n.sessions.GetSessionsByMediaFileID(fileID) { + if session != nil && session.ID != "" { + seen[session.ID] = struct{}{} + } n.consider(ctx, session, fileID, true) } + n.sweepLateSessionsAfter(ctx, fileID, seen, n.settleWindow()) +} + +// sweepLateSessionsAfter re-lists the file's sessions once the settle window +// has passed and considers only the ones the immediate pass never saw. Like +// reconsiderAfter, it must not inherit the scan context's cancellation. +func (n *CopySafetyNotifier) sweepLateSessionsAfter(ctx context.Context, fileID int, seen map[string]struct{}, wait time.Duration) { + parent := context.WithoutCancel(ctx) + go func() { + timer := time.NewTimer(wait) + defer timer.Stop() + <-timer.C + ctx, cancel := context.WithTimeout(parent, copySafetyReconsiderTimeout) + defer cancel() + for _, session := range n.sessions.GetSessionsByMediaFileID(fileID) { + if session == nil || session.ID == "" { + continue + } + if _, handled := seen[session.ID]; handled { + continue + } + // The settle window has already elapsed since the verdict; a session + // still younger than that was planned after the verdict persisted and + // should never have been given a copy route at all. + n.consider(ctx, session, fileID, false) + } + }() } // consider decides what to do with one session the file lookup returned. @@ -185,6 +224,16 @@ func (n *CopySafetyNotifier) reconsiderAfter(ctx context.Context, sessionID stri // settleRemaining reports how much of the settle window a session still has // left. A session with no recorded start is treated as settled. +// settleWindow is the delay before the late-session sweep; zero settle keeps a +// small floor so the sweep still runs after the start path has had time to +// register the session. +func (n *CopySafetyNotifier) settleWindow() time.Duration { + if n.settle <= 0 { + return CopySafetySessionSettleWindow + } + return n.settle +} + func (n *CopySafetyNotifier) settleRemaining(session *Session) time.Duration { if n.settle <= 0 || session.StartedAt.IsZero() { return 0 diff --git a/internal/playback/copy_safety_notifier_test.go b/internal/playback/copy_safety_notifier_test.go index ef6d48f85..438c725de 100644 --- a/internal/playback/copy_safety_notifier_test.go +++ b/internal/playback/copy_safety_notifier_test.go @@ -464,3 +464,35 @@ func TestCopySafetyNotifierCompletedResultKeepsSession(t *testing.T) { t.Fatalf("stopped = %v, want the session kept after a completed replan", stopped) } } + +// Regression for the scan winning the race against the start path by +// milliseconds: the verdict lands while the session is being built, so the +// immediate pass finds nothing at all. The deferred file-wide sweep must catch +// the session that appears moments later, or it plays a condemned route +// forever (observed live: plan decided at t, verdict persisted at t+4ms, +// session registered after both, zero notifier action). +func TestCopySafetyNotifierSweepsSessionsThatAppearAfterTheVerdict(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + notifier := NewCopySafetyNotifier(sessions, nil, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 20 * time.Millisecond + + // Verdict lands first; no session exists yet. + notifier.VideoCopyUnsafe(context.Background(), 100) + + // The start path finishes registering the session just after. + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for { + if stopped := control.stoppedSessions(); len(stopped) == 1 && stopped[0] == session.ID { + return + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the late-registered session swept after the settle window", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } +} From d917f5928736bdd323339bf54626db34e6b64e1e Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:54:24 -0400 Subject: [PATCH 38/44] fix(playback): harden copy-safety invalidation against review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes from PR review: the web client defers a plan_invalidated that races an in-flight replan adoption instead of no-opping it; a race scan that finds another replica already persisted an unsafe verdict still notifies its own sessions; stopping a session now interrupts an in-flight progressive remux response (previously only the client could end it — ffmpeg was bound solely to the request context); and background scans are capped at four concurrent ffmpeg processes globally. Co-Authored-By: Claude Fable 5 --- internal/api/handlers/playback.go | 5 + internal/api/handlers/playback_test.go | 4 + internal/api/handlers/stream.go | 7 + internal/playback/copy_safety_race.go | 59 +++++- internal/playback/copy_safety_race_test.go | 124 ++++++++++- internal/playback/remux.go | 34 +++ internal/playback/session.go | 74 ++++++- internal/playback/transport_stop_test.go | 196 ++++++++++++++++++ .../player/hooks/usePlaybackSession.test.ts | 189 +++++++++++++++++ web/src/player/hooks/usePlaybackSession.ts | 65 +++++- 10 files changed, 743 insertions(+), 14 deletions(-) create mode 100644 internal/playback/transport_stop_test.go diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index c02bfeb00..b3d6ca81e 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -55,6 +55,11 @@ type SessionManagerInterface interface { TouchActivity(sessionID string) error BeginTransport(sessionID string) error EndTransport(sessionID string) error + // WatchTransportStop is required rather than probed for at run time: it is + // the only thing that can interrupt a single-response transport, so an + // implementation without it would serve progressive remuxes that no session + // stop can withdraw. + WatchTransportStop(sessionID string) (<-chan struct{}, func()) SetRemoteTransport(sessionID string, remote bool) error SetEffectiveMediaFileID(sessionID string, fileID int) error SetTranscodeNodeURL(sessionID, url string) error diff --git a/internal/api/handlers/playback_test.go b/internal/api/handlers/playback_test.go index 80f1400a9..c02df3742 100644 --- a/internal/api/handlers/playback_test.go +++ b/internal/api/handlers/playback_test.go @@ -185,6 +185,10 @@ func (failingSessionManager) BeginTransport(string) error { return nil } func (failingSessionManager) EndTransport(string) error { return nil } +func (failingSessionManager) WatchTransportStop(string) (<-chan struct{}, func()) { + return nil, func() {} +} + func (failingSessionManager) SetRemoteTransport(string, bool) error { return nil } func (failingSessionManager) SetEffectiveMediaFileID(string, int) error { return nil } diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index 3f354de32..8adf4e989 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -166,6 +166,12 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { _ = h.sessionMgr.EndTransport(sessionID) }() } + // A progressive remux runs for the length of the title behind a single + // response, so a stop decided while it is playing — a copy-safety + // verdict withdrawing the route, an admin kill — has to reach the + // stream itself. Nothing else can: the ffmpeg belongs to this request. + abort, releaseAbort := h.sessionMgr.WatchTransportStop(sessionID) + defer releaseAbort() seekSeconds := 0.0 if seekStr := r.URL.Query().Get("seek"); seekStr != "" { if s, err := strconv.ParseFloat(seekStr, 64); err == nil && s >= 0 { @@ -183,6 +189,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { AudioOnly: file.IsAudioOnly(), TargetAudioChannels: session.TargetAudioChannels, TargetAudioBitrateKbps: session.TargetAudioBitrateKbps, + Abort: abort, }); err != nil { h.handleTransportStartFailure(r.Context(), session, file, err) } diff --git a/internal/playback/copy_safety_race.go b/internal/playback/copy_safety_race.go index 96843f31f..fa72c0d55 100644 --- a/internal/playback/copy_safety_race.go +++ b/internal/playback/copy_safety_race.go @@ -17,6 +17,15 @@ import ( // belongs to the HTTP request that triggered it. const copySafetyScanTimeout = time.Minute +// copySafetyScanConcurrency caps how many copy-safety scans run at once across +// the whole replica. Per-file dedupe collapses the repeat requests for one +// popular file, but nothing bounds the number of *distinct* unknown files a +// burst of watch-page loads can name, and each one costs an ffmpeg process +// reading the opening seconds off remote storage. Excess races block their +// goroutine on the semaphore rather than being dropped: the scan is cheap to +// defer and must still happen, goroutines are cheap, ffmpeg is not. +const copySafetyScanConcurrency = 4 + // CopySafetyScanner is the scanner-side half of the race: it decides whether a // file still needs the H.264 multi-PPS scan and runs it. *scanner.PlaybackProbeEnsurer // implements it. @@ -48,7 +57,11 @@ type CopySafetyRace struct { // load for a popular file would otherwise stack a goroutine that does // nothing but wait on it. inFlight sync.Map // file ID -> struct{} - timeout time.Duration + // slots is the replica-wide scan semaphore. A goroutine holds its per-file + // inFlight entry while it waits for a slot, so queueing never lets a second + // goroutine for the same file through. + slots chan struct{} + timeout time.Duration } // NewCopySafetyRace returns a racer, or nil when it has nothing to scan with. A @@ -57,7 +70,13 @@ func NewCopySafetyRace(scanner CopySafetyScanner, files CopySafetyFileLoader, no if scanner == nil || files == nil { return nil } - return &CopySafetyRace{scanner: scanner, files: files, notifier: notifier, timeout: copySafetyScanTimeout} + return &CopySafetyRace{ + scanner: scanner, + files: files, + notifier: notifier, + slots: make(chan struct{}, copySafetyScanConcurrency), + timeout: copySafetyScanTimeout, + } } // RaceScan resolves the copy-safety verdict for fileID in the background. It @@ -76,10 +95,28 @@ func (r *CopySafetyRace) RaceScan(fileID int) { } go func() { defer r.inFlight.Delete(fileID) + // The slot is taken before the scan's own deadline starts: time spent + // queueing behind other files is not time the scan was given to run. + r.acquireSlot() + defer r.releaseSlot() r.scan(fileID) }() } +func (r *CopySafetyRace) acquireSlot() { + if r.slots == nil { + return + } + r.slots <- struct{}{} +} + +func (r *CopySafetyRace) releaseSlot() { + if r.slots == nil { + return + } + <-r.slots +} + func (r *CopySafetyRace) scan(fileID int) { timeout := r.timeout if timeout <= 0 { @@ -97,6 +134,24 @@ func (r *CopySafetyRace) scan(fileID int) { return } if !r.scanner.NeedsCopySafetyScan(file) { + // Nothing left to scan, but that is not the same as nothing to do. The + // verdict may have been reached by another replica between this race + // being requested and the file being loaded: that replica notified its + // own sessions and has no way to reach ours, so a persisted unsafe + // verdict has to be applied locally even though no scan runs here. A + // known-safe verdict is silent, as always. + // + // This closes the window for sessions this replica raced against another + // replica's write. It is not distributed invalidation: a verdict that + // lands after every replica has stopped racing still reaches only the + // replica that reached it. Pushing invalidations across replicas — + // Redis-backed, like the other cross-replica playback signals — is + // follow-up work. + if multi, known := file.PersistedVideoCopyVerdict(); known && multi { + slog.InfoContext(ctx, "applying a persisted copy-unsafe verdict reached elsewhere", + "component", "playback", "file_id", fileID) + r.notifier.VideoCopyUnsafe(ctx, fileID) + } return } diff --git a/internal/playback/copy_safety_race_test.go b/internal/playback/copy_safety_race_test.go index 7ea0c7733..24626f9ab 100644 --- a/internal/playback/copy_safety_race_test.go +++ b/internal/playback/copy_safety_race_test.go @@ -11,13 +11,15 @@ import ( ) type fakeCopySafetyScanner struct { - mu sync.Mutex - needs bool - multi bool - err error - scans int - release chan struct{} - scanning chan struct{} + mu sync.Mutex + needs bool + multi bool + err error + scans int + active int + maxActive int + release chan struct{} + scanning chan struct{} } func (s *fakeCopySafetyScanner) NeedsCopySafetyScan(*models.MediaFile) bool { @@ -29,7 +31,16 @@ func (s *fakeCopySafetyScanner) NeedsCopySafetyScan(*models.MediaFile) bool { func (s *fakeCopySafetyScanner) ScanCopySafety(context.Context, *models.MediaFile) (bool, error) { s.mu.Lock() s.scans++ + s.active++ + if s.active > s.maxActive { + s.maxActive = s.active + } s.mu.Unlock() + defer func() { + s.mu.Lock() + s.active-- + s.mu.Unlock() + }() if s.scanning != nil { s.scanning <- struct{}{} } @@ -45,6 +56,14 @@ func (s *fakeCopySafetyScanner) scanCount() int { return s.scans } +// peakConcurrency is the largest number of scans that were ever running at the +// same time. +func (s *fakeCopySafetyScanner) peakConcurrency() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.maxActive +} + type fakeFileLoader struct { mu sync.Mutex file *models.MediaFile @@ -62,6 +81,17 @@ func (l *fakeFileLoader) GetByID(context.Context, int) (*models.MediaFile, error // raceFixture wires a racer whose notifier reports into a fake control, so a // multi-PPS verdict is observable as a session stop. func raceFixture(t *testing.T, scanner *fakeCopySafetyScanner) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { + t.Helper() + return raceFixtureForFile(t, scanner, &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + }) +} + +// raceFixtureForFile is raceFixture over a specific media file, for the cases +// that care about what the row carries — a persisted verdict, above all. +func raceFixtureForFile(t *testing.T, scanner *fakeCopySafetyScanner, file *models.MediaFile) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { t.Helper() sessions := NewSessionManager(0, 0) hub := NewRealtimeHub() @@ -72,7 +102,7 @@ func raceFixture(t *testing.T, scanner *fakeCopySafetyScanner) (*CopySafetyRace, // These tests are about the race, not about waiting out the window a // just-started session gets before it can be stopped. notifier.settle = 0 - loader := &fakeFileLoader{file: &models.MediaFile{ID: 100, CodecVideo: "h264", VideoTracks: []models.VideoTrack{{Codec: "h264"}}}} + loader := &fakeFileLoader{file: file} return NewCopySafetyRace(scanner, loader, notifier), sessions, control } @@ -178,6 +208,84 @@ func TestCopySafetyRaceDedupesInFlightScans(t *testing.T) { } } +// fileWithPersistedVerdict is a media file row whose multi-PPS verdict is +// already recorded and still describes the file, as it would be on a replica +// that loads the row after another replica wrote it. +func fileWithPersistedVerdict(multi bool) *models.MediaFile { + size := int64(4096) + return &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264", MultiplePPS: &multi}}, + FileSize: size, + MultiplePPS: &multi, + MultiplePPSScanSize: &size, + } +} + +// Another replica can reach the verdict first: it persists it, notifies its own +// sessions, and cannot reach ours. Loading a row that already carries an unsafe +// verdict therefore has to withdraw this replica's copy-routed sessions even +// though there is nothing left to scan. +func TestCopySafetyRaceAppliesPersistedUnsafeVerdict(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: false} + race, sessions, control := raceFixtureForFile(t, scanner, fileWithPersistedVerdict(true)) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForStop(t, control, session.ID) + if got := scanner.scanCount(); got != 0 { + t.Fatalf("scans = %d, want 0 for a verdict that was already reached", got) + } +} + +// A persisted copy-safe verdict is the common resolved state and must stay +// silent: it is exactly the evidence that the route the session is on is fine. +func TestCopySafetyRaceIgnoresPersistedSafeVerdict(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: false} + race, sessions, control := raceFixtureForFile(t, scanner, fileWithPersistedVerdict(false)) + if _, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false); err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + time.Sleep(20 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want no session touched by a persisted copy-safe verdict", stopped) + } +} + +// Per-file dedupe bounds the races for one popular file; nothing bounds the +// number of distinct files a burst of watch-page loads names, and each scan +// costs an ffmpeg process against remote storage. +func TestCopySafetyRaceCapsConcurrentScans(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, release: make(chan struct{})} + race, _, _ := raceFixture(t, scanner) + + const files = 12 + for i := 0; i < files; i++ { + race.RaceScan(200 + i) + } + + waitForScans(t, scanner, copySafetyScanConcurrency) + time.Sleep(50 * time.Millisecond) + if got := scanner.scanCount(); got != copySafetyScanConcurrency { + t.Fatalf("scans = %d, want %d in flight while every slot is held", got, copySafetyScanConcurrency) + } + + // Every queued race still runs; the cap defers work rather than dropping it. + close(scanner.release) + waitForScans(t, scanner, files) + if got := scanner.peakConcurrency(); got > copySafetyScanConcurrency { + t.Fatalf("peak concurrency = %d, want at most %d", got, copySafetyScanConcurrency) + } +} + // The route test lives with the racer so start and replan cannot disagree: only // a plan that actually stream-copies video is worth chasing. func TestCopySafetyRaceForPlanOnlyChasesCopyRoutes(t *testing.T) { diff --git a/internal/playback/remux.go b/internal/playback/remux.go index 2c5c833ab..75875cd2a 100644 --- a/internal/playback/remux.go +++ b/internal/playback/remux.go @@ -328,6 +328,20 @@ func (s *RemuxSession) Read(p []byte) (int, error) { return s.outputPipe.Read(p) } +// Abort kills the ffmpeg process without draining or reaping it. +// +// It exists for callers that are not the owner of the session: killing ffmpeg +// closes the output pipe, which is what unblocks a copy loop parked in Read, and +// the owner's deferred Close then does the draining and the wait. Close itself +// cannot be used for that — it reads the pipe and calls cmd.Wait, neither of +// which may run concurrently with the owner's Read. +func (s *RemuxSession) Abort() { + if s == nil || s.cancel == nil { + return + } + s.cancel() +} + // Close stops the ffmpeg process and cleans up all resources. // It is safe to call Close multiple times. func (s *RemuxSession) Close() error { @@ -370,6 +384,12 @@ type RemuxServeOptions struct { // output. Zero values retain the historical stereo 192 kbps behavior. TargetAudioChannels int TargetAudioBitrateKbps int + // Abort ends the response early when it is closed. A progressive remux is + // one long response, so without it the only thing that can stop the stream + // is the client itself — a server-initiated session stop cannot withdraw a + // route the client is still being fed. Callers that serve a session pass + // SessionManager.WatchTransportStop's channel. + Abort <-chan struct{} } // RemuxContentType returns the override required for an audio-only fMP4. @@ -421,6 +441,20 @@ func ServeRemuxWithOptions(w http.ResponseWriter, r *http.Request, filePath, out } defer session.Close() + if opts.Abort != nil { + // Deferred after session.Close, so it runs before it: the watcher is + // gone by the time the owner drains and reaps the process. + served := make(chan struct{}) + defer close(served) + go func() { + select { + case <-opts.Abort: + session.Abort() + case <-served: + } + }() + } + contentType := opts.ContentType if contentType == "" { contentType = containerMIME(outputFormat) diff --git a/internal/playback/session.go b/internal/playback/session.go index 86256f676..bf65138e6 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -257,6 +257,9 @@ type SessionManager struct { activeGrace time.Duration pausedGrace time.Duration expireHook func(*Session) + // transportStops holds the stop channels of media transports this replica + // is currently serving, keyed by session ID. See WatchTransportStop. + transportStops map[string]map[chan struct{}]struct{} } // SessionLimits stores per-user admission limits. Zero values mean unlimited. @@ -1270,7 +1273,75 @@ func (m *SessionManager) EndTransport(sessionID string) error { return nil } -// StopSession removes a session from the manager. +// WatchTransportStop returns a channel that is closed when the session is +// stopped, and the release the caller must run when its transport ends. +// +// A progressive remux is a single HTTP response whose ffmpeg is owned by the +// serving handler and canceled only by that request's context, so removing the +// session from this manager does not reach it: an unnegotiated client would go +// on consuming a stream the server has already disowned — for a copy-unsafe +// source, corrupt bytes its decoder cannot recover from. This is the smallest +// handle that lets a stop interrupt one. Segmented transports (HLS, transcode) +// do not need it: each of their requests is short, and the next one is refused +// once the session is gone. +// +// The channel is closed at most once: StopSession takes the whole watcher set +// out of the map under the lock before closing it, and release drops a watcher +// that was never signaled. +func (m *SessionManager) WatchTransportStop(sessionID string) (<-chan struct{}, func()) { + if m == nil || sessionID == "" { + return nil, func() {} + } + + stop := make(chan struct{}) + m.mu.Lock() + if m.transportStops == nil { + m.transportStops = make(map[string]map[chan struct{}]struct{}) + } + watchers, ok := m.transportStops[sessionID] + if !ok { + watchers = make(map[chan struct{}]struct{}) + m.transportStops[sessionID] = watchers + } + watchers[stop] = struct{}{} + m.mu.Unlock() + + var once sync.Once + return stop, func() { + once.Do(func() { m.releaseTransportStop(sessionID, stop) }) + } +} + +func (m *SessionManager) releaseTransportStop(sessionID string, stop chan struct{}) { + m.mu.Lock() + defer m.mu.Unlock() + + watchers, ok := m.transportStops[sessionID] + if !ok { + return + } + delete(watchers, stop) + if len(watchers) == 0 { + delete(m.transportStops, sessionID) + } +} + +// stopTransportsLocked signals every transport registered for the session. The +// close is cheap and never blocks, and the watchers it wakes cancel an ffmpeg +// rather than calling back into the manager, so it is safe to do under the lock. +func (m *SessionManager) stopTransportsLocked(sessionID string) { + watchers, ok := m.transportStops[sessionID] + if !ok { + return + } + delete(m.transportStops, sessionID) + for stop := range watchers { + close(stop) + } +} + +// StopSession removes a session from the manager and interrupts any media +// transport it is still serving. func (m *SessionManager) StopSession(sessionID string) error { m.mu.Lock() defer m.mu.Unlock() @@ -1280,6 +1351,7 @@ func (m *SessionManager) StopSession(sessionID string) error { } delete(m.sessions, sessionID) + m.stopTransportsLocked(sessionID) return nil } diff --git a/internal/playback/transport_stop_test.go b/internal/playback/transport_stop_test.go new file mode 100644 index 000000000..a87f56609 --- /dev/null +++ b/internal/playback/transport_stop_test.go @@ -0,0 +1,196 @@ +package playback + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +// countingResponseWriter discards a streamed body and reports how much of it +// has been written, which is how a test observes that bytes are flowing without +// buffering a stream that never ends. +type countingResponseWriter struct { + mu sync.Mutex + header http.Header + written int64 +} + +func (w *countingResponseWriter) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *countingResponseWriter) Write(p []byte) (int, error) { + w.mu.Lock() + w.written += int64(len(p)) + w.mu.Unlock() + return len(p), nil +} + +func (w *countingResponseWriter) WriteHeader(int) {} + +func (w *countingResponseWriter) bytesWritten() int64 { + w.mu.Lock() + defer w.mu.Unlock() + return w.written +} + +// streamingFakeFFmpegScript stands in for a remux: it writes to stdout forever +// and only ever stops when it is killed. Capability probes (`-bsfs`) answer +// immediately instead, so the serve path is not blocked before it starts. +func streamingFakeFFmpegScript(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "ffmpeg") + script := "#!/bin/sh\n" + + "for arg in \"$@\"; do\n" + + " if [ \"$arg\" = \"pipe:1\" ]; then\n" + + " while :; do printf '0123456789'; sleep 0.01; done\n" + + " fi\n" + + "done\n" + + "exit 0\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return path +} + +func waitForBytes(t *testing.T, w *countingResponseWriter) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for w.bytesWritten() == 0 { + if time.Now().After(deadline) { + t.Fatal("no remux bytes reached the response") + } + time.Sleep(time.Millisecond) + } +} + +// A progressive remux is one long response whose ffmpeg belongs to the request +// that started it. Withdrawing the route — a copy-safety verdict, an admin kill +// — only reaches the client if the stop can end the response itself. +func TestServeRemuxAbortEndsTheResponse(t *testing.T) { + source := filepath.Join(t.TempDir(), "movie.mkv") + if err := os.WriteFile(source, []byte("not really a movie"), 0o644); err != nil { + t.Fatal(err) + } + + abort := make(chan struct{}) + recorder := &countingResponseWriter{} + request := httptest.NewRequest(http.MethodGet, "/api/v1/stream/session-1", nil) + + served := make(chan error, 1) + go func() { + served <- ServeRemuxWithOptions(recorder, request, source, "mp4", 0, false, 0, 0, RemuxServeOptions{ + FFmpegPath: streamingFakeFFmpegScript(t), + Abort: abort, + }) + }() + + waitForBytes(t, recorder) + select { + case err := <-served: + t.Fatalf("the remux response ended on its own: %v", err) + case <-time.After(50 * time.Millisecond): + } + + close(abort) + select { + case err := <-served: + if err != nil { + t.Fatalf("ServeRemuxWithOptions after abort = %v, want nil", err) + } + case <-time.After(10 * time.Second): + t.Fatal("the remux response outlived the stop that withdrew its session") + } +} + +func TestWatchTransportStopSignalsOnStopSession(t *testing.T) { + sessions := NewSessionManager(0, 0) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + stop, release := sessions.WatchTransportStop(session.ID) + defer release() + + select { + case <-stop: + t.Fatal("the transport was signaled while its session was still live") + default: + } + + if err := sessions.StopSession(session.ID); err != nil { + t.Fatalf("StopSession: %v", err) + } + select { + case <-stop: + case <-time.After(time.Second): + t.Fatal("stopping the session did not signal its in-flight transport") + } + + // A stop that already signaled must not be signaled again by the release + // the serving handler runs on its way out; a second close would panic. + release() + release() +} + +// Two transports can share a session — a client that reconnects while the old +// response is still draining — and a stop has to end both. +func TestWatchTransportStopSignalsEveryTransport(t *testing.T) { + sessions := NewSessionManager(0, 0) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + first, releaseFirst := sessions.WatchTransportStop(session.ID) + defer releaseFirst() + second, releaseSecond := sessions.WatchTransportStop(session.ID) + defer releaseSecond() + + if err := sessions.StopSession(session.ID); err != nil { + t.Fatalf("StopSession: %v", err) + } + for i, stop := range []<-chan struct{}{first, second} { + select { + case <-stop: + case <-time.After(time.Second): + t.Fatalf("transport %d was left streaming after its session was stopped", i) + } + } +} + +// A transport that finished normally unregisters itself, so a later stop for +// the same session ID has nothing to signal and nothing to leak. +func TestWatchTransportStopReleaseUnregisters(t *testing.T) { + sessions := NewSessionManager(0, 0) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + stop, release := sessions.WatchTransportStop(session.ID) + release() + + if err := sessions.StopSession(session.ID); err != nil { + t.Fatalf("StopSession: %v", err) + } + select { + case <-stop: + t.Fatal("a released transport was signaled") + default: + } + sessions.mu.RLock() + remaining := len(sessions.transportStops) + sessions.mu.RUnlock() + if remaining != 0 { + t.Fatalf("transportStops holds %d sessions, want 0", remaining) + } +} diff --git a/web/src/player/hooks/usePlaybackSession.test.ts b/web/src/player/hooks/usePlaybackSession.test.ts index 0bff22276..1f4d9241c 100644 --- a/web/src/player/hooks/usePlaybackSession.test.ts +++ b/web/src/player/hooks/usePlaybackSession.test.ts @@ -1529,6 +1529,195 @@ describe("usePlaybackSession server-invalidated plans", () => { unmount(); }); + function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((settle) => { + resolve = settle; + }); + return { promise, resolve }; + } + + /** + * A start and a replan whose response is held open, so a test can act while + * the client has a decision in flight — the state the server is always in + * when it pushes an invalidation: it commits the replacement plan and starts + * the copy-safety scan behind it before the response is on the wire. + */ + function gatedReplanFetchMock( + replanBodies: Array>, + replans: unknown[], + gate: Promise, + ) { + return vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/playback/start")) { + return jsonResponse( + { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3(), + }, + { status: 201 }, + ); + } + if (url.endsWith("/playback/session-1/replan")) { + replanBodies.push(JSON.parse(String(init?.body)) as Record); + if (replanBodies.length === 1) await gate; + return jsonResponse(replans.shift()); + } + if (url.endsWith("/playback/route-events")) { + return new Response(null, { status: 202 }); + } + if (init?.method === "DELETE") { + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected request: ${url}`); + }); + } + + function playableDecision(plan: ReturnType) { + return { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: plan, + }; + } + + // The server commits the replacement plan and starts the scan behind it + // before the client can read the response, so the invalidation can name a + // plan this client has not adopted yet. Deciding against the plan on screen + // would complete the command as a no-op and then let the pending response + // install the very route the server withdrew. + it("waits out an in-flight replan and recovers off the plan it adopts", async () => { + const replanBodies: Array> = []; + const gate = deferred(); + vi.stubGlobal( + "fetch", + gatedReplanFetchMock( + replanBodies, + [ + playableDecision( + fixturePlanV3({ + plan_id: "plan:2222222222222222", + plan_attempt_key: "v3:2222222222222222", + }), + ), + playableDecision( + fixturePlanV3({ + plan_id: "plan:3333333333333333", + plan_attempt_key: "v3:3333333333333333", + delivery: "server_transcode_hls", + }), + ), + ], + gate.promise, + ), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + act(() => { + result.current.changeQuality("720p", 100); + }); + await waitFor(() => expect(replanBodies).toHaveLength(1)); + + let invalidation: Promise | undefined; + act(() => { + invalidation = result.current.invalidatePlan( + "plan:2222222222222222", + "video_copy_unsafe", + 100, + ); + }); + // Nothing may be decided yet: the plan the command names is still in the + // response the client has not read. + expect(replanBodies).toHaveLength(1); + + let outcome: boolean | undefined; + await act(async () => { + gate.resolve(); + outcome = await invalidation; + }); + + expect(outcome).toBe(true); + expect(replanBodies).toHaveLength(2); + expect(replanBodies[1]).toMatchObject({ + operation: "failure_recovery", + failed_plan_id: "plan:2222222222222222", + // The plan that was invalidated mid-adoption is the one excluded, not the + // one that was on screen when the command arrived. + attempted_plan_keys: ["v3:2222222222222222"], + failure: { classification: "video_copy_unsafe" }, + }); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:3333333333333333")); + + unmount(); + }); + + // The mirror image: the client really did move past the invalidated plan + // while the command was in flight. Waiting must not turn that into a replan — + // it would evict a route the server never complained about. + it("stays a no-op for a plan the in-flight replan replaced", async () => { + const replanBodies: Array> = []; + const gate = deferred(); + vi.stubGlobal( + "fetch", + gatedReplanFetchMock( + replanBodies, + [ + playableDecision( + fixturePlanV3({ + plan_id: "plan:2222222222222222", + plan_attempt_key: "v3:2222222222222222", + }), + ), + ], + gate.promise, + ), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + act(() => { + result.current.changeQuality("720p", 100); + }); + await waitFor(() => expect(replanBodies).toHaveLength(1)); + + let invalidation: Promise | undefined; + act(() => { + invalidation = result.current.invalidatePlan( + "plan:0123456789abcdef", + "video_copy_unsafe", + 100, + ); + }); + + let outcome: boolean | undefined; + await act(async () => { + gate.resolve(); + outcome = await invalidation; + }); + + // Reported as handled, with no second replan: the invalidated route is gone. + expect(outcome).toBe(true); + expect(replanBodies).toHaveLength(1); + expect(result.current.plan?.plan_id).toBe("plan:2222222222222222"); + + unmount(); + }); + it("reports failure when the replan produces no replacement plan", async () => { const replanBodies: Array> = []; vi.stubGlobal( diff --git a/web/src/player/hooks/usePlaybackSession.ts b/web/src/player/hooks/usePlaybackSession.ts index aab0d1286..4a53a1845 100644 --- a/web/src/player/hooks/usePlaybackSession.ts +++ b/web/src/player/hooks/usePlaybackSession.ts @@ -316,6 +316,14 @@ export function usePlaybackSession( const attemptedPlanKeysRef = useRef([]); const attemptCountRef = useRef(1); const replanInFlightRef = useRef(false); + // Adoptions in flight: a start or a replan whose decision has not been + // applied yet. The server commits a replacement plan — and starts the + // copy-safety scan behind it — before the client can read the response, so a + // `plan_invalidated` command can name a plan this client is still adopting. + // Waiters registered here are woken once nothing is in flight, which lets an + // invalidation decide against the plan that actually won. + const adoptionsInFlightRef = useRef(0); + const adoptionWaitersRef = useRef void>>([]); const pendingReplanRef = useRef<{ options: ReplanOptions; loadSequence: number; @@ -332,6 +340,37 @@ export function usePlaybackSession( stateRef.current = state; }, [state]); + const beginAdoption = useCallback(() => { + adoptionsInFlightRef.current += 1; + }, []); + + /** + * Counts one in-flight adoption out. + * + * Waiters are woken only when nothing is left in flight: a queued replan is + * dispatched from its predecessor's `finally` before the predecessor is + * counted out, so the count tracks the whole chain rather than one request. + */ + const endAdoption = useCallback(() => { + adoptionsInFlightRef.current = Math.max(0, adoptionsInFlightRef.current - 1); + if (adoptionsInFlightRef.current > 0) return; + const waiters = adoptionWaitersRef.current; + if (waiters.length === 0) return; + adoptionWaitersRef.current = []; + for (const wake of waiters) wake(); + }, []); + + /** + * Resolves once no start or replan is in flight, or null when none is — + * callers act synchronously in the common case rather than deferring a turn. + */ + const awaitAdoptionSettled = useCallback((): Promise | null => { + if (adoptionsInFlightRef.current === 0) return null; + return new Promise((resolve) => { + adoptionWaitersRef.current.push(resolve); + }); + }, []); + const reportEvent = useCallback( ( event: RouteEventNameV3, @@ -580,6 +619,7 @@ export function usePlaybackSession( })); }; + beginAdoption(); try { const selectedFileId = selectFileId(preferredFileId); if (!selectedFileId) { @@ -642,9 +682,11 @@ export function usePlaybackSession( const nextError = describePlaybackSessionError(err, initialErrorMessage); retirePreviousSession(nextError); + } finally { + endAdoption(); } }, - [adoptDecision, requestStart, selectFileId, stopSession], + [adoptDecision, beginAdoption, endAdoption, requestStart, selectFileId, stopSession], ); useEffect(() => { @@ -818,6 +860,7 @@ export function usePlaybackSession( const loadSequence = loadSequenceRef.current; replanInFlightRef.current = true; + beginAdoption(); setState((current) => ({ ...current, replanning: true, @@ -905,13 +948,18 @@ export function usePlaybackSession( } else { pendingReplan?.resolve(false); } + // Last: a queued replan dispatched just above has already counted + // itself in, so waiters are not woken between the two links of a chain. + endAdoption(); } }, [ adoptDecision, + beginAdoption, clientCapabilities, clientPlaybackContext, config, + endAdoption, maxBitrateKbps, retireActiveSession, ], @@ -1026,14 +1074,25 @@ export function usePlaybackSession( * plan without the client reasoning about deliveries at all. The plan * revision the adopted plan bumps rebuilds the transport and restores the * position, exactly as it does after a client-detected failure. + * + * A start or replan already in flight is waited out first. The server commits + * a replacement plan and starts the copy-safety scan behind it *before* the + * response reaches the client, so an invalidation can name a plan this client + * has not adopted yet. Deciding against the plan currently on screen would + * complete the command as a no-op and then let the pending response install + * the very route the server just withdrew. */ const invalidatePlan = useCallback( async (planId: string, reason: string, currentPosition: number): Promise => { + const settling = awaitAdoptionSettled(); + if (settling) await settling; const plan = planRef.current; if (!plan) return false; // The command names the plan the server invalidated. Once the client has // moved past it there is nothing to recover from, and replanning anyway - // would evict a route the server never complained about. + // would evict a route the server never complained about. That stays true + // for a plan id this client has never seen: it is a verdict for a route + // that has already been replaced, not a reason to tear the session down. if (plan.plan_id !== planId) return true; const classification = reason.trim().slice(0, 64) || "plan_invalidated"; reportEvent("plan_invalidated", { fallbackReason: classification }); @@ -1043,7 +1102,7 @@ export function usePlaybackSession( failure: { classification, message: "The server invalidated this plan." }, }); }, - [replan, reportEvent], + [awaitAdoptionSettled, replan, reportEvent], ); const reanchorSeek = useCallback( From fc0919529b215a9b97da1fedc97ec0cb1876ae56 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:04:03 -0400 Subject: [PATCH 39/44] fix(playback): validate realtime command ownership before consuming it Review fixes: a realtime result naming another session's command is now rejected before the tracker deadline is canceled or the record dropped; the concurrent-scan test waits on observable state (a gated fake ffmpeg) instead of a fixed sleep; changelog wording no longer overclaims verdict permanence. Co-Authored-By: Claude Fable 5 --- docs/feature-changelog.md | 2 +- internal/api/handlers/session_ws.go | 13 ++- .../session_ws_plan_invalidated_test.go | 44 ++++++++++ .../probe_repair_copy_safety_persist_test.go | 86 ++++++++++++++++--- 4 files changed, 126 insertions(+), 19 deletions(-) diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index a71065745..a5f21331f 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -12,7 +12,7 @@ If the analysis then finds the file genuinely cannot be copied, Silo moves the s ### Browsing no longer waits on H.264 stream-copy analysis Silo checks each H.264 file once for a bitstream quirk that makes stream-copying unsafe. That check reads the opening seconds of the file, and it used to run while a media page was loading and be forgotten on every restart — so browsing a library, especially after a reboot, re-read part of every H.264 file. On remote or cloud storage that was the difference between an instant page and a slow one. -Three things change. Media pages no longer trigger the analysis at all; it now happens when a play is actually being prepared, so browsing is fast regardless of where the files live. The result is stored on the file instead of being kept only in memory, so it survives restarts and is computed at most once per file. And the check itself reads 5 seconds instead of 15. +Three things change. Media pages no longer trigger the analysis at all; it now happens when a play is actually being prepared, so browsing is fast regardless of where the files live. The result is stored on the file instead of being kept only in memory, so it survives restarts and is reused while the file's size and modification time still match. And the check itself reads 5 seconds instead of 15. A file that changes on disk is re-checked automatically: the stored answer is only trusted while the file's size and modification time still match, so re-encoding or replacing a file in place invalidates it without any manual step. Nothing is recorded when an analysis fails, so a transient error never turns into a stale verdict — the next request simply retries. No configuration changes, and playback behavior is unchanged. diff --git a/internal/api/handlers/session_ws.go b/internal/api/handlers/session_ws.go index b6338c7d2..59c4b1d0f 100644 --- a/internal/api/handlers/session_ws.go +++ b/internal/api/handlers/session_ws.go @@ -166,17 +166,22 @@ func (h *PlaybackHandler) handleRealtimeClientMessage(sessionID string, data []b return playback.ErrInvalidRealtimePayload } h.touchSessionActivity(sessionID) + // Establish ownership before mutating anything: a result naming another + // session's command must be rejected without canceling that command's + // deadline or dropping its record. An unknown command_id is not an + // error — a duplicate or late result for an already-completed command + // is normal traffic. + record, ok := h.getRealtimeCommand(result.CommandID) + if ok && record.SessionID != sessionID { + return playback.ErrInvalidRealtimePayload + } if h.CommandTracker != nil { h.CommandTracker.Result(result.CommandID) } - record, ok := h.getRealtimeCommand(result.CommandID) if !ok { return nil } h.forgetRealtimeCommand(result.CommandID) - if record.SessionID != sessionID { - return playback.ErrInvalidRealtimePayload - } if result.Status != playback.RealtimeResultStatusCompleted { // A rejected plan_invalidated leaves the client running a route the // server has withdrawn, and the tracker's deadline was already diff --git a/internal/api/handlers/session_ws_plan_invalidated_test.go b/internal/api/handlers/session_ws_plan_invalidated_test.go index 6d3c5dfc1..143460278 100644 --- a/internal/api/handlers/session_ws_plan_invalidated_test.go +++ b/internal/api/handlers/session_ws_plan_invalidated_test.go @@ -2,7 +2,9 @@ package handlers import ( "encoding/json" + "errors" "testing" + "time" "github.com/Silo-Server/silo-server/internal/playback" ) @@ -69,3 +71,45 @@ func TestRealtimeCompletedPlanInvalidationKeepsSession(t *testing.T) { t.Fatalf("GetSession after a completed replan: %v, want the session kept", err) } } + +// Rejecting a result has to be side-effect-free: a client claiming another +// session's command_id must not cancel that command's fallback deadline or +// drop its record, or one session could silently disarm another's recovery. +func TestRealtimeResultForOtherSessionCommandLeavesTrackerArmed(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr) + handler.CommandTracker = playback.NewCommandTracker() + defer handler.CommandTracker.Close() + + sessionA, err := sessionMgr.StartSession(1, "profile-1", 100, playback.PlayRemux, false) + if err != nil { + t.Fatalf("StartSession A: %v", err) + } + sessionB, err := sessionMgr.StartSession(2, "profile-2", 200, playback.PlayRemux, false) + if err != nil { + t.Fatalf("StartSession B: %v", err) + } + if sessionA.ID == sessionB.ID { + t.Fatal("StartSession returned colliding session IDs, want distinct sessions") + } + + handler.rememberRealtimeCommand("cmd-1", sessionB.ID, playback.CommandPlanInvalidated) + fired := make(chan struct{}) + handler.CommandTracker.Track("cmd-1", 20*time.Millisecond, func() { close(fired) }) + + err = handler.handleRealtimeClientMessage(sessionA.ID, + realtimeResultMessage(t, sessionA.ID, "cmd-1", playback.RealtimeResultStatusCompleted)) + if !errors.Is(err, playback.ErrInvalidRealtimePayload) { + t.Fatalf("handleRealtimeClientMessage: %v, want ErrInvalidRealtimePayload", err) + } + + if _, ok := handler.getRealtimeCommand("cmd-1"); !ok { + t.Fatal("a rejected cross-session result forgot the command record, want it kept") + } + + select { + case <-fired: + case <-time.After(2 * time.Second): + t.Fatal("command deadline never fired, want it still armed after a rejected cross-session result") + } +} diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index b8e86ff2b..d9d96ac9e 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "testing" "time" @@ -34,22 +35,51 @@ func fakeFFmpeg(t *testing.T, stdoutPayload string, delay time.Duration) (string if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { t.Fatalf("write fake ffmpeg: %v", err) } - return ffmpegPath, func() int { - data, err := os.ReadFile(logPath) - if err != nil { - if os.IsNotExist(err) { - return 0 - } - t.Fatalf("read fake ffmpeg log: %v", err) + return ffmpegPath, func() int { return countFFmpegRuns(t, logPath) } +} + +// fakeFFmpegGated is like fakeFFmpeg but blocks after recording its invocation +// until the returned release func is called, so a test can observe that a scan +// has actually started — rather than assume it via a fixed sleep — before +// letting it complete. +func fakeFFmpegGated(t *testing.T, stdoutPayload string) (ffmpegPath string, runs func() int, release func()) { + t.Helper() + dir := t.TempDir() + logPath := filepath.Join(dir, "invocations.log") + releasePath := filepath.Join(dir, "release") + ffmpegPath = filepath.Join(dir, "ffmpeg") + // The invocation is logged before the gate so the log is the signal that + // this process has started, not that it has finished. + script := fmt.Sprintf("#!/bin/sh\necho run >> %q\nwhile [ ! -f %q ]; do sleep 0.01; done\nprintf '%s'\n", logPath, releasePath, stdoutPayload) + if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil { + t.Fatalf("write gated fake ffmpeg: %v", err) + } + runs = func() int { return countFFmpegRuns(t, logPath) } + release = func() { + if err := os.WriteFile(releasePath, nil, 0o644); err != nil { + t.Fatalf("write gated fake ffmpeg release file: %v", err) } - runs := 0 - for _, b := range data { - if b == '\n' { - runs++ - } + } + return ffmpegPath, runs, release +} + +// countFFmpegRuns reports how many invocations a fake ffmpeg has logged. +func countFFmpegRuns(t *testing.T, logPath string) int { + t.Helper() + data, err := os.ReadFile(logPath) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read fake ffmpeg log: %v", err) + } + runs := 0 + for _, b := range data { + if b == '\n' { + runs++ } - return runs } + return runs } type recordedPPSWrite struct { @@ -338,7 +368,7 @@ func TestEnsureProbeOnlySkipsCopySafetyScan(t *testing.T) { } func TestEnsureCopySafetyConcurrentCallsScanOnce(t *testing.T) { - ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 200*time.Millisecond) + ffmpegPath, runs, release := fakeFFmpegGated(t, conflictingPPSAnnexB) writer := &fakeCopySafetyWriter{} ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} @@ -346,6 +376,7 @@ func TestEnsureCopySafetyConcurrentCallsScanOnce(t *testing.T) { const callers = 8 var wg sync.WaitGroup + var entered atomic.Int64 results := make([]*models.MediaFile, callers) errs := make([]error, callers) start := make(chan struct{}) @@ -354,11 +385,38 @@ func TestEnsureCopySafetyConcurrentCallsScanOnce(t *testing.T) { go func(i int) { defer wg.Done() <-start + entered.Add(1) results[i], errs[i] = ensurer.ensureCopySafety(context.Background(), copySafetyTestFile(mtime)) }(i) } close(start) + + // Hold the winning ffmpeg inside its scan until observable state says the + // dedup path is genuinely under test: every caller has reached the + // ensureCopySafety call site, and exactly one ffmpeg process has started. + // + // ensureCopySafety dedupes through a singleflight.Group, which exposes no + // waiter count, so "all callers entered" is the strongest signal available + // from outside the package. It is sufficient here: between the call site + // and singleflight.Do, ensureCopySafety only does non-blocking work (a + // codec check and a sync.Map lookup), so a caller that has entered reaches + // the dedup point without waiting on anything — and the scan it would + // otherwise start for itself is still blocked when it gets there. + timeout := "" + deadline := time.Now().Add(5 * time.Second) + for int(entered.Load()) != callers || runs() != 1 { + if time.Now().After(deadline) { + timeout = fmt.Sprintf("timed out waiting for the deduped scan to start: %d/%d callers entered, %d ffmpeg runs", entered.Load(), callers, runs()) + break + } + time.Sleep(5 * time.Millisecond) + } + // Always release, even on timeout, so the callers are not left blocked. + release() wg.Wait() + if timeout != "" { + t.Fatal(timeout) + } for i, err := range errs { if err != nil { From cd0c31f35c1f6d8ba8e4000e63cc84e6faa5525f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:25:55 -0400 Subject: [PATCH 40/44] fix(playback): close copy-safety races in replan commits, transport stops, and reconstruction Review round two: sessions the notifier could not classify mid-replan-commit stay eligible for the post-settle sweep instead of being marked handled; WatchTransportStop returns an already-closed channel for a session stopped before registration; reconstructing a video stream-copy transport (progressive or HLS) now consults the persisted verdict, closing the replica-failover hole where a condemned remux could be re-served with nothing left to withdraw it; and a verdict whose database write failed is memoized as unpersisted and the write retried on later requests without rerunning ffmpeg. Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 13 ++ internal/api/handlers/playback.go | 12 +- internal/api/handlers/playback_copy_safety.go | 81 ++++++++ internal/api/handlers/stream.go | 13 +- .../api/handlers/stream_copy_safety_test.go | 177 ++++++++++++++++++ internal/playback/copy_safety_notifier.go | 57 ++++-- .../playback/copy_safety_notifier_test.go | 105 ++++++++++- internal/playback/recipecard.go | 13 ++ internal/playback/session.go | 12 ++ internal/playback/transcode_manager.go | 26 ++- internal/playback/transport_stop_test.go | 37 ++++ internal/scanner/probe_repair.go | 84 ++++++++- .../probe_repair_copy_safety_cached_test.go | 4 +- .../probe_repair_copy_safety_persist_test.go | 60 ++++++ 14 files changed, 648 insertions(+), 46 deletions(-) create mode 100644 internal/api/handlers/playback_copy_safety.go create mode 100644 internal/api/handlers/stream_copy_safety_test.go diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 8c6d177b9..1375ecefb 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -822,6 +822,19 @@ Delivery is in-process: the replica that owns the session owns its realtime connection, so a verdict resolved on one node acts on the sessions that node is serving. +The row is what covers the gap that leaves. A signed stream URL is a durable +capability the client replays on whichever replica answers next, and a replica +that dies between persisting a verdict and pushing the invalidation takes the +only notifier that knew about it with it. The replacement replica has no live +session, so it rebuilds one from the recipe card — which would replay the exact +remux the verdict condemned. Reconstruction therefore re-reads the persisted +verdict before it rebuilds a video stream-copy transport (a progressive remux, +or an HLS transport whose video target is `copy`) and refuses with the ordinary +playback-session not-found when the verdict says the source is unsafe. The +client's existing recovery mints a fresh attempt, which plans against the same +row and lands on a transcode. Transcode reconstruction is untouched: re-encoding +the bitstream is unaffected by conflicting parameter sets. + --- ## 7. Registries diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index b3d6ca81e..b2a57c521 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -1440,11 +1440,7 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h // Local transcode whose process state was lost: reconstruct it from the // token recipe. The manifest path has no segment context, so pass -1 (use // the token's seek position). - if card == nil { - writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") - return - } - transcodeSession = h.tm.ReconstructTranscode(r.Context(), sessionID, -1, *card) + transcodeSession = h.reconstructTransportForServe(r.Context(), sessionID, -1, card) if transcodeSession == nil { writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") return @@ -1504,11 +1500,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht if segNum, parseErr := playback.ParseSegmentNumber(chi.URLParam(r, "name")); parseErr == nil { requestedSegment = segNum } - if card == nil { - writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") - return - } - transcodeSession = h.tm.ReconstructTranscode(r.Context(), sessionID, requestedSegment, *card) + transcodeSession = h.reconstructTransportForServe(r.Context(), sessionID, requestedSegment, card) if transcodeSession == nil { writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") return diff --git a/internal/api/handlers/playback_copy_safety.go b/internal/api/handlers/playback_copy_safety.go new file mode 100644 index 000000000..d28c0bef1 --- /dev/null +++ b/internal/api/handlers/playback_copy_safety.go @@ -0,0 +1,81 @@ +package handlers + +import ( + "context" + "log/slog" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// Reconstruction replays a recipe that was committed before the H.264 +// copy-safety verdict for its source was known. That is safe for everything +// except a video stream-copy: the optimistic-remux race resolves the verdict +// behind the play, and the only mechanism that withdraws a condemned remux — +// CopySafetyNotifier — reaches the in-process sessions of the replica that +// reached the verdict. +// +// A client whose replica dies between the verdict landing and the notification +// retries its signed stream URL elsewhere. The replacement replica has no live +// session, so it rebuilds one from the card and re-serves the same unsafe +// remux, with nothing left to withdraw it. Re-checking the persisted verdict at +// the moment of reconstruction closes that hole: the row is the one piece of +// state every replica shares. +// +// The refusal is a plain not-found, matching an expired or missing recipe, +// because that is the failure a client's recovery already knows how to handle +// — it mints a fresh attempt, which plans against the persisted verdict and +// lands on a transcode. + +// videoCopyReconstructRefused reports whether rebuilding a lost transport from +// card must be refused because the persisted verdict now says its source cannot +// be video stream-copied. Only copy deliveries are gated; a transcode +// reconstruct is never touched. +// +// An unreadable row is not evidence of anything and does not refuse: the +// verdict is re-checked on every request, so a database blip costs a later +// refusal rather than a spurious one. The same applies to a handler with no +// file resolver wired (optional on PlaybackHandler). +func videoCopyReconstructRefused(ctx context.Context, files FilePathResolver, card *playback.RecipeCard) bool { + if card == nil || files == nil || card.MediaFileID <= 0 || !card.VideoStreamCopy() { + return false + } + file, err := files.GetByID(ctx, card.MediaFileID) + if err != nil || file == nil { + return false + } + return videoCopyUnsafeByVerdict(ctx, file, card.SessionID) +} + +// reconstructTransportForServe rebuilds a lost local transport from the token +// recipe for the manifest and segment serve routes, refusing a video +// stream-copy the persisted copy-safety verdict has since condemned. A nil +// result is the caller's not-found, which is also what a missing card yields — +// the two are the same thing to a client: this recipe is no longer serveable. +func (h *PlaybackHandler) reconstructTransportForServe(ctx context.Context, sessionID string, requestedSegment int, card *playback.RecipeCard) *playback.TranscodeSession { + if card == nil { + return nil + } + if videoCopyReconstructRefused(ctx, h.fileResolver, card) { + return nil + } + return h.tm.ReconstructTranscode(ctx, sessionID, requestedSegment, *card) +} + +// videoCopyUnsafeByVerdict reports whether the media_files row carries a valid +// verdict condemning a video stream-copy of this file, logging the refusal it +// is about to cause. +func videoCopyUnsafeByVerdict(ctx context.Context, file *models.MediaFile, sessionID string) bool { + multi, known := file.PersistedVideoCopyVerdict() + if !known || !multi { + return false + } + slog.InfoContext(ctx, "refusing to reconstruct a copy-unsafe video stream-copy", + "component", "api", + "session", sessionID, + "playback_session_id", sessionID, + "file_id", file.ID, + "reason", playback.PlanInvalidatedVideoCopyUnsafe, + ) + return true +} diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index 8adf4e989..8fb208ea1 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -106,7 +106,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { // Without a token (or signing secret) reconstruct is off, collapsing to a // plain GetSession + ownership check. card, claims := verifiedStreamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) - session, status := h.TM.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, userID, card) + session, status, reconstructed := h.TM.LoadOrReconstructSessionDetail(r.Context(), h.sessionMgr.GetSession, sessionID, userID, card) switch status { case playback.SessionMissing: writePlaybackSessionNotFound(w) @@ -149,6 +149,17 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { } attachPlaybackSession(r.Context(), session, claims) + // A reconstructed remux replays a recipe committed before the copy-safety + // verdict existed, and no notifier can reach it — see playback_copy_safety.go. + if reconstructed && session.PlayMethod == playback.PlayRemux && + videoCopyUnsafeByVerdict(r.Context(), file, sessionID) { + // The reconstruct already registered the session; tear it down again so + // the refusal leaves no half-live session behind the client's replan. + h.abortPlaybackSession(r.Context(), session) + writePlaybackSessionNotFound(w) + return + } + switch session.PlayMethod { case playback.PlayDirect: if err := h.sessionMgr.BeginTransport(sessionID); err == nil { diff --git a/internal/api/handlers/stream_copy_safety_test.go b/internal/api/handlers/stream_copy_safety_test.go new file mode 100644 index 000000000..71d0b0142 --- /dev/null +++ b/internal/api/handlers/stream_copy_safety_test.go @@ -0,0 +1,177 @@ +package handlers + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// copySafetyStreamFile builds an H.264 file on disk with an optional persisted +// multi-PPS verdict, valid for the size and mtime the row reports. +func copySafetyStreamFile(t *testing.T, multiplePPS *bool) *models.MediaFile { + t.Helper() + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + modified := mtime + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie.mkv"), + FileSize: 1234, + FileModifiedAt: &modified, + CodecVideo: "h264", + CodecAudio: "aac", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + Duration: 3600, + } + if multiplePPS != nil { + verdict := *multiplePPS + scanSize := file.FileSize + scanMtime := mtime + file.MultiplePPS = &verdict + file.MultiplePPSScanSize = &scanSize + file.MultiplePPSScanMtime = &scanMtime + } + return file +} + +// A signed stream URL is a durable capability: the client replays it on +// whichever replica answers next. When the replica that started the play dies +// between the copy-safety scan persisting a multi-PPS verdict and the in-process +// notification going out, the retry lands somewhere with no live session, and +// the recipe card rebuilds the very remux the verdict condemned — with no +// notifier left anywhere that could withdraw it. The reconstruct has to consult +// the row, which is the only state the replicas share. +func TestHandleStream_RefusesReconstructingACopyUnsafeRemux(t *testing.T) { + const ( + secret = "test-stream-signing-secret" + sessionID = "lost-remux-session" + ) + unsafe := true + file := copySafetyStreamFile(t, &unsafe) + + sessionMgr := playback.NewSessionManager(0, 0) + tm := playback.NewTranscodeManager() + tm.Sessions = sessionMgr + + handler := NewStreamHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.TM = tm + handler.JWTSecret = secret + + card := playback.NewRemuxRecipeCard(sessionID, 1, "profile-1", file.ID, false, 0) + card.InputPath = file.FilePath + token, err := streamtoken.Sign(card.ToClaims(), secret, playback.MaxTokenTTL) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+sessionID+"?st="+token, nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleStream(rr, req) + + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s; want the reconstruct refused as not-found", rr.Code, rr.Body.String()) + } + // The refusal must not leave the session it rebuilt behind: the client's + // recovery mints a fresh attempt, and this one has no route left. + if _, err := sessionMgr.GetSession(sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetSession error = %v, want the refused reconstruction torn down", err) + } +} + +// The gate is the verdict, not the route: a remux whose source has no verdict, +// or a verdict saying the copy is safe, still reconstructs and streams. +func TestHandleStream_ReconstructsARemuxThatIsStillCopySafe(t *testing.T) { + const secret = "test-stream-signing-secret" + safe := false + + for _, tc := range []struct { + name string + sessionID string + verdict *bool + }{ + {name: "verdict says safe", sessionID: "lost-remux-safe", verdict: &safe}, + {name: "verdict unknown", sessionID: "lost-remux-unknown"}, + } { + t.Run(tc.name, func(t *testing.T) { + sessionID := tc.sessionID + file := copySafetyStreamFile(t, tc.verdict) + + sessionMgr := playback.NewSessionManager(0, 0) + tm := playback.NewTranscodeManager() + tm.Sessions = sessionMgr + + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg") + if err := os.WriteFile(ffmpeg, []byte("#!/bin/sh\nprintf muxed\n"), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + handler := NewStreamHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.TM = tm + handler.JWTSecret = secret + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{FFmpegPath: ffmpeg} + } + + card := playback.NewRemuxRecipeCard(sessionID, 1, "profile-1", file.ID, false, 0) + card.InputPath = file.FilePath + token, err := streamtoken.Sign(card.ToClaims(), secret, playback.MaxTokenTTL) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+sessionID+"?st="+token, nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleStream(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s; want the reconstruct served", rr.Code, rr.Body.String()) + } + }) + } +} + +// Only video stream-copy deliveries are gated. A transcode re-encodes the +// bitstream, so conflicting parameter sets cannot reach the client's decoder +// and the recipe stays serveable whatever the verdict says. +func TestVideoCopyReconstructRefusedOnlyGatesCopyDeliveries(t *testing.T) { + unsafe := true + file := copySafetyStreamFile(t, &unsafe) + files := testPlaybackFileResolver{file: file} + + remux := playback.NewRemuxRecipeCard("s", 1, "profile-1", file.ID, false, 0) + copyHLS := playback.RecipeCard{SessionID: "s", UserID: 1, MediaFileID: file.ID, PlayMethod: playback.PlayTranscode, TargetCodecVideo: "copy"} + transcode := playback.RecipeCard{SessionID: "s", UserID: 1, MediaFileID: file.ID, PlayMethod: playback.PlayTranscode, TargetCodecVideo: "h264"} + direct := playback.NewDirectRecipeCard("s", 1, "profile-1", file.ID) + + for _, tc := range []struct { + name string + card playback.RecipeCard + want bool + }{ + {name: "progressive remux", card: remux, want: true}, + {name: "hls video copy", card: copyHLS, want: true}, + {name: "real transcode", card: transcode}, + {name: "direct play", card: direct}, + } { + t.Run(tc.name, func(t *testing.T) { + card := tc.card + if got := videoCopyReconstructRefused(t.Context(), files, &card); got != tc.want { + t.Fatalf("videoCopyReconstructRefused() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/playback/copy_safety_notifier.go b/internal/playback/copy_safety_notifier.go index 0778727cb..78648d544 100644 --- a/internal/playback/copy_safety_notifier.go +++ b/internal/playback/copy_safety_notifier.go @@ -108,6 +108,24 @@ func NewCopySafetyNotifier( } } +// copySafetyDisposition is what one call to consider did with a session, and +// therefore whether the post-settle sweep still owes it a second look. +type copySafetyDisposition int + +const ( + // copySafetyUnresolved: consider reached no decision about this session. It + // did not look like it was stream-copying this file — which is the ordinary + // answer for an unrelated session, and also the answer during the brief + // window where a replan has already moved the live session but has not yet + // committed the durable attempt that names the new file and plan. Only the + // sweep can tell those apart, so the session stays eligible for it. + copySafetyUnresolved copySafetyDisposition = iota + // copySafetyDisposed: consider dealt with the session — invalidated it, + // stopped it, scheduled its own deferred second look, or deliberately + // exempted it. The sweep must leave it alone. + copySafetyDisposed +) + // VideoCopyUnsafe reports that fileID cannot be video stream-copied after all. // Sessions that are not on a copy route for that file are left alone. // @@ -116,8 +134,17 @@ func NewCopySafetyNotifier( // scan can win by milliseconds. One immediate pass would miss such a session // entirely and leave it playing a route the verdict just condemned, so a second // file-wide look runs after the settle window for sessions that appeared late. -// Sessions the first pass saw are excluded: they were either acted on or have -// their own per-session deferred look. +// +// Only sessions the first pass actually disposed of are excluded from that +// sweep. A session the first pass merely could not classify has to stay +// eligible: mid-replan, the live session already names the replacement file +// while the durable attempt still names the previous one, and the durable +// record wins the identity test — so the session reads as "serving another +// file" for as long as the commit takes. Marking it seen there would strand it +// permanently, because the persisted verdict also stops any later scan from +// re-notifying. Re-considering a genuinely unrelated session in the sweep costs +// one plan-store read and re-runs every check, so the conservative direction is +// free. func (n *CopySafetyNotifier) VideoCopyUnsafe(ctx context.Context, fileID int) { if n == nil || fileID <= 0 { return @@ -125,17 +152,16 @@ func (n *CopySafetyNotifier) VideoCopyUnsafe(ctx context.Context, fileID int) { seen := make(map[string]struct{}) for _, session := range n.sessions.GetSessionsByMediaFileID(fileID) { - if session != nil && session.ID != "" { + if n.consider(ctx, session, fileID, true) == copySafetyDisposed { seen[session.ID] = struct{}{} } - n.consider(ctx, session, fileID, true) } n.sweepLateSessionsAfter(ctx, fileID, seen, n.settleWindow()) } // sweepLateSessionsAfter re-lists the file's sessions once the settle window -// has passed and considers only the ones the immediate pass never saw. Like -// reconsiderAfter, it must not inherit the scan context's cancellation. +// has passed and considers every one the immediate pass did not dispose of. +// Like reconsiderAfter, it must not inherit the scan context's cancellation. func (n *CopySafetyNotifier) sweepLateSessionsAfter(ctx context.Context, fileID int, seen map[string]struct{}, wait time.Duration) { parent := context.WithoutCancel(ctx) go func() { @@ -159,19 +185,19 @@ func (n *CopySafetyNotifier) sweepLateSessionsAfter(ctx context.Context, fileID }() } -// consider decides what to do with one session the file lookup returned. -// maySettle is false on the deferred second look, so a session can never be -// postponed twice. -func (n *CopySafetyNotifier) consider(ctx context.Context, session *Session, fileID int, maySettle bool) { +// consider decides what to do with one session the file lookup returned, and +// reports whether it disposed of it. maySettle is false on the deferred second +// look, so a session can never be postponed twice. +func (n *CopySafetyNotifier) consider(ctx context.Context, session *Session, fileID int, maySettle bool) copySafetyDisposition { if session == nil || session.ID == "" { - return + return copySafetyUnresolved } record := n.attempt(ctx, session.ID) if !sessionServesFileV3(session, record, fileID) { - return + return copySafetyUnresolved } if !sessionOnVideoCopyRouteV3(session, record) { - return + return copySafetyUnresolved } if session.IsJellyfinCompat { // Stopping only helps a client whose recovery re-decides the route @@ -188,15 +214,16 @@ func (n *CopySafetyNotifier) consider(ctx context.Context, session *Session, fil "file_id", fileID, "reason", PlanInvalidatedVideoCopyUnsafe, ) - return + return copySafetyDisposed } if maySettle && !n.canTellClient(session, record) { if wait := n.settleRemaining(session); wait > 0 { n.reconsiderAfter(ctx, session.ID, fileID, wait) - return + return copySafetyDisposed } } n.invalidate(ctx, session, record, fileID) + return copySafetyDisposed } // reconsiderAfter re-examines one session once the settle window has passed. diff --git a/internal/playback/copy_safety_notifier_test.go b/internal/playback/copy_safety_notifier_test.go index 438c725de..58e66802f 100644 --- a/internal/playback/copy_safety_notifier_test.go +++ b/internal/playback/copy_safety_notifier_test.go @@ -52,16 +52,31 @@ func (c *fakeCopySafetyControl) trackedCommands() []playbackCommandNote { return append([]playbackCommandNote(nil), c.remembered...) } +// fakeAttemptLookup is read by the notifier's deferred goroutines while a test +// is still publishing attempts to it, so it is mutex-guarded and hands out +// copies rather than the stored record. type fakeAttemptLookup struct { + mu sync.Mutex records map[string]*AttemptRecordV3 } func (l *fakeAttemptLookup) GetAttempt(_ context.Context, sessionID string) (*AttemptRecordV3, error) { + l.mu.Lock() + defer l.mu.Unlock() record, ok := l.records[sessionID] if !ok { return nil, ErrSessionNotFound } - return record, nil + copied := *record + return &copied, nil +} + +// set publishes an attempt, replacing any previous one. Callers must supply a +// fresh record rather than mutate one they already handed over. +func (l *fakeAttemptLookup) set(sessionID string, record *AttemptRecordV3) { + l.mu.Lock() + defer l.mu.Unlock() + l.records[sessionID] = record } func remuxAttempt(sessionID, planID string, features ...string) *AttemptRecordV3 { @@ -305,7 +320,7 @@ func TestCopySafetyNotifierWaitsOutTheSettleWindowBeforeStopping(t *testing.T) { // The start finishes inside the window: the attempt lands and the client // connects, so the second look finds a session it can tell instead of kill. - attempts.records[session.ID] = remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3) + attempts.set(session.ID, remuxAttempt(session.ID, "plan-abc", FeaturePlanInvalidatedV3)) if err := sessions.SetRealtimeConnection(session.ID, true); err != nil { t.Fatalf("SetRealtimeConnection: %v", err) } @@ -465,6 +480,92 @@ func TestCopySafetyNotifierCompletedResultKeepsSession(t *testing.T) { } } +// Regression for the verdict landing inside a version-changing replan. Between +// applySession and CompleteReplan the live session already names the +// replacement file while the durable attempt still names the previous one, and +// the durable record wins the identity test — so the immediate pass reads the +// session as "serving another file" and does nothing. Marking it seen there +// would strand it: the sweep would skip it, and the now-persisted verdict stops +// any later scan from re-notifying, leaving it remuxing a condemned route for +// the rest of the title. +func TestCopySafetyNotifierSweepsSessionSkippedDuringAReplanCommit(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + // The live session has already been moved onto file 11 by applySession. + session, err := sessions.StartSessionWithFiles(1, "profile-1", 11, 10, PlayRemux, false) + if err != nil { + t.Fatalf("StartSessionWithFiles: %v", err) + } + + // The durable attempt has not been committed yet, so it still names the + // file the replan moved off. + stale := remuxAttempt(session.ID, "plan-old") + stale.EffectiveMediaFileID = 10 + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{session.ID: stale}} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 20 * time.Millisecond + + // The verdict for the replacement file lands mid-commit. + notifier.VideoCopyUnsafe(context.Background(), 11) + + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want nothing while the identities disagree", stopped) + } + + // CompleteReplan lands: live and durable state now agree on file 11. + committed := remuxAttempt(session.ID, "plan-new") + committed.EffectiveMediaFileID = 11 + attempts.set(session.ID, committed) + + deadline := time.Now().Add(2 * time.Second) + for { + if stopped := control.stoppedSessions(); len(stopped) == 1 && stopped[0] == session.ID { + return + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the mid-replan session swept once its identities agreed", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } +} + +// The same hole on the route test rather than the file test: mid-commit the +// durable plan still describes the transcode the replan moved off, so the +// session reads as "not on a copy route" even though it is now remuxing. +func TestCopySafetyNotifierSweepsSessionWhoseDurablePlanLagsTheRoute(t *testing.T) { + sessions, hub, tracker, control := newCopySafetyFixture(t) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + stale := remuxAttempt(session.ID, "plan-old") + stale.CurrentPlan.Delivery = DeliveryTranscodeHLSV3 + attempts := &fakeAttemptLookup{records: map[string]*AttemptRecordV3{session.ID: stale}} + notifier := NewCopySafetyNotifier(sessions, attempts, NewCommandDispatcher(sessions, hub, tracker), control) + notifier.settle = 20 * time.Millisecond + + notifier.VideoCopyUnsafe(context.Background(), 100) + + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want nothing while the durable plan still says transcode", stopped) + } + + committed := remuxAttempt(session.ID, "plan-new") + committed.CurrentPlan.Delivery = DeliveryRemuxProgressiveV3 + attempts.set(session.ID, committed) + + deadline := time.Now().Add(2 * time.Second) + for { + if stopped := control.stoppedSessions(); len(stopped) == 1 && stopped[0] == session.ID { + return + } + if time.Now().After(deadline) { + t.Fatalf("stopped = %v, want the session swept once its durable plan caught up", control.stoppedSessions()) + } + time.Sleep(time.Millisecond) + } +} + // Regression for the scan winning the race against the start path by // milliseconds: the verdict lands while the session is being built, so the // immediate pass finds nothing at all. The deferred file-wide sweep must catch diff --git a/internal/playback/recipecard.go b/internal/playback/recipecard.go index 610d8196d..ba02a78c2 100644 --- a/internal/playback/recipecard.go +++ b/internal/playback/recipecard.go @@ -1,6 +1,7 @@ package playback import ( + "strings" "time" "github.com/Silo-Server/silo-server/internal/streamtoken" @@ -161,6 +162,18 @@ func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFil } } +// VideoStreamCopy reports whether this recipe delivers the source video +// bitstream without re-encoding it: a progressive remux, or an HLS transport +// whose video target was pinned to an explicit copy. Those are exactly the +// routes an H.264 multi-PPS verdict disqualifies — a real transcode re-encodes +// the bitstream and is unaffected by conflicting in-band parameter sets. +func (c RecipeCard) VideoStreamCopy() bool { + if c.PlayMethod == PlayRemux { + return true + } + return strings.EqualFold(strings.TrimSpace(c.TargetCodecVideo), "copy") +} + // TranscodeOpts rebuilds the encode parameters for a reconstruct. outputDir, // ffmpegPath and logSink are supplied by the caller from live config because // they are environment-specific and not pinned in the card. diff --git a/internal/playback/session.go b/internal/playback/session.go index bf65138e6..fabb0333a 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -1285,6 +1285,13 @@ func (m *SessionManager) EndTransport(sessionID string) error { // do not need it: each of their requests is short, and the next one is refused // once the session is gone. // +// A session that is already gone yields an immediately-closed channel. The stop +// that removed it has run and will never run again, so a watcher registered +// after it would be closed by nobody: the caller's BeginTransport can succeed +// and the session be stopped before the registration lands, and the progressive +// remux that hole leaves behind runs to EOF serving bytes the server disowned. +// Reporting the stop it missed collapses that race into the ordinary path. +// // The channel is closed at most once: StopSession takes the whole watcher set // out of the map under the lock before closing it, and release drops a watcher // that was never signaled. @@ -1295,6 +1302,11 @@ func (m *SessionManager) WatchTransportStop(sessionID string) (<-chan struct{}, stop := make(chan struct{}) m.mu.Lock() + if _, live := m.sessions[sessionID]; !live { + m.mu.Unlock() + close(stop) + return stop, func() {} + } if m.transportStops == nil { m.transportStops = make(map[string]map[chan struct{}]struct{}) } diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go index 45bb5b232..82cf2e41d 100644 --- a/internal/playback/transcode_manager.go +++ b/internal/playback/transcode_manager.go @@ -348,15 +348,27 @@ const ( // no shared per-session store to fall back on — so a not-found session with a nil // card is a genuine miss. func (m *TranscodeManager) LoadOrReconstructSession(ctx context.Context, getSession func(string) (*Session, error), sessionID string, requestUserID int, card *RecipeCard) (*Session, SessionLoadStatus) { + session, status, _ := m.LoadOrReconstructSessionDetail(ctx, getSession, sessionID, requestUserID, card) + return session, status +} + +// LoadOrReconstructSessionDetail is LoadOrReconstructSession plus whether the +// session it returned was rebuilt from the card rather than found live. +// +// A handler needs the distinction when the card pins a route that may have been +// withdrawn since it was signed: a live session was already re-decided by +// whatever withdrew it, while a reconstruction replays the recipe verbatim and +// has to re-check it. See the copy-safety refusal on the stream serve path. +func (m *TranscodeManager) LoadOrReconstructSessionDetail(ctx context.Context, getSession func(string) (*Session, error), sessionID string, requestUserID int, card *RecipeCard) (*Session, SessionLoadStatus, bool) { session, err := getSession(sessionID) if err != nil { if !errors.Is(err, ErrSessionNotFound) { - return nil, SessionLoadFailed + return nil, SessionLoadFailed, false } // A nil manager (documented optional on StreamHandler) cannot reconstruct, // so a missing session is simply not-found rather than a panic. if m == nil || card == nil { - return nil, SessionMissing + return nil, SessionMissing, false } // Lost the in-memory session (e.g. restart): rebuild it from the token's // recipe. ReconstructSession re-binds the session to the card owner and @@ -364,9 +376,9 @@ func (m *TranscodeManager) LoadOrReconstructSession(ctx context.Context, getSess // the authless bearer routes), so a nil result here is a genuine not-found. session = m.ReconstructSession(ctx, sessionID, requestUserID, *card) if session == nil { - return nil, SessionMissing + return nil, SessionMissing, false } - return session, SessionLoaded + return session, SessionLoaded, true } // Live session: enforce the existing ownership check. A zero caller is // allowed (these routes treat the session UUID as a bearer when auth is @@ -379,12 +391,12 @@ func (m *TranscodeManager) LoadOrReconstructSession(ctx context.Context, getSess // negotiated the mode (legacy v3 and jellycompat alike) keep the bearer // behavior unchanged. if requestUserID == 0 && session.RequireMediaAuthorization { - return nil, SessionUnauthorized + return nil, SessionUnauthorized, false } if requestUserID != 0 && session.UserID != requestUserID { - return nil, SessionForbidden + return nil, SessionForbidden, false } - return session, SessionLoaded + return session, SessionLoaded, false } // ReconstructSession rebuilds the in-memory playback Session from a persisted diff --git a/internal/playback/transport_stop_test.go b/internal/playback/transport_stop_test.go index a87f56609..d72eb2657 100644 --- a/internal/playback/transport_stop_test.go +++ b/internal/playback/transport_stop_test.go @@ -167,6 +167,43 @@ func TestWatchTransportStopSignalsEveryTransport(t *testing.T) { } } +// The serving handler calls BeginTransport and WatchTransportStop as two +// separate steps. A stop landing between them used to register a watcher under +// an id nothing would ever signal again, and the progressive remux it guarded +// ran to EOF serving a route the server had already withdrawn. A watch for a +// session that is already gone reports the stop it missed. +func TestWatchTransportStopAfterStopSessionIsAlreadyClosed(t *testing.T) { + sessions := NewSessionManager(0, 0) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessions.BeginTransport(session.ID); err != nil { + t.Fatalf("BeginTransport: %v", err) + } + if err := sessions.StopSession(session.ID); err != nil { + t.Fatalf("StopSession: %v", err) + } + + stop, release := sessions.WatchTransportStop(session.ID) + select { + case <-stop: + default: + t.Fatal("a transport registered after its session was stopped was left waiting for a signal that can never come") + } + + // The release is a no-op for a watcher that was never registered, and must + // stay safe to call from the serving handler's defer. + release() + release() + sessions.mu.RLock() + remaining := len(sessions.transportStops) + sessions.mu.RUnlock() + if remaining != 0 { + t.Fatalf("transportStops holds %d sessions, want 0", remaining) + } +} + // A transport that finished normally unregisters itself, so a later stop for // the same session ID has nothing to signal and nothing to leak. func TestWatchTransportStopReleaseUnregisters(t *testing.T) { diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index 4ada4adea..491520cac 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -121,6 +121,11 @@ type copySafetyResult struct { size int64 mtime *time.Time multi bool + // persisted records whether this verdict reached the media_files row. A + // verdict memoized with persisted=false is correct for this process but + // invisible to every other replica, so a later lookup retries the write — + // the write only, never the scan. + persisted bool } // matches reports whether a memoized verdict still describes the given file. @@ -193,6 +198,7 @@ func (e *PlaybackProbeEnsurer) EnsureCopySafetyCached(ctx context.Context, file return current, nil } if multi, ok := e.knownCopySafetyVerdict(current); ok { + e.retryUnpersistedCopySafety(ctx, current) return fileWithMultiplePPS(current, multi), nil } return current, nil @@ -234,13 +240,12 @@ func (e *PlaybackProbeEnsurer) knownCopySafetyVerdict(file *models.MediaFile) (b if e == nil || file == nil { return false, false } - if cached, ok := e.copySafety.Load(file.ID); ok { - if result, ok := cached.(copySafetyResult); ok && result.matches(file) { - return result.multi, true - } + if entry, ok := e.memoizedCopySafety(file); ok { + return entry.multi, true } if multi, ok := persistedCopySafetyVerdict(file); ok { - e.storeCopySafety(file, multi) + // The row already holds it, so there is nothing left to write. + e.storeCopySafety(file, multi, true) return multi, true } return false, false @@ -289,6 +294,7 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model } if multi, ok := e.knownCopySafetyVerdict(file); ok { + e.retryUnpersistedCopySafety(ctx, file) return fileWithMultiplePPS(file, multi), nil } @@ -312,7 +318,9 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model // scanAndPersistCopySafety runs the multi-PPS bitstream scan, persists the // verdict, and memoizes it. Concurrent callers for the same file share one // scan; a failed database write is logged and the scan result is still used, -// since it is correct for this request and the next one will retry the write. +// since it is correct for this request. The memo remembers that the write did +// not land, so the next lookup for the file retries it — see +// retryUnpersistedCopySafety. func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { fileID := file.ID filePath := file.FilePath @@ -331,8 +339,12 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil return false, err } + // With no writer there is nowhere for the verdict to land, so it is not + // pending: nothing would ever clear the flag. + persisted := true if e.copySafetyRepo != nil { if writeErr := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, multi, fileSize, fileModifiedAt); writeErr != nil { + persisted = false slog.WarnContext(ctx, "persisting video copy-safety verdict failed", "component", "scanner", "file_id", fileID, @@ -340,7 +352,7 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil ) } } - e.storeCopySafety(file, multi) + e.storeCopySafety(file, multi, persisted) return multi, nil }) if err != nil { @@ -350,8 +362,62 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil return result, nil } -func (e *PlaybackProbeEnsurer) storeCopySafety(file *models.MediaFile, multi bool) { - entry := copySafetyResult{size: file.FileSize, multi: multi} +// retryUnpersistedCopySafety re-attempts the media_files write for a verdict +// this process already reached but never managed to store. No ffmpeg runs: the +// memo holds the answer, so this is a bare UPDATE. +// +// Without the retry a single failed write is lost until the process restarts. +// The verdict stays correct here, but every other replica keeps rescanning the +// same file and keeps planning fresh sessions onto the copy route it condemns. +// The write shares scanAndPersistCopySafety's singleflight key, so a burst of +// playback requests for one file cannot stampede the row, and a retry racing a +// scan simply joins it. +func (e *PlaybackProbeEnsurer) retryUnpersistedCopySafety(ctx context.Context, file *models.MediaFile) { + if e == nil || file == nil || e.copySafetyRepo == nil { + return + } + if entry, ok := e.memoizedCopySafety(file); !ok || entry.persisted { + return + } + + fileID := file.ID + _, _, _ = e.copySafetyFlight.Do(strconv.Itoa(fileID), func() (any, error) { + // Re-read inside the flight: a concurrent scan or retry may have landed + // the write while this caller queued behind it. + entry, ok := e.memoizedCopySafety(file) + if !ok || entry.persisted { + return entry.multi, nil + } + if err := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, entry.multi, entry.size, entry.mtime); err != nil { + slog.WarnContext(ctx, "retrying the video copy-safety verdict write failed", + "component", "scanner", + "file_id", fileID, + "error", err, + ) + return entry.multi, nil + } + entry.persisted = true + e.copySafety.Store(fileID, entry) + return entry.multi, nil + }) +} + +// memoizedCopySafety returns the process-cached verdict for file, but only +// while it still describes the file as it stands. +func (e *PlaybackProbeEnsurer) memoizedCopySafety(file *models.MediaFile) (copySafetyResult, bool) { + cached, ok := e.copySafety.Load(file.ID) + if !ok { + return copySafetyResult{}, false + } + entry, ok := cached.(copySafetyResult) + if !ok || !entry.matches(file) { + return copySafetyResult{}, false + } + return entry, true +} + +func (e *PlaybackProbeEnsurer) storeCopySafety(file *models.MediaFile, multi, persisted bool) { + entry := copySafetyResult{size: file.FileSize, multi: multi, persisted: persisted} if file.FileModifiedAt != nil { mtime := *file.FileModifiedAt entry.mtime = &mtime diff --git a/internal/scanner/probe_repair_copy_safety_cached_test.go b/internal/scanner/probe_repair_copy_safety_cached_test.go index 11ce3c99d..2b52bff14 100644 --- a/internal/scanner/probe_repair_copy_safety_cached_test.go +++ b/internal/scanner/probe_repair_copy_safety_cached_test.go @@ -73,7 +73,7 @@ func TestEnsureCopySafetyCachedStampsKnownVerdicts(t *testing.T) { ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} file := copySafetyTestFile(mtime) - ensurer.storeCopySafety(file, false) + ensurer.storeCopySafety(file, false, true) got, err := ensurer.EnsureCopySafetyCached(context.Background(), file) if err != nil { @@ -103,7 +103,7 @@ func TestNeedsCopySafetyScan(t *testing.T) { t.Run("known verdict needs none", func(t *testing.T) { ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath} file := copySafetyTestFile(mtime) - ensurer.storeCopySafety(file, true) + ensurer.storeCopySafety(file, true, true) if ensurer.NeedsCopySafetyScan(file) { t.Fatal("NeedsCopySafetyScan() = true, want false once the verdict is known") } diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index d9d96ac9e..017b90ea5 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -114,6 +114,14 @@ func (w *fakeCopySafetyWriter) UpdateMultiplePPS(_ context.Context, fileID int, return w.err } +// setErr changes what the next write returns, so a test can bring a failed +// backing store back up. +func (w *fakeCopySafetyWriter) setErr(err error) { + w.mu.Lock() + defer w.mu.Unlock() + w.err = err +} + func (w *fakeCopySafetyWriter) recorded() []recordedPPSWrite { w.mu.Lock() defer w.mu.Unlock() @@ -248,6 +256,58 @@ func TestEnsureCopySafetyScanSurvivesPersistFailure(t *testing.T) { } } +// A verdict whose write failed is correct in this process but invisible to +// every other replica, which keeps rescanning the file and keeps planning fresh +// sessions onto the copy route it condemns. The next lookup has to retry the +// write — and only the write: the answer is already memoized, so re-running +// ffmpeg would pay the whole bitstream read again for nothing. +func TestEnsureCopySafetyRetriesAFailedPersistWithoutRescanning(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{err: fmt.Errorf("database unavailable")} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + want := recordedPPSWrite{fileID: 42, multiplePPS: true, scanSize: 1234, scanMtime: mtime, scanMtimeSet: true} + + if _, err := ensurer.ensureCopySafety(context.Background(), copySafetyTestFile(mtime)); err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times for the first call, want 1", runs()) + } + if writes := writer.recorded(); len(writes) != 1 || writes[0] != want { + t.Fatalf("first-call writes = %+v, want exactly [%+v]", writes, want) + } + + // The database comes back. The next lookup answers from the memo and + // retries the write behind it. + writer.setErr(nil) + got, err := ensurer.ensureCopySafety(context.Background(), copySafetyTestFile(mtime)) + if err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want the retry to write only", runs()) + } + if track := got.VideoTracks[0]; track.MultiplePPS == nil || !*track.MultiplePPS { + t.Fatalf("MultiplePPS = %v, want the memoized verdict", track.MultiplePPS) + } + if writes := writer.recorded(); len(writes) != 2 || writes[1] != want { + t.Fatalf("writes after the retry = %+v, want the verdict written a second time as %+v", writes, want) + } + + // The verdict has landed, so a third lookup must not touch the row again. + if _, err := ensurer.ensureCopySafety(context.Background(), copySafetyTestFile(mtime)); err != nil { + t.Fatalf("ensureCopySafety() error = %v", err) + } + if writes := writer.recorded(); len(writes) != 2 { + t.Fatalf("UpdateMultiplePPS called %d times, want the successful write to stop the retries", len(writes)) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times overall, want 1", runs()) + } +} + // Rows predating the file_modified_at column carry no mtime. Their verdict is // still persisted and still honored on read — refusing to write it would leave // them permanently unverdicted, so every replica would rescan the same file and From aa0bf358556df5087a4233f6e0c456100c666738 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:58:14 -0400 Subject: [PATCH 41/44] fix(playback): gate copy-unsafe revivals before reconstruction and per file generation Round three review fixes: the reconstruction verdict gate moves ahead of session registration in loadTranscodeServeSession, so refused revivals cover the remote-node proxy branch and can no longer poison stream admission with a leaked session; a failed local scan re-reads the row and applies a verdict another replica persisted concurrently; and the scan singleflight is keyed by file generation (id+size+mtime) so a replaced file cannot consume the old generation's verdict. Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 23 ++- internal/api/handlers/playback.go | 10 + internal/api/handlers/playback_copy_safety.go | 23 ++- .../api/handlers/stream_copy_safety_test.go | 174 ++++++++++++++++++ internal/playback/copy_safety_race.go | 53 +++++- internal/playback/copy_safety_race_test.go | 71 ++++++- internal/scanner/probe_repair.go | 48 ++++- .../probe_repair_copy_safety_persist_test.go | 124 +++++++++++++ 8 files changed, 495 insertions(+), 31 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 1375ecefb..118aafc46 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -827,13 +827,22 @@ capability the client replays on whichever replica answers next, and a replica that dies between persisting a verdict and pushing the invalidation takes the only notifier that knew about it with it. The replacement replica has no live session, so it rebuilds one from the recipe card — which would replay the exact -remux the verdict condemned. Reconstruction therefore re-reads the persisted -verdict before it rebuilds a video stream-copy transport (a progressive remux, -or an HLS transport whose video target is `copy`) and refuses with the ordinary -playback-session not-found when the verdict says the source is unsafe. The -client's existing recovery mints a fresh attempt, which plans against the same -row and lands on a transcode. Transcode reconstruction is untouched: re-encoding -the bitstream is unaffected by conflicting parameter sets. +remux the verdict condemned. The serve routes therefore re-read the persisted +verdict for a video stream-copy recipe (a progressive remux, or an HLS transport +whose video target is `copy`) and refuse with the ordinary playback-session +not-found when the row says the source is unsafe. The client's existing recovery +mints a fresh attempt, which plans against the same row and lands on a +transcode. Transcode reconstruction is untouched: re-encoding the bitstream is +unaffected by conflicting parameter sets. + +On the HLS routes the check runs *before* the session is rebuilt, not before the +transport is. Rebuilding registers the playback session against the user's +stream caps, so a later refusal would leave a session nobody serves holding a +slot the replacement attempt needs; and an HLS recipe pinned to a transcode node +is revived by proxying to that node, a path that never reaches a local transport +rebuild at all. The progressive route decides after the load, because the same +file lookup serves its other preflight checks, and tears the reconstructed +session back down when it refuses. --- diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index b2a57c521..63e9d2ae8 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -507,6 +507,16 @@ func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID s // Genuine miss (e.g. after a restart): now — and only now — pay for the token // decode so the recipe is available for reconstruction. card, claims := verifiedStreamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) + // The copy-safety verdict gates the revival before it happens, not after. + // Reconstruction registers the playback session against the user's stream + // caps, so a refusal that ran later would leave a session nobody serves + // holding an admission slot the client's fresh attempt needs — and a + // remote-node recipe never reaches the local transport reconstruct at all + // (the serve handlers proxy to the node instead), so a gate down there would + // miss it entirely. See playback_copy_safety.go. + if videoCopyReconstructRefused(r.Context(), h.fileResolver, card) { + return nil, playback.SessionMissing, nil, nil + } session, status := h.tm.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, requestUserID, card) return session, status, card, claims } diff --git a/internal/api/handlers/playback_copy_safety.go b/internal/api/handlers/playback_copy_safety.go index d28c0bef1..90855e61f 100644 --- a/internal/api/handlers/playback_copy_safety.go +++ b/internal/api/handlers/playback_copy_safety.go @@ -22,6 +22,14 @@ import ( // the moment of reconstruction closes that hole: the row is the one piece of // state every replica shares. // +// The check belongs BEFORE the session is rebuilt, for two reasons. Rebuilding +// registers the session against the user's stream caps, so a later refusal +// leaves a session nobody serves holding a slot the client's replacement +// attempt has to admit through. And the transport is not the only thing a card +// can revive: an HLS recipe pinned to a transcode node is served by proxying to +// that node, a path that never reaches a local transport rebuild — gating there +// would let exactly the remote remux keep streaming. +// // The refusal is a plain not-found, matching an expired or missing recipe, // because that is the failure a client's recovery already knows how to handle // — it mints a fresh attempt, which plans against the persisted verdict and @@ -48,17 +56,18 @@ func videoCopyReconstructRefused(ctx context.Context, files FilePathResolver, ca } // reconstructTransportForServe rebuilds a lost local transport from the token -// recipe for the manifest and segment serve routes, refusing a video -// stream-copy the persisted copy-safety verdict has since condemned. A nil -// result is the caller's not-found, which is also what a missing card yields — -// the two are the same thing to a client: this recipe is no longer serveable. +// recipe for the manifest and segment serve routes. A nil card yields a nil +// result, which is the caller's not-found: to a client, a recipe it cannot +// present and a recipe that no longer rebuilds are the same thing. +// +// The copy-safety verdict is not consulted here. It is consulted in +// loadTranscodeServeSession, which is the only producer of the cards that reach +// this function and runs before the session is registered — see the file +// comment for why the earlier point is the correct one. func (h *PlaybackHandler) reconstructTransportForServe(ctx context.Context, sessionID string, requestedSegment int, card *playback.RecipeCard) *playback.TranscodeSession { if card == nil { return nil } - if videoCopyReconstructRefused(ctx, h.fileResolver, card) { - return nil - } return h.tm.ReconstructTranscode(ctx, sessionID, requestedSegment, *card) } diff --git a/internal/api/handlers/stream_copy_safety_test.go b/internal/api/handlers/stream_copy_safety_test.go index 71d0b0142..58505ff99 100644 --- a/internal/api/handlers/stream_copy_safety_test.go +++ b/internal/api/handlers/stream_copy_safety_test.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "errors" "net/http" "net/http/httptest" @@ -9,6 +10,8 @@ import ( "testing" "time" + "github.com/go-chi/chi/v5" + "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" @@ -144,6 +147,177 @@ func TestHandleStream_ReconstructsARemuxThatIsStillCopySafe(t *testing.T) { } } +// copySafetyHLSCard is a lost HLS transport whose video target was pinned to a +// stream copy — the remux_hls recipe. nodeURL pins it to a transcode node, +// which is what makes the serve handlers proxy rather than rebuild locally. +func copySafetyHLSCard(sessionID string, fileID int, nodeURL string) playback.RecipeCard { + return playback.RecipeCard{ + SessionID: sessionID, + UserID: 1, + ProfileID: "profile-1", + MediaFileID: fileID, + PlayMethod: playback.PlayTranscode, + TargetCodecVideo: "copy", + TargetCodecAudio: "copy", + SegmentDuration: 2, + TranscodeNodeURL: nodeURL, + TranscodeTransportID: sessionID + "-transport", + } +} + +// copySafetyHLSRequest builds one signed manifest or segment request for a +// session this replica has never seen. +func copySafetyHLSRequest(t *testing.T, secret, segmentName string, card playback.RecipeCard) *http.Request { + t.Helper() + token, err := streamtoken.Sign(card.ToClaims(), secret, playback.MaxTokenTTL) + if err != nil { + t.Fatalf("Sign: %v", err) + } + path := "/api/v1/playback/transcode/" + card.SessionID + "/master.m3u8?st=" + token + if segmentName != "" { + path = "/api/v1/playback/transcode/" + card.SessionID + "/segment/" + segmentName + "?st=" + token + } + req := httptest.NewRequest(http.MethodGet, path, nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("session_id", card.SessionID) + if segmentName != "" { + routeCtx.URLParams.Add("name", segmentName) + } + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) +} + +// The HLS serve routes revive a lost session in two different ways: locally, by +// rebuilding the ffmpeg from the card, and remotely, by proxying to the +// transcode node the card names. Both start by registering the playback session +// the card describes, so both have to be refused before that happens — a gate +// on the local transport rebuild alone would never see the remote recipe at all, +// and would leave the local one holding a stream slot it can no longer use. +func TestHandleTranscodeServe_RefusesRevivingACopyUnsafeHLSRecipe(t *testing.T) { + const secret = "test-stream-signing-secret" + + nodeHits := 0 + node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + nodeHits++ + w.WriteHeader(http.StatusOK) + })) + defer node.Close() + + for _, route := range []struct { + name string + segment string + handle func(*PlaybackHandler) func(http.ResponseWriter, *http.Request) + }{ + { + name: "manifest", + handle: func(h *PlaybackHandler) func(http.ResponseWriter, *http.Request) { return h.HandleGetTranscodeManifest }, + }, + { + name: "segment", + segment: "seg_00001.m4s", + handle: func(h *PlaybackHandler) func(http.ResponseWriter, *http.Request) { return h.HandleGetTranscodeSegment }, + }, + } { + for _, executor := range []struct { + name string + nodeURL string + }{ + {name: "local"}, + {name: "transcode-node", nodeURL: node.URL}, + } { + t.Run(route.name+"/"+executor.name, func(t *testing.T) { + unsafe := true + file := copySafetyStreamFile(t, &unsafe) + + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.JWTSecret = secret + + sessionID := "lost-hls-" + route.name + "-" + executor.name + card := copySafetyHLSCard(sessionID, file.ID, executor.nodeURL) + + rr := httptest.NewRecorder() + route.handle(handler)(rr, copySafetyHLSRequest(t, secret, route.segment, card)) + + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s; want the revival refused as not-found", rr.Code, rr.Body.String()) + } + // Nothing may be registered: the refused session would otherwise + // count against the user's stream cap with no transport behind it. + if _, err := sessionMgr.GetSession(sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetSession error = %v, want no session registered by a refused revival", err) + } + if nodeHits != 0 { + t.Fatalf("transcode node received %d proxied requests, want the condemned stream never proxied", nodeHits) + } + }) + } + } +} + +// The refusal is only worth anything if the client's recovery can get back in. +// A user at their stream cap has exactly one slot, and a session left registered +// by a refused revival would spend it on a stream nobody is serving. +func TestHandleTranscodeServe_RefusedRevivalLeavesTheStreamSlotFree(t *testing.T) { + const ( + secret = "test-stream-signing-secret" + sessionID = "lost-hls-capped" + ) + unsafe := true + file := copySafetyStreamFile(t, &unsafe) + + // One stream per user: the refused revival and the recovery attempt cannot + // both be admitted. + sessionMgr := playback.NewSessionManager(1, 1) + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.JWTSecret = secret + + rr := httptest.NewRecorder() + handler.HandleGetTranscodeManifest(rr, copySafetyHLSRequest(t, secret, "", copySafetyHLSCard(sessionID, file.ID, ""))) + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s; want the revival refused", rr.Code, rr.Body.String()) + } + + // The client's ordinary recovery: a fresh attempt, which plans against the + // persisted verdict and lands on a transcode. + recovery, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayTranscode, false) + if err != nil { + t.Fatalf("StartSession after a refused revival: %v, want the stream slot free", err) + } + if recovery == nil { + t.Fatal("StartSession returned no session after a refused revival") + } +} + +// The gate is the verdict and the route, not the reconstruct: a lost HLS +// transport that re-encodes its video is unaffected by multi-PPS and must still +// rebuild its session, whatever the row says. +func TestHandleTranscodeServe_RevivesARealTranscodeForACopyUnsafeSource(t *testing.T) { + const ( + secret = "test-stream-signing-secret" + sessionID = "lost-hls-transcode" + ) + unsafe := true + file := copySafetyStreamFile(t, &unsafe) + + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.JWTSecret = secret + + card := copySafetyHLSCard(sessionID, file.ID, "") + card.TargetCodecVideo = "h264" + + rr := httptest.NewRecorder() + handler.HandleGetTranscodeManifest(rr, copySafetyHLSRequest(t, secret, "", card)) + + // The ffmpeg behind the transport is deliberately absent in this fixture, so + // the response is still a not-found; the registered session is what proves + // the copy-safety gate let the recipe through. + if _, err := sessionMgr.GetSession(sessionID); err != nil { + t.Fatalf("GetSession error = %v, want a real transcode reconstructed despite the verdict", err) + } +} + // Only video stream-copy deliveries are gated. A transcode re-encodes the // bitstream, so conflicting parameter sets cannot reach the client's decoder // and the recipe stays serveable whatever the verdict says. diff --git a/internal/playback/copy_safety_race.go b/internal/playback/copy_safety_race.go index fa72c0d55..4dd4bd742 100644 --- a/internal/playback/copy_safety_race.go +++ b/internal/playback/copy_safety_race.go @@ -26,6 +26,12 @@ const copySafetyScanTimeout = time.Minute // defer and must still happen, goroutines are cheap, ffmpeg is not. const copySafetyScanConcurrency = 4 +// copySafetyRecheckTimeout bounds the verdict re-read that follows a failed +// scan. It cannot inherit the scan's context: a scan that failed because its +// deadline expired leaves that context already dead, which is exactly the case +// the re-read exists for. +const copySafetyRecheckTimeout = 15 * time.Second + // CopySafetyScanner is the scanner-side half of the race: it decides whether a // file still needs the H.264 multi-PPS scan and runs it. *scanner.PlaybackProbeEnsurer // implements it. @@ -147,11 +153,7 @@ func (r *CopySafetyRace) scan(fileID int) { // replica that reached it. Pushing invalidations across replicas — // Redis-backed, like the other cross-replica playback signals — is // follow-up work. - if multi, known := file.PersistedVideoCopyVerdict(); known && multi { - slog.InfoContext(ctx, "applying a persisted copy-unsafe verdict reached elsewhere", - "component", "playback", "file_id", fileID) - r.notifier.VideoCopyUnsafe(ctx, fileID) - } + r.notifyPersistedUnsafe(ctx, file) return } @@ -164,6 +166,13 @@ func (r *CopySafetyRace) scan(fileID int) { // world where the scan ran before playback started. slog.WarnContext(ctx, "video copy-safety scan failed", "component", "playback", "file_id", fileID, "error", err) + // Inconclusive here does not mean inconclusive everywhere. Another + // replica may have persisted an unsafe verdict while this scan was + // failing, and that verdict now suppresses every later local scan of the + // file (NeedsCopySafetyScan reads the row) — so the sessions this replica + // owns would wait for an unrelated future race to withdraw their route. + // Re-reading the row is the one thing that can still resolve them. + r.recheckPersistedVerdict(ctx, fileID) return } if !multi { @@ -175,6 +184,40 @@ func (r *CopySafetyRace) scan(fileID int) { r.notifier.VideoCopyUnsafe(ctx, fileID) } +// recheckPersistedVerdict re-reads the row after a local scan failed and applies +// an unsafe verdict another replica reached in the meantime. A read that fails, +// or a row with no valid verdict on it, changes nothing: inconclusive stays +// inconclusive. +func (r *CopySafetyRace) recheckPersistedVerdict(ctx context.Context, fileID int) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), copySafetyRecheckTimeout) + defer cancel() + + file, err := r.files.GetByID(ctx, fileID) + if err != nil || file == nil { + if err != nil { + slog.WarnContext(ctx, "video copy-safety verdict re-read after a failed scan could not load the file", + "component", "playback", "file_id", fileID, "error", err) + } + return + } + r.notifyPersistedUnsafe(ctx, file) +} + +// notifyPersistedUnsafe pushes the withdrawal for a row that already carries a +// valid copy-unsafe verdict. A known-safe or unverdicted row is silent, as +// always. +func (r *CopySafetyRace) notifyPersistedUnsafe(ctx context.Context, file *models.MediaFile) { + if file == nil { + return + } + if multi, known := file.PersistedVideoCopyVerdict(); !known || !multi { + return + } + slog.InfoContext(ctx, "applying a persisted copy-unsafe verdict reached elsewhere", + "component", "playback", "file_id", file.ID) + r.notifier.VideoCopyUnsafe(ctx, file.ID) +} + // RaceScanForPlan starts a race only when the plan actually stream-copies video // for this file. Callers on the playback start and replan paths use it so the // route test lives in one place. diff --git a/internal/playback/copy_safety_race_test.go b/internal/playback/copy_safety_race_test.go index 24626f9ab..d311dee4a 100644 --- a/internal/playback/copy_safety_race_test.go +++ b/internal/playback/copy_safety_race_test.go @@ -65,8 +65,11 @@ func (s *fakeCopySafetyScanner) peakConcurrency() int { } type fakeFileLoader struct { - mu sync.Mutex - file *models.MediaFile + mu sync.Mutex + file *models.MediaFile + // later, when set, is what every read after the first returns — the row as + // another replica has since rewritten it. + later *models.MediaFile err error loads int } @@ -75,6 +78,9 @@ func (l *fakeFileLoader) GetByID(context.Context, int) (*models.MediaFile, error l.mu.Lock() defer l.mu.Unlock() l.loads++ + if l.loads > 1 && l.later != nil { + return l.later, l.err + } return l.file, l.err } @@ -92,6 +98,13 @@ func raceFixture(t *testing.T, scanner *fakeCopySafetyScanner) (*CopySafetyRace, // raceFixtureForFile is raceFixture over a specific media file, for the cases // that care about what the row carries — a persisted verdict, above all. func raceFixtureForFile(t *testing.T, scanner *fakeCopySafetyScanner, file *models.MediaFile) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { + t.Helper() + return raceFixtureWithLoader(t, scanner, &fakeFileLoader{file: file}) +} + +// raceFixtureWithLoader is raceFixtureForFile over a loader the caller controls, +// for the cases that care about the row changing between two reads. +func raceFixtureWithLoader(t *testing.T, scanner *fakeCopySafetyScanner, loader *fakeFileLoader) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { t.Helper() sessions := NewSessionManager(0, 0) hub := NewRealtimeHub() @@ -102,7 +115,6 @@ func raceFixtureForFile(t *testing.T, scanner *fakeCopySafetyScanner, file *mode // These tests are about the race, not about waiting out the window a // just-started session gets before it can be stopped. notifier.settle = 0 - loader := &fakeFileLoader{file: file} return NewCopySafetyRace(scanner, loader, notifier), sessions, control } @@ -170,6 +182,59 @@ func TestCopySafetyRaceLeavesSessionsAloneOnScanError(t *testing.T) { } } +// A failed local scan and a verdict reached elsewhere can happen at once, and +// the combination is the worst of both: the persisted verdict suppresses every +// later local scan of the file, so the failure that just returned empty-handed +// is this replica's last chance to notice it. Without the re-read, the sessions +// this replica owns keep the condemned route until some unrelated future race +// happens to load the row. +func TestCopySafetyRaceAppliesAVerdictPersistedWhileTheScanFailed(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, err: errors.New("ffmpeg exploded")} + loader := &fakeFileLoader{ + file: &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + }, + later: fileWithPersistedVerdict(true), + } + race, sessions, control := raceFixtureWithLoader(t, scanner, loader) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForStop(t, control, session.ID) +} + +// The mirror image: a scan that fails over a row that still carries no verdict +// anywhere proves nothing, and the re-read must not invent one. +func TestCopySafetyRaceScanFailureWithNoVerdictAnywhereStaysSilent(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, err: errors.New("ffmpeg exploded")} + loader := &fakeFileLoader{ + file: &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + }, + later: fileWithPersistedVerdict(false), + } + race, sessions, control := raceFixtureWithLoader(t, scanner, loader) + if _, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false); err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForScans(t, scanner, 1) + time.Sleep(20 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want live sessions untouched when no verdict exists anywhere", stopped) + } +} + func TestCopySafetyRaceSkipsFilesWithNothingToScan(t *testing.T) { scanner := &fakeCopySafetyScanner{needs: false, multi: true} race, _, control := raceFixture(t, scanner) diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index 491520cac..e64ad0f99 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -315,19 +315,47 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model return fileWithMultiplePPS(file, multi), nil } +// copySafetyFlightKey identifies one generation of one file. The file ID alone +// is not enough: a rewrite in place keeps the row ID and changes the bytes, so a +// caller holding the replacement would otherwise join the flight scanning the +// old file and consume its verdict — condemning a copy-safe replacement, or +// worse, clearing the condemnation of a copy-unsafe one. Only callers that agree +// on the size and mtime are looking at the same bitstream, and therefore only +// they may share a scan. +// +// The mtime is normalized exactly as sameFileModifiedAt normalizes it, so two +// reads of the same generation that differ only in stored precision still share +// a flight. A row with no mtime gets a marker no timestamp can produce: it is a +// generation of its own, not a match for every other. +func copySafetyFlightKey(file *models.MediaFile) string { + if file == nil { + return "" + } + mtime := "none" + if file.FileModifiedAt != nil { + mtime = strconv.FormatInt(normalizeFileModifiedAt(*file.FileModifiedAt).UnixMicro(), 10) + } + return strconv.Itoa(file.ID) + ":" + strconv.FormatInt(file.FileSize, 10) + ":" + mtime +} + // scanAndPersistCopySafety runs the multi-PPS bitstream scan, persists the -// verdict, and memoizes it. Concurrent callers for the same file share one -// scan; a failed database write is logged and the scan result is still used, -// since it is correct for this request. The memo remembers that the write did -// not land, so the next lookup for the file retries it — see +// verdict, and memoizes it. Concurrent callers for the same file *generation* +// share one scan; a failed database write is logged and the scan result is still +// used, since it is correct for this request. The memo remembers that the write +// did not land, so the next lookup for the file retries it — see // retryUnpersistedCopySafety. +// +// Everything the flight closure records — the persisted row and the process memo +// — is bound to the leader's own snapshot of the file, so a joiner never writes +// another generation's facts. The key is what keeps a joiner from *reading* +// them. func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { fileID := file.ID filePath := file.FilePath fileSize := file.FileSize fileModifiedAt := file.FileModifiedAt - multi, err, _ := e.copySafetyFlight.Do(strconv.Itoa(fileID), func() (any, error) { + multi, err, _ := e.copySafetyFlight.Do(copySafetyFlightKey(file), func() (any, error) { timeout := e.timeout if timeout < 30*time.Second { timeout = 30 * time.Second @@ -369,9 +397,11 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil // Without the retry a single failed write is lost until the process restarts. // The verdict stays correct here, but every other replica keeps rescanning the // same file and keeps planning fresh sessions onto the copy route it condemns. -// The write shares scanAndPersistCopySafety's singleflight key, so a burst of -// playback requests for one file cannot stampede the row, and a retry racing a -// scan simply joins it. +// The write shares scanAndPersistCopySafety's singleflight key — the same +// generation-scoped key, so a retry never joins a flight scanning a different +// generation of the row — and so a burst of playback requests for one file +// cannot stampede the row, while a retry racing a scan of its own generation +// simply joins it. func (e *PlaybackProbeEnsurer) retryUnpersistedCopySafety(ctx context.Context, file *models.MediaFile) { if e == nil || file == nil || e.copySafetyRepo == nil { return @@ -381,7 +411,7 @@ func (e *PlaybackProbeEnsurer) retryUnpersistedCopySafety(ctx context.Context, f } fileID := file.ID - _, _, _ = e.copySafetyFlight.Do(strconv.Itoa(fileID), func() (any, error) { + _, _, _ = e.copySafetyFlight.Do(copySafetyFlightKey(file), func() (any, error) { // Re-read inside the flight: a concurrent scan or retry may have landed // the write while this caller queued behind it. entry, ok := e.memoizedCopySafety(file) diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index 017b90ea5..790902888 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -494,6 +494,130 @@ func TestEnsureCopySafetyConcurrentCallsScanOnce(t *testing.T) { } } +// A file rewritten in place keeps its row ID and changes its bytes. Two callers +// that disagree about the size and mtime are therefore looking at two different +// bitstreams, and must not share a scan: the joiner would take the leader's +// verdict for its own file — clearing the condemnation of a copy-unsafe +// replacement, or condemning a copy-safe one. +func TestCopySafetyScanDoesNotShareAFlightAcrossFileGenerations(t *testing.T) { + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + + for _, tc := range []struct { + name string + mutate func(*models.MediaFile) + }{ + { + name: "size changed", + mutate: func(f *models.MediaFile) { f.FileSize = 9999 }, + }, + { + name: "mtime changed", + mutate: func(f *models.MediaFile) { + later := mtime.Add(time.Hour) + f.FileModifiedAt = &later + }, + }, + { + name: "mtime dropped", + mutate: func(f *models.MediaFile) { f.FileModifiedAt = nil }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ffmpegPath, runs, release := fakeFFmpegGated(t, conflictingPPSAnnexB) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + replaced := copySafetyTestFile(mtime) + tc.mutate(replaced) + + var wg sync.WaitGroup + errs := make([]error, 2) + for i, file := range []*models.MediaFile{copySafetyTestFile(mtime), replaced} { + wg.Add(1) + go func(i int, file *models.MediaFile) { + defer wg.Done() + _, errs[i] = ensurer.ensureCopySafety(context.Background(), file) + }(i, file) + } + + // Both scans are held inside the stub, so "two ffmpeg processes have + // started" is only reachable if the two generations were given + // separate flights. A shared flight leaves the second caller parked + // on the first and the count stuck at one. + timeout := "" + deadline := time.Now().Add(5 * time.Second) + for runs() != 2 { + if time.Now().After(deadline) { + timeout = fmt.Sprintf("timed out with %d ffmpeg runs, want one scan per file generation", runs()) + break + } + time.Sleep(5 * time.Millisecond) + } + release() + wg.Wait() + if timeout != "" { + t.Fatal(timeout) + } + for i, err := range errs { + if err != nil { + t.Fatalf("ensureCopySafety() caller %d error = %v", i, err) + } + } + if writes := writer.recorded(); len(writes) != 2 { + t.Fatalf("UpdateMultiplePPS called %d times, want one verdict per generation: %+v", len(writes), writes) + } + }) + } +} + +// The same generation read twice — with the mtime differing only below the +// precision the row can store — is one bitstream and must still share one scan. +func TestCopySafetyScanSharesAFlightAcrossSubMicrosecondMtimeDrift(t *testing.T) { + ffmpegPath, runs, release := fakeFFmpegGated(t, conflictingPPSAnnexB) + writer := &fakeCopySafetyWriter{} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + jittered := copySafetyTestFile(mtime) + drifted := mtime.Add(17 * time.Nanosecond).Local() + jittered.FileModifiedAt = &drifted + + var wg sync.WaitGroup + var entered atomic.Int64 + errs := make([]error, 2) + for i, file := range []*models.MediaFile{copySafetyTestFile(mtime), jittered} { + wg.Add(1) + go func(i int, file *models.MediaFile) { + defer wg.Done() + entered.Add(1) + _, errs[i] = ensurer.ensureCopySafety(context.Background(), file) + }(i, file) + } + + timeout := "" + deadline := time.Now().Add(5 * time.Second) + for entered.Load() != 2 || runs() != 1 { + if time.Now().After(deadline) { + timeout = fmt.Sprintf("timed out: %d callers entered, %d ffmpeg runs", entered.Load(), runs()) + break + } + time.Sleep(5 * time.Millisecond) + } + release() + wg.Wait() + if timeout != "" { + t.Fatal(timeout) + } + for i, err := range errs { + if err != nil { + t.Fatalf("ensureCopySafety() caller %d error = %v", i, err) + } + } + if got := runs(); got != 1 { + t.Fatalf("ffmpeg ran %d times, want sub-microsecond mtime drift to share one scan", got) + } +} + func TestPersistedCopySafetyVerdict(t *testing.T) { mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) base := func() *models.MediaFile { From 120547b0cf53405e88822ccf5ec8265fd1aca801 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:53:21 -0400 Subject: [PATCH 42/44] fix(playback): re-engage the copy-safety race on revival and close generation races Round four review fixes, closed as one gap: a video stream-copy transport revived or replanned while the verdict was unknown or unpersisted never re-engaged the race machinery. KnownCopySafetyVerdict answers from memo then row (retrying an unpersisted write, never running ffmpeg); both revival paths consult it and kick the racer when nothing condemns the card; and a race request arriving mid-scan queues one follow-up pass instead of being dropped. Verdict writes are now conditional on the scanned file generation so a slow old-generation scan can neither overwrite the replacement's verdict nor notify its sessions. The web client scopes its adoption-settle wait to the load sequence that owns the session, so a hung superseded start cannot stall an invalidation past the command deadline. Test hygiene: atomic node-hit counter, observable wait instead of a sleep. Co-Authored-By: Claude Fable 5 --- docs/architecture/playback-protocol-v3.md | 27 ++ internal/api/handlers/playback.go | 15 +- internal/api/handlers/playback_copy_safety.go | 65 +++-- .../api/handlers/playback_copy_safety_test.go | 24 ++ internal/api/handlers/stream.go | 13 +- .../api/handlers/stream_copy_safety_test.go | 161 +++++++++++- internal/api/router.go | 8 + internal/playback/copy_safety_race.go | 161 ++++++++++-- internal/playback/copy_safety_race_test.go | 236 ++++++++++++++++-- internal/scanner/file_repo.go | 37 ++- .../scanner/file_repo_copy_safety_db_test.go | 142 +++++++++++ internal/scanner/probe_repair.go | 94 +++++-- .../probe_repair_copy_safety_cached_test.go | 11 +- .../probe_repair_copy_safety_persist_test.go | 73 ++++++ .../player/hooks/usePlaybackSession.test.ts | 157 ++++++++++++ web/src/player/hooks/usePlaybackSession.ts | 80 ++++-- 16 files changed, 1187 insertions(+), 117 deletions(-) create mode 100644 internal/scanner/file_repo_copy_safety_db_test.go diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 118aafc46..ea19473ed 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -844,6 +844,33 @@ rebuild at all. The progressive route decides after the load, because the same file lookup serves its other preflight checks, and tears the reconstructed session back down when it refuses. +The row alone is not the whole answer, in two directions. + +A verdict can be **known but unwritten**: the scan reached it and the +`media_files` write failed, so it lives only in the memo of the process that +reached it, and the row cannot tell that apart from "never scanned". The revival +gate therefore asks the row first and the local scanner second, and the scanner +retries the failed write — without ffmpeg — while it answers. + +A verdict can be **not yet reached at all**, which is the ordinary optimistic +case and is allowed. But the gate runs once, at the revival request, while a +progressive remux is a single response that runs for the length of the title: +nothing later re-examines it, and the replica that is racing for the verdict can +only reach its own sessions. So a revival the verdict does not condemn +*re-engages the race on the reviving replica*, which makes the session it just +built the property of a race running here. That pass costs no ffmpeg when the +answer is already known locally or on the row; it re-runs the notification for +the sessions this replica now holds. + +Two smaller rules keep that machinery honest. A race request that arrives while +a scan for the same file is running is folded into one follow-up pass rather +than dropped, because the sessions a pass acts on are the ones that exist when +it runs and a replan can commit a replacement stream-copy mid-scan. And the +verdict write is conditional on the row still holding the size and mtime that +were scanned: a file rewritten in place while the scan read it produces a +verdict about bytes nobody is serving, which is neither persisted nor pushed at +any session. + --- ## 7. Registries diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 63e9d2ae8..1c7b444a0 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -148,6 +148,15 @@ type PlaybackProbeEnsurer interface { // *playback.CopySafetyRace implements it. type PlaybackCopySafetyRacer interface { RaceScanForPlan(fileID int, plan *playback.PlanV3) + // RaceScan re-engages the race for a file whose verdict is still open. The + // serve paths use it when they revive a stream-copy transport: the replica + // that planned it may be gone, and only a race running *here* can withdraw + // the route from the session this replica just rebuilt. + RaceScan(fileID int) + // VideoCopyUnsafeKnown answers, without ffmpeg and without waiting, whether + // this replica can already condemn a video stream-copy of the file — + // including from a verdict whose write to the row failed. + VideoCopyUnsafeKnown(ctx context.Context, file *models.MediaFile) bool } type PlaybackChapterThumbnailQueuer interface { @@ -513,8 +522,10 @@ func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID s // holding an admission slot the client's fresh attempt needs — and a // remote-node recipe never reaches the local transport reconstruct at all // (the serve handlers proxy to the node instead), so a gate down there would - // miss it entirely. See playback_copy_safety.go. - if videoCopyReconstructRefused(r.Context(), h.fileResolver, card) { + // miss it entirely. A revival the verdict does not condemn re-engages the + // race here, so the session about to be rebuilt is covered by a race this + // replica owns. See playback_copy_safety.go. + if videoCopyReconstructRefused(r.Context(), h.fileResolver, h.CopySafetyRacer, card) { return nil, playback.SessionMissing, nil, nil } session, status := h.tm.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, requestUserID, card) diff --git a/internal/api/handlers/playback_copy_safety.go b/internal/api/handlers/playback_copy_safety.go index 90855e61f..78d839fcb 100644 --- a/internal/api/handlers/playback_copy_safety.go +++ b/internal/api/handlers/playback_copy_safety.go @@ -34,17 +34,60 @@ import ( // because that is the failure a client's recovery already knows how to handle // — it mints a fresh attempt, which plans against the persisted verdict and // lands on a transcode. +// +// The gate is only half the answer, because the row is only half the state. A +// verdict is reached on one replica and may not be on the row at all: the write +// can fail, or the scan may still be running elsewhere. A revival whose verdict +// is *unknown* is therefore allowed — the whole point of optimistic remuxing — +// but this replica puts itself back on the race for the file before serving it, +// so the session it just built is owned by a live race here rather than by a +// verdict on a replica that may already be gone. That race costs no ffmpeg when +// the answer is already known locally or on the row; it simply re-runs the +// notification for the sessions this replica now holds. + +// videoCopyRevivalRefused decides one revived video stream-copy: refuse it when +// this replica can already call the source copy-unsafe, and otherwise re-engage +// the race so a verdict reached later still reaches the session being revived. +// +// The persisted row is consulted first because it is the state every replica +// shares. When it says nothing, the racer is asked, because a verdict this +// process reached but failed to write lives only in its memo — and the row +// cannot tell that apart from "never scanned". Without that second question a +// failed write would leave the condemned recipe rebuildable forever. +// +// A missing racer (tests, minimal setups) collapses this to the row check plus +// no race, which is the pre-optimistic behavior. +func videoCopyRevivalRefused(ctx context.Context, racer PlaybackCopySafetyRacer, file *models.MediaFile, sessionID string) bool { + if file == nil { + return false + } + if multi, known := file.PersistedVideoCopyVerdict(); known { + if multi { + logVideoCopyRevivalRefusal(ctx, file, sessionID) + } + return multi + } + if racer == nil { + return false + } + if racer.VideoCopyUnsafeKnown(ctx, file) { + logVideoCopyRevivalRefusal(ctx, file, sessionID) + return true + } + racer.RaceScan(file.ID) + return false +} // videoCopyReconstructRefused reports whether rebuilding a lost transport from -// card must be refused because the persisted verdict now says its source cannot -// be video stream-copied. Only copy deliveries are gated; a transcode -// reconstruct is never touched. +// card must be refused because the verdict now says its source cannot be video +// stream-copied, and re-engages the race when it does not. Only copy deliveries +// are gated; a transcode reconstruct is never touched. // // An unreadable row is not evidence of anything and does not refuse: the // verdict is re-checked on every request, so a database blip costs a later // refusal rather than a spurious one. The same applies to a handler with no // file resolver wired (optional on PlaybackHandler). -func videoCopyReconstructRefused(ctx context.Context, files FilePathResolver, card *playback.RecipeCard) bool { +func videoCopyReconstructRefused(ctx context.Context, files FilePathResolver, racer PlaybackCopySafetyRacer, card *playback.RecipeCard) bool { if card == nil || files == nil || card.MediaFileID <= 0 || !card.VideoStreamCopy() { return false } @@ -52,7 +95,7 @@ func videoCopyReconstructRefused(ctx context.Context, files FilePathResolver, ca if err != nil || file == nil { return false } - return videoCopyUnsafeByVerdict(ctx, file, card.SessionID) + return videoCopyRevivalRefused(ctx, racer, file, card.SessionID) } // reconstructTransportForServe rebuilds a lost local transport from the token @@ -71,20 +114,12 @@ func (h *PlaybackHandler) reconstructTransportForServe(ctx context.Context, sess return h.tm.ReconstructTranscode(ctx, sessionID, requestedSegment, *card) } -// videoCopyUnsafeByVerdict reports whether the media_files row carries a valid -// verdict condemning a video stream-copy of this file, logging the refusal it -// is about to cause. -func videoCopyUnsafeByVerdict(ctx context.Context, file *models.MediaFile, sessionID string) bool { - multi, known := file.PersistedVideoCopyVerdict() - if !known || !multi { - return false - } - slog.InfoContext(ctx, "refusing to reconstruct a copy-unsafe video stream-copy", +func logVideoCopyRevivalRefusal(ctx context.Context, file *models.MediaFile, sessionID string) { + slog.InfoContext(ctx, "refusing to revive a copy-unsafe video stream-copy", "component", "api", "session", sessionID, "playback_session_id", sessionID, "file_id", file.ID, "reason", playback.PlanInvalidatedVideoCopyUnsafe, ) - return true } diff --git a/internal/api/handlers/playback_copy_safety_test.go b/internal/api/handlers/playback_copy_safety_test.go index 5f866a92d..f3ca3fc93 100644 --- a/internal/api/handlers/playback_copy_safety_test.go +++ b/internal/api/handlers/playback_copy_safety_test.go @@ -44,6 +44,12 @@ type recordingCopySafetyRacer struct { mu sync.Mutex plans []playback.DeliveryV3 files []int + // bare records the files handed to RaceScan — the revival path, which has + // no plan to hand over. + bare []int + // knownUnsafe stands in for a verdict this replica holds without the row + // carrying it: an unsafe scan whose write to media_files failed. + knownUnsafe bool } func (r *recordingCopySafetyRacer) RaceScanForPlan(fileID int, plan *playback.PlanV3) { @@ -55,12 +61,30 @@ func (r *recordingCopySafetyRacer) RaceScanForPlan(fileID int, plan *playback.Pl } } +func (r *recordingCopySafetyRacer) RaceScan(fileID int) { + r.mu.Lock() + defer r.mu.Unlock() + r.bare = append(r.bare, fileID) +} + +func (r *recordingCopySafetyRacer) VideoCopyUnsafeKnown(context.Context, *models.MediaFile) bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.knownUnsafe +} + func (r *recordingCopySafetyRacer) raced() ([]int, []playback.DeliveryV3) { r.mu.Lock() defer r.mu.Unlock() return append([]int(nil), r.files...), append([]playback.DeliveryV3(nil), r.plans...) } +func (r *recordingCopySafetyRacer) bareRaces() []int { + r.mu.Lock() + defer r.mu.Unlock() + return append([]int(nil), r.bare...) +} + // Starting playback must never wait on the H.264 copy-safety scan: it takes the // cached-only ensure, and the plan it issues is handed to the racer that // resolves the verdict behind it. diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index 8fb208ea1..7ec51d899 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -51,6 +51,12 @@ type StreamHandler struct { // PlaybackConfig returns the current playback config; read it through // ffmpegPath(). May be nil (tests). PlaybackConfig func() config.PlaybackConfig + // CopySafetyRacer gates and covers a revived progressive remux: it answers + // whether this replica already condemns a video stream-copy of the source, + // and re-engages the copy-safety race for one whose verdict is still open. + // Optional — without it a revived remux is gated on the persisted row alone + // and no race is started here. + CopySafetyRacer PlaybackCopySafetyRacer // SubtitleCache stores full-track PGS (.sup) extracts under the transcode // dir so repeat selections skip the whole-file ffmpeg demux. May be nil // (tests / minimal setups) — extraction then always streams uncached. @@ -151,8 +157,13 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { // A reconstructed remux replays a recipe committed before the copy-safety // verdict existed, and no notifier can reach it — see playback_copy_safety.go. + // One that the verdict does not condemn re-engages the race here, which is + // what gives the single long response below something able to withdraw it. + // Starting that race before the abort watcher is registered is safe: a + // verdict fast enough to stop the session first leaves WatchTransportStop + // with no session to watch, and it reports the stop it missed. if reconstructed && session.PlayMethod == playback.PlayRemux && - videoCopyUnsafeByVerdict(r.Context(), file, sessionID) { + videoCopyRevivalRefused(r.Context(), h.CopySafetyRacer, file, sessionID) { // The reconstruct already registered the session; tear it down again so // the refusal leaves no half-live session behind the client's replan. h.abortPlaybackSession(r.Context(), session) diff --git a/internal/api/handlers/stream_copy_safety_test.go b/internal/api/handlers/stream_copy_safety_test.go index 58505ff99..255f0c112 100644 --- a/internal/api/handlers/stream_copy_safety_test.go +++ b/internal/api/handlers/stream_copy_safety_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync/atomic" "testing" "time" @@ -147,6 +148,102 @@ func TestHandleStream_ReconstructsARemuxThatIsStillCopySafe(t *testing.T) { } } +// Refusing a revival is only possible where a verdict exists. The dangerous +// case is the one where it does not yet: replica A is still scanning (or has +// scanned and failed to write the row) when replica B revives the transport +// from the card. B's response is a single progressive remux that runs for the +// length of the title, and the gate it just passed never runs again — so unless +// B puts itself on the race for the file, no withdrawal can ever reach it, and +// A's notifier cannot: it only reaches A's own sessions. +func TestHandleStream_RevivedRemuxWithNoVerdictReEngagesTheRace(t *testing.T) { + const ( + secret = "test-stream-signing-secret" + sessionID = "lost-remux-unverdicted" + ) + file := copySafetyStreamFile(t, nil) + + sessionMgr := playback.NewSessionManager(0, 0) + tm := playback.NewTranscodeManager() + tm.Sessions = sessionMgr + + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg") + if err := os.WriteFile(ffmpeg, []byte("#!/bin/sh\nprintf muxed\n"), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + racer := &recordingCopySafetyRacer{} + handler := NewStreamHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.TM = tm + handler.JWTSecret = secret + handler.CopySafetyRacer = racer + handler.PlaybackConfig = func() config.PlaybackConfig { + return config.PlaybackConfig{FFmpegPath: ffmpeg} + } + + card := playback.NewRemuxRecipeCard(sessionID, 1, "profile-1", file.ID, false, 0) + card.InputPath = file.FilePath + token, err := streamtoken.Sign(card.ToClaims(), secret, playback.MaxTokenTTL) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+sessionID+"?st="+token, nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleStream(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s; want an undecided remux served optimistically", rr.Code, rr.Body.String()) + } + if got := racer.bareRaces(); len(got) != 1 || got[0] != file.ID { + t.Fatalf("raced files = %v, want the revived file %d raced on this replica", got, file.ID) + } +} + +// A verdict whose write to media_files failed lives only in the memo of the +// process that reached it. The row cannot tell that apart from "never scanned", +// so a revival gated on the row alone would rebuild the condemned remux for as +// long as the client keeps retrying its stream URL — and nothing would +// re-invoke the ensurer that holds the answer. Asking the racer closes it. +func TestHandleStream_RefusesARevivalTheRowDoesNotKnowIsUnsafe(t *testing.T) { + const ( + secret = "test-stream-signing-secret" + sessionID = "lost-remux-unpersisted" + ) + file := copySafetyStreamFile(t, nil) + + sessionMgr := playback.NewSessionManager(0, 0) + tm := playback.NewTranscodeManager() + tm.Sessions = sessionMgr + + handler := NewStreamHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.TM = tm + handler.JWTSecret = secret + handler.CopySafetyRacer = &recordingCopySafetyRacer{knownUnsafe: true} + + card := playback.NewRemuxRecipeCard(sessionID, 1, "profile-1", file.ID, false, 0) + card.InputPath = file.FilePath + token, err := streamtoken.Sign(card.ToClaims(), secret, playback.MaxTokenTTL) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+sessionID+"?st="+token, nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleStream(rr, req) + + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s; want the revival refused on the unpersisted verdict", rr.Code, rr.Body.String()) + } + if _, err := sessionMgr.GetSession(sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetSession error = %v, want the refused revival torn down", err) + } +} + // copySafetyHLSCard is a lost HLS transport whose video target was pinned to a // stream copy — the remux_hls recipe. nodeURL pins it to a transcode node, // which is what makes the serve handlers proxy rather than rebuild locally. @@ -196,9 +293,11 @@ func copySafetyHLSRequest(t *testing.T, secret, segmentName string, card playbac func TestHandleTranscodeServe_RefusesRevivingACopyUnsafeHLSRecipe(t *testing.T) { const secret = "test-stream-signing-secret" - nodeHits := 0 + // The httptest handler runs on its own goroutine; the assertions read this + // from the test's. + var nodeHits atomic.Int64 node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - nodeHits++ + nodeHits.Add(1) w.WriteHeader(http.StatusOK) })) defer node.Close() @@ -247,8 +346,8 @@ func TestHandleTranscodeServe_RefusesRevivingACopyUnsafeHLSRecipe(t *testing.T) if _, err := sessionMgr.GetSession(sessionID); !errors.Is(err, playback.ErrSessionNotFound) { t.Fatalf("GetSession error = %v, want no session registered by a refused revival", err) } - if nodeHits != 0 { - t.Fatalf("transcode node received %d proxied requests, want the condemned stream never proxied", nodeHits) + if got := nodeHits.Load(); got != 0 { + t.Fatalf("transcode node received %d proxied requests, want the condemned stream never proxied", got) } }) } @@ -318,6 +417,58 @@ func TestHandleTranscodeServe_RevivesARealTranscodeForACopyUnsafeSource(t *testi } } +// The HLS revival path needs the same two halves as the progressive one: an +// undecided recipe is revived and raced here, and one this replica already +// knows is unsafe — from a verdict whose write never landed — is refused even +// though the row says nothing. +func TestHandleTranscodeServe_UndecidedRevivalIsRacedAndKnownUnsafeIsRefused(t *testing.T) { + const secret = "test-stream-signing-secret" + + t.Run("undecided revival re-engages the race", func(t *testing.T) { + const sessionID = "lost-hls-unverdicted" + file := copySafetyStreamFile(t, nil) + + sessionMgr := playback.NewSessionManager(0, 0) + racer := &recordingCopySafetyRacer{} + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.JWTSecret = secret + handler.CopySafetyRacer = racer + + rr := httptest.NewRecorder() + handler.HandleGetTranscodeManifest(rr, copySafetyHLSRequest(t, secret, "", copySafetyHLSCard(sessionID, file.ID, ""))) + + // The ffmpeg behind the transport is absent in this fixture, so the + // response is a not-found either way; the registered session is what + // proves the gate let the undecided recipe through. + if _, err := sessionMgr.GetSession(sessionID); err != nil { + t.Fatalf("GetSession error = %v, want an undecided recipe revived optimistically", err) + } + if got := racer.bareRaces(); len(got) != 1 || got[0] != file.ID { + t.Fatalf("raced files = %v, want the revived file %d raced on this replica", got, file.ID) + } + }) + + t.Run("verdict known only in memory still refuses", func(t *testing.T) { + const sessionID = "lost-hls-unpersisted" + file := copySafetyStreamFile(t, nil) + + sessionMgr := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.JWTSecret = secret + handler.CopySafetyRacer = &recordingCopySafetyRacer{knownUnsafe: true} + + rr := httptest.NewRecorder() + handler.HandleGetTranscodeManifest(rr, copySafetyHLSRequest(t, secret, "", copySafetyHLSCard(sessionID, file.ID, ""))) + + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s; want the revival refused", rr.Code, rr.Body.String()) + } + if _, err := sessionMgr.GetSession(sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetSession error = %v, want no session registered by a refused revival", err) + } + }) +} + // Only video stream-copy deliveries are gated. A transcode re-encodes the // bitstream, so conflicting parameter sets cannot reach the client's decoder // and the recipe stays serveable whatever the verdict says. @@ -343,7 +494,7 @@ func TestVideoCopyReconstructRefusedOnlyGatesCopyDeliveries(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { card := tc.card - if got := videoCopyReconstructRefused(t.Context(), files, &card); got != tc.want { + if got := videoCopyReconstructRefused(t.Context(), files, nil, &card); got != tc.want { t.Fatalf("videoCopyReconstructRefused() = %v, want %v", got, tc.want) } }) diff --git a/internal/api/router.go b/internal/api/router.go index cab28d88e..ecc558257 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1070,6 +1070,14 @@ func NewRouter(deps Dependencies) chi.Router { if detailSvc != nil { detailSvc.SetCopySafetyRacer(copySafetyRace) } + if streamHandler != nil { + // The progressive remux serve path revives stream-copy + // transports of its own, and its single long response is + // the one thing no later request can gate. It needs the + // same racer to refuse a condemned revival and to cover an + // undecided one. + streamHandler.CopySafetyRacer = copySafetyRace + } } } // A resolver lets subtitle realtime events carry the combined ordinal diff --git a/internal/playback/copy_safety_race.go b/internal/playback/copy_safety_race.go index 4dd4bd742..b37e1b5ec 100644 --- a/internal/playback/copy_safety_race.go +++ b/internal/playback/copy_safety_race.go @@ -33,11 +33,25 @@ const copySafetyScanConcurrency = 4 const copySafetyRecheckTimeout = 15 * time.Second // CopySafetyScanner is the scanner-side half of the race: it decides whether a -// file still needs the H.264 multi-PPS scan and runs it. *scanner.PlaybackProbeEnsurer +// file still needs the H.264 multi-PPS scan, runs it, and reports what this +// process already knows without scanning. *scanner.PlaybackProbeEnsurer // implements it. type CopySafetyScanner interface { NeedsCopySafetyScan(file *models.MediaFile) bool - ScanCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) + // ScanCopySafety resolves an unknown verdict. Its second result reports + // that the verdict was superseded — computed from a generation of the file + // the row no longer holds — which is neither an error nor something any + // caller may act on. + ScanCopySafety(ctx context.Context, file *models.MediaFile) (multi bool, stale bool, err error) + // KnownCopySafetyVerdict answers from the process memo or the persisted + // row, never from ffmpeg, and retries a write that never landed. + // + // It is required rather than probed for: an unsafe verdict whose write + // failed lives only in the scanning process's memo, and the row — the only + // other place to look — cannot distinguish that from "never scanned". A + // racer that could not ask this question would leave the sessions on the + // condemned route with nothing left to withdraw them. + KnownCopySafetyVerdict(ctx context.Context, file *models.MediaFile) (multi bool, known bool) } // CopySafetyFileLoader loads the media file a race was requested for. @@ -62,7 +76,7 @@ type CopySafetyRace struct { // already collapses concurrent scans, but every start, replan and watch-page // load for a popular file would otherwise stack a goroutine that does // nothing but wait on it. - inFlight sync.Map // file ID -> struct{} + inFlight sync.Map // file ID -> *copySafetyRaceState // slots is the replica-wide scan semaphore. A goroutine holds its per-file // inFlight entry while it waits for a slot, so queueing never lets a second // goroutine for the same file through. @@ -85,9 +99,36 @@ func NewCopySafetyRace(scanner CopySafetyScanner, files CopySafetyFileLoader, no } } +// copySafetyRaceState is the per-file entry in inFlight. It exists so a request +// that arrives while a scan is running is remembered rather than dropped. +// +// Dropping it was wrong in one specific, reachable way: the sessions a pass +// notifies are the ones that exist when it runs, and the notifier excludes the +// sessions it disposed of from its own late sweep. A replan that commits a +// replacement stream-copy while the scan is in flight therefore produces a +// session no pass will ever look at — its race request was swallowed by the +// dedupe, and the plan it is running was never considered by the pass that +// preceded it. +type copySafetyRaceState struct { + mu sync.Mutex + // recheck records that somebody asked for this file while the owner was + // mid-pass, so the owner owes one more pass before it lets go. + recheck bool + // done marks the owner as retired and its map entry as already removed; + // a request that sees it must store a fresh state instead. + done bool +} + // RaceScan resolves the copy-safety verdict for fileID in the background. It -// returns immediately, and does nothing when the verdict is already known, the -// file is not H.264, or a scan for the file is already running. +// returns immediately, and does nothing when the verdict is already known or +// the file is not H.264. +// +// A request that lands while a scan for the same file is running does not start +// a second scan — and is not dropped either: it is folded into one follow-up +// pass the running goroutine makes before it retires. That pass normally costs +// no ffmpeg at all (the verdict it would scan for is by then memoized or +// persisted, so it takes the known-verdict path) and exists to re-examine the +// sessions that appeared while the scan ran. // // The caller's request context is deliberately not used: the scan outlives the // request that noticed the verdict was missing, and the whole point is that no @@ -96,17 +137,53 @@ func (r *CopySafetyRace) RaceScan(fileID int) { if r == nil || fileID <= 0 { return } - if _, running := r.inFlight.LoadOrStore(fileID, struct{}{}); running { - return + for { + actual, running := r.inFlight.LoadOrStore(fileID, ©SafetyRaceState{}) + state, _ := actual.(*copySafetyRaceState) + if state == nil { + return + } + if !running { + go r.runRace(fileID, state) + return + } + state.mu.Lock() + if !state.done { + state.recheck = true + state.mu.Unlock() + return + } + state.mu.Unlock() + // The owner retired between the load and the lock. It deletes its map + // entry before marking itself done, both under this lock, so observing + // done proves the entry is gone and the next LoadOrStore installs a + // fresh state: this loop runs at most twice. } - go func() { - defer r.inFlight.Delete(fileID) - // The slot is taken before the scan's own deadline starts: time spent - // queueing behind other files is not time the scan was given to run. - r.acquireSlot() - defer r.releaseSlot() +} + +// runRace owns one file's races until nothing is left asking for it. +func (r *CopySafetyRace) runRace(fileID int, state *copySafetyRaceState) { + // The slot is taken before the scan's own deadline starts: time spent + // queueing behind other files is not time the scan was given to run. + r.acquireSlot() + defer r.releaseSlot() + for { r.scan(fileID) - }() + + state.mu.Lock() + if state.recheck { + state.recheck = false + state.mu.Unlock() + continue + } + // Retire under the lock, deleting first: a request that already holds + // this state can then tell, from done alone, that it has to install a + // new one. + r.inFlight.Delete(fileID) + state.done = true + state.mu.Unlock() + return + } } func (r *CopySafetyRace) acquireSlot() { @@ -143,9 +220,12 @@ func (r *CopySafetyRace) scan(fileID int) { // Nothing left to scan, but that is not the same as nothing to do. The // verdict may have been reached by another replica between this race // being requested and the file being loaded: that replica notified its - // own sessions and has no way to reach ours, so a persisted unsafe - // verdict has to be applied locally even though no scan runs here. A - // known-safe verdict is silent, as always. + // own sessions and has no way to reach ours, so an unsafe verdict has to + // be applied locally even though no scan runs here. It may equally have + // been reached by *this* process and failed to persist, which is why the + // question goes to the scanner (memo first, then row) rather than to the + // row alone — the row cannot tell an unpersisted verdict from an unknown + // one. A known-safe verdict is silent, as always. // // This closes the window for sessions this replica raced against another // replica's write. It is not distributed invalidation: a verdict that @@ -153,11 +233,11 @@ func (r *CopySafetyRace) scan(fileID int) { // replica that reached it. Pushing invalidations across replicas — // Redis-backed, like the other cross-replica playback signals — is // follow-up work. - r.notifyPersistedUnsafe(ctx, file) + r.notifyKnownUnsafe(ctx, file) return } - multi, err := r.scanner.ScanCopySafety(ctx, file) + multi, stale, err := r.scanner.ScanCopySafety(ctx, file) if err != nil { // An inconclusive scan is not evidence of anything. Nothing is persisted // (the scanner only records a verdict it reached), live sessions keep @@ -175,6 +255,18 @@ func (r *CopySafetyRace) scan(fileID int) { r.recheckPersistedVerdict(ctx, fileID) return } + if stale { + // The scan read a generation of the file the row has since moved past — + // it was rewritten in place while ffmpeg was working. The verdict is + // about bytes nobody is serving, so it neither persists nor withdraws + // anything: notifying on it would tear down sessions playing the + // replacement over evidence from the file it replaced. The replacement's + // own verdict is unknown again, and the next start, replan or revival + // for it asks for a fresh race. + slog.InfoContext(ctx, "discarding a video copy-safety verdict for a superseded generation of the file", + "component", "playback", "file_id", fileID) + return + } if !multi { return } @@ -200,24 +292,39 @@ func (r *CopySafetyRace) recheckPersistedVerdict(ctx context.Context, fileID int } return } - r.notifyPersistedUnsafe(ctx, file) + r.notifyKnownUnsafe(ctx, file) } -// notifyPersistedUnsafe pushes the withdrawal for a row that already carries a -// valid copy-unsafe verdict. A known-safe or unverdicted row is silent, as -// always. -func (r *CopySafetyRace) notifyPersistedUnsafe(ctx context.Context, file *models.MediaFile) { - if file == nil { +// notifyKnownUnsafe pushes the withdrawal for a file this replica can already +// call copy-unsafe without scanning — from the persisted row, or from a verdict +// this process reached whose write never landed. A known-safe or unresolved +// file is silent, as always. +func (r *CopySafetyRace) notifyKnownUnsafe(ctx context.Context, file *models.MediaFile) { + if r == nil || file == nil { return } - if multi, known := file.PersistedVideoCopyVerdict(); !known || !multi { + multi, known := r.scanner.KnownCopySafetyVerdict(ctx, file) + if !known || !multi { return } - slog.InfoContext(ctx, "applying a persisted copy-unsafe verdict reached elsewhere", + slog.InfoContext(ctx, "applying a copy-unsafe verdict that needed no scan", "component", "playback", "file_id", file.ID) r.notifier.VideoCopyUnsafe(ctx, file.ID) } +// VideoCopyUnsafeKnown reports whether this replica can already say the file +// cannot be video stream-copied, without running ffmpeg and without waiting on +// anything. It is what the serve paths gate a revived stream-copy on: the +// persisted row alone would miss a verdict this process reached but failed to +// write, which is exactly the case where nothing else is left to catch it. +func (r *CopySafetyRace) VideoCopyUnsafeKnown(ctx context.Context, file *models.MediaFile) bool { + if r == nil || file == nil { + return false + } + multi, known := r.scanner.KnownCopySafetyVerdict(ctx, file) + return known && multi +} + // RaceScanForPlan starts a race only when the plan actually stream-copies video // for this file. Callers on the playback start and replan paths use it so the // route test lives in one place. diff --git a/internal/playback/copy_safety_race_test.go b/internal/playback/copy_safety_race_test.go index d311dee4a..f04b16fd3 100644 --- a/internal/playback/copy_safety_race_test.go +++ b/internal/playback/copy_safety_race_test.go @@ -11,15 +11,23 @@ import ( ) type fakeCopySafetyScanner struct { - mu sync.Mutex - needs bool - multi bool - err error - scans int - active int - maxActive int - release chan struct{} - scanning chan struct{} + mu sync.Mutex + needs bool + multi bool + // stale makes the scan report a verdict for a generation of the file the + // row has moved past. + stale bool + err error + // known and knownMulti stand in for the process memo: a verdict this + // replica reached whose write to media_files never landed, and which is + // therefore invisible on the row. + known bool + knownMulti bool + scans int + active int + maxActive int + release chan struct{} + scanning chan struct{} } func (s *fakeCopySafetyScanner) NeedsCopySafetyScan(*models.MediaFile) bool { @@ -28,7 +36,7 @@ func (s *fakeCopySafetyScanner) NeedsCopySafetyScan(*models.MediaFile) bool { return s.needs } -func (s *fakeCopySafetyScanner) ScanCopySafety(context.Context, *models.MediaFile) (bool, error) { +func (s *fakeCopySafetyScanner) ScanCopySafety(context.Context, *models.MediaFile) (bool, bool, error) { s.mu.Lock() s.scans++ s.active++ @@ -47,7 +55,22 @@ func (s *fakeCopySafetyScanner) ScanCopySafety(context.Context, *models.MediaFil if s.release != nil { <-s.release } - return s.multi, s.err + return s.multi, s.stale, s.err +} + +// KnownCopySafetyVerdict mirrors the real ensurer: the process memo first, then +// the verdict the row carries. +func (s *fakeCopySafetyScanner) KnownCopySafetyVerdict(_ context.Context, file *models.MediaFile) (bool, bool) { + s.mu.Lock() + known, multi := s.known, s.knownMulti + s.mu.Unlock() + if known { + return multi, true + } + if file == nil { + return false, false + } + return file.PersistedVideoCopyVerdict() } func (s *fakeCopySafetyScanner) scanCount() int { @@ -84,6 +107,12 @@ func (l *fakeFileLoader) GetByID(context.Context, int) (*models.MediaFile, error return l.file, l.err } +func (l *fakeFileLoader) loadCount() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.loads +} + // raceFixture wires a racer whose notifier reports into a fake control, so a // multi-PPS verdict is observable as a session stop. func raceFixture(t *testing.T, scanner *fakeCopySafetyScanner) (*CopySafetyRace, *SessionManager, *fakeCopySafetyControl) { @@ -228,8 +257,11 @@ func TestCopySafetyRaceScanFailureWithNoVerdictAnywhereStaysSilent(t *testing.T) race.RaceScan(100) - waitForScans(t, scanner, 1) - time.Sleep(20 * time.Millisecond) + // The re-read after the failed scan is the last thing the pass does, and it + // is the only thing that could have produced a stop. Waiting for it is what + // makes "nothing was stopped" an assertion about a finished pass rather + // than about a moment in time. + waitForLoads(t, loader, 2) if stopped := control.stoppedSessions(); len(stopped) != 0 { t.Fatalf("stopped = %v, want live sessions untouched when no verdict exists anywhere", stopped) } @@ -251,14 +283,20 @@ func TestCopySafetyRaceSkipsFilesWithNothingToScan(t *testing.T) { } // Every start, replan and watch-page load for a popular file asks for the same -// race; only one goroutine may be in flight for it. +// race; only one goroutine may be in flight for it, and a burst arriving while +// it runs collapses into a single follow-up pass rather than one scan each. func TestCopySafetyRaceDedupesInFlightScans(t *testing.T) { scanner := &fakeCopySafetyScanner{ needs: true, release: make(chan struct{}), - scanning: make(chan struct{}, 1), + scanning: make(chan struct{}, 8), } - race, _, _ := raceFixture(t, scanner) + loader := &fakeFileLoader{file: &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + }} + race, _, _ := raceFixtureWithLoader(t, scanner, loader) race.RaceScan(100) <-scanner.scanning @@ -267,9 +305,152 @@ func TestCopySafetyRaceDedupesInFlightScans(t *testing.T) { } close(scanner.release) - time.Sleep(50 * time.Millisecond) - if got := scanner.scanCount(); got != 1 { - t.Fatalf("scans = %d, want 1 while a scan for the file is already running", got) + // One follow-up pass for the whole burst: two row reads, and never five. + waitForLoads(t, loader, 2) + if got := scanner.scanCount(); got > 2 { + t.Fatalf("scans = %d, want at most 2 — the in-flight scan and one follow-up for the burst", got) + } +} + +// The dedupe used to drop a request that arrived mid-scan, and that dropped +// request was sometimes the only thing that would have looked at a session. +// A replan can commit a replacement stream-copy while the scan runs: the pass +// already under way lists the sessions as they were, and the notifier excludes +// the ones it disposed of from its own late sweep, so the replacement is +// considered by nobody. The follow-up pass is what re-lists them. +func TestCopySafetyRaceReconsidersSessionsThatAppearedDuringAScan(t *testing.T) { + scanner := &fakeCopySafetyScanner{ + needs: true, + multi: true, + release: make(chan struct{}), + scanning: make(chan struct{}, 1), + } + race, sessions, control := raceFixture(t, scanner) + + race.RaceScan(100) + waitForScanning(t, scanner, "the first pass") + // Asked for while the scan is running: under the old dedupe this request + // was dropped on the floor. + race.RaceScan(100) + releaseScan(t, scanner, "the first pass") + + // The second scan starting proves the first pass — its scan, its verdict + // and its notification — is over. No session existed for it to touch, so + // the follow-up pass is the only thing left that can reach one. + waitForScanning(t, scanner, "the follow-up pass") + late, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + releaseScan(t, scanner, "the follow-up pass") + + waitForStop(t, control, late.ID) +} + +// waitForScanning blocks until a pass is inside its scan, failing rather than +// hanging when the pass never comes. +func waitForScanning(t *testing.T, scanner *fakeCopySafetyScanner, pass string) { + t.Helper() + select { + case <-scanner.scanning: + case <-time.After(2 * time.Second): + t.Fatalf("%s never started scanning", pass) + } +} + +// releaseScan lets a waiting scan return, failing rather than hanging when +// nothing is waiting for it. +func releaseScan(t *testing.T, scanner *fakeCopySafetyScanner, pass string) { + t.Helper() + select { + case scanner.release <- struct{}{}: + case <-time.After(2 * time.Second): + t.Fatalf("nothing was waiting to be released for %s", pass) + } +} + +// A file rewritten in place while the scan reads it produces a verdict about +// bytes the server is no longer serving. Persisting it would re-validate a dead +// generation, and notifying on it would tear down sessions playing the +// replacement over evidence from the file it replaced. +func TestCopySafetyRaceIgnoresAVerdictForASupersededGeneration(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: true, multi: true, stale: true} + race, sessions, control := raceFixture(t, scanner) + if _, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false); err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForScans(t, scanner, 1) + time.Sleep(20 * time.Millisecond) + if stopped := control.stoppedSessions(); len(stopped) != 0 { + t.Fatalf("stopped = %v, want no session withdrawn on a verdict for bytes it is not playing", stopped) + } +} + +// A verdict this process reached and failed to write is invisible on the row, +// and the row is all a later pass would otherwise consult. The scanner is the +// only thing that can still answer, so the pass with nothing left to scan asks +// it rather than the row. +func TestCopySafetyRaceAppliesAnUnpersistedUnsafeVerdict(t *testing.T) { + scanner := &fakeCopySafetyScanner{needs: false, known: true, knownMulti: true} + race, sessions, control := raceFixture(t, scanner) + session, err := sessions.StartSession(1, "profile-1", 100, PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + race.RaceScan(100) + + waitForStop(t, control, session.ID) + if got := scanner.scanCount(); got != 0 { + t.Fatalf("scans = %d, want 0 for a verdict this process already holds", got) + } +} + +// VideoCopyUnsafeKnown is what the serve paths gate a revived stream-copy on, +// so it has to answer from the same place the racer does — including the memo +// the row knows nothing about. +func TestCopySafetyRaceVideoCopyUnsafeKnown(t *testing.T) { + unverdicted := &models.MediaFile{ + ID: 100, + CodecVideo: "h264", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + } + + for _, tc := range []struct { + name string + scanner *fakeCopySafetyScanner + file *models.MediaFile + want bool + }{ + {name: "unknown", scanner: &fakeCopySafetyScanner{}, file: unverdicted}, + {name: "persisted unsafe", scanner: &fakeCopySafetyScanner{}, file: fileWithPersistedVerdict(true), want: true}, + {name: "persisted safe", scanner: &fakeCopySafetyScanner{}, file: fileWithPersistedVerdict(false)}, + { + name: "unpersisted unsafe", + scanner: &fakeCopySafetyScanner{known: true, knownMulti: true}, + file: unverdicted, + want: true, + }, + { + name: "unpersisted safe", + scanner: &fakeCopySafetyScanner{known: true}, + file: unverdicted, + }, + } { + t.Run(tc.name, func(t *testing.T) { + race, _, _ := raceFixtureForFile(t, tc.scanner, tc.file) + if got := race.VideoCopyUnsafeKnown(t.Context(), tc.file); got != tc.want { + t.Fatalf("VideoCopyUnsafeKnown() = %v, want %v", got, tc.want) + } + }) + } + + var nilRace *CopySafetyRace + if nilRace.VideoCopyUnsafeKnown(t.Context(), unverdicted) { + t.Fatal("VideoCopyUnsafeKnown() on a nil racer = true, want false") } } @@ -392,6 +573,23 @@ func TestCopySafetyRaceNilIsSafe(t *testing.T) { } } +// waitForLoads waits for the racer to have read the row want times. Each pass +// opens with exactly one read and a failed scan adds its re-read, so the count +// is the observable "this much of the pass has happened". +func waitForLoads(t *testing.T, loader *fakeFileLoader, want int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if loader.loadCount() >= want { + return + } + if time.Now().After(deadline) { + t.Fatalf("file loads = %d, want %d", loader.loadCount(), want) + } + time.Sleep(time.Millisecond) + } +} + func waitForScans(t *testing.T, scanner *fakeCopySafetyScanner, want int) { t.Helper() deadline := time.Now().Add(2 * time.Second) diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 04576f229..30acb627c 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -20,6 +20,12 @@ import ( // Sentinel errors for file repository operations. var ( ErrFileNotFound = errors.New("media file not found") + // ErrStaleCopySafetyScan reports that a multi-PPS verdict was computed from + // a generation of the file the row no longer holds — it was rewritten (or + // removed) while the scan ran. The verdict is not wrong, it just describes + // bytes nobody is serving any more, so it must not overwrite the row and + // must not be pushed at live sessions. + ErrStaleCopySafetyScan = errors.New("copy-safety verdict superseded by a newer generation of the file") ) // FileRepository provides CRUD operations for the media_files table. @@ -1188,24 +1194,49 @@ func (r *FileRepository) SetChapterThumbnailFailure( // It deliberately does not go through Upsert: that path also clears // match_suppressed_at and missing_since, which a copy-safety scan has no // business touching. +// +// The write is conditional on the row still holding the generation that was +// scanned. A scan reads the opening seconds of a file over storage that can be +// slow, so an old-generation scan finishing late would otherwise stamp its +// verdict — and its stale size and mtime — over the replacement generation's, +// re-validating a verdict for bytes that are gone and condemning (or clearing +// the condemnation of) a file nobody scanned. A superseded write reports +// ErrStaleCopySafetyScan rather than succeeding silently, because the caller +// must also refrain from notifying live sessions on the strength of it. +// +// Both sides of the mtime predicate are normalized to microseconds, exactly as +// MediaFile.PersistedVideoCopyVerdict normalizes them when it reads the verdict +// back: Postgres stores timestamptz at microsecond resolution while a +// filesystem mtime carries nanoseconds, so comparing the raw values would make +// every write for a row whose mtime came from a stat call fail. func (r *FileRepository) UpdateMultiplePPS(ctx context.Context, fileID int, multiplePPS bool, scanSize int64, scanMtime *time.Time) error { + var normalizedMtime *time.Time + if scanMtime != nil { + normalized := models.NormalizeFileModifiedAt(*scanMtime) + normalizedMtime = &normalized + } tag, err := r.pool.Exec(ctx, ` UPDATE media_files SET multiple_pps = $2, multiple_pps_scan_size = $3, multiple_pps_scan_mtime = $4, updated_at = NOW() - WHERE id = $1`, + WHERE id = $1 + AND file_size = $3 + AND date_trunc('microseconds', file_modified_at) IS NOT DISTINCT FROM $4::timestamptz`, fileID, multiplePPS, scanSize, - scanMtime, + normalizedMtime, ) if err != nil { return fmt.Errorf("updating multiple pps verdict: %w", err) } if tag.RowsAffected() == 0 { - return ErrFileNotFound + // The row is gone, or it no longer carries the size and mtime that were + // scanned. Both mean the same thing to every caller: this verdict does + // not describe the file as it stands. + return ErrStaleCopySafetyScan } return nil } diff --git a/internal/scanner/file_repo_copy_safety_db_test.go b/internal/scanner/file_repo_copy_safety_db_test.go new file mode 100644 index 000000000..cdea86efc --- /dev/null +++ b/internal/scanner/file_repo_copy_safety_db_test.go @@ -0,0 +1,142 @@ +package scanner + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// copySafetyTestRow inserts one media_files row with a known size and mtime and +// returns its ID. The row is torn down with the test. +func copySafetyTestRow(t *testing.T, ctx context.Context, pool *pgxpool.Pool, size int64, mtime *time.Time) int { + t.Helper() + + suffix := time.Now().UnixNano() + var folderID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_folders (type, name, enabled) VALUES ('movies', $1, true) RETURNING id`, + fmt.Sprintf("PPS Test %d", suffix), + ).Scan(&folderID); err != nil { + t.Fatalf("insert media folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + + var fileID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (content_id, media_folder_id, file_path, file_size, file_modified_at) + VALUES ($1, $2, $3, $4, $5) RETURNING id`, + fmt.Sprintf("pps-content-%d", suffix), + folderID, + fmt.Sprintf("/tmp/pps-%d/Movie (2020)/Movie (2020).mkv", suffix), + size, + mtime, + ).Scan(&fileID); err != nil { + t.Fatalf("insert media file: %v", err) + } + return fileID +} + +func readCopySafetyRow(t *testing.T, ctx context.Context, pool *pgxpool.Pool, fileID int) (*bool, *int64, *time.Time) { + t.Helper() + var multi *bool + var size *int64 + var mtime *time.Time + if err := pool.QueryRow(ctx, ` + SELECT multiple_pps, multiple_pps_scan_size, multiple_pps_scan_mtime + FROM media_files WHERE id = $1`, fileID).Scan(&multi, &size, &mtime); err != nil { + t.Fatalf("read verdict: %v", err) + } + return multi, size, mtime +} + +// UpdateMultiplePPS is the one write that has to be generation-guarded. A scan +// reads the opening seconds of a file over storage that can be slow, so a file +// rewritten in place while the scan runs would otherwise have the old verdict — +// and the old size and mtime — stamped over the replacement's row, making a +// verdict for bytes that are gone read back as valid. It also drives a +// notification, so the refusal has to be visible rather than a silent no-op. +func TestUpdateMultiplePPSGuardsAgainstAStaleGeneration(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + repo := NewFileRepository(pool) + + // The mtime deliberately carries nanoseconds the row cannot store: a + // predicate that compared the raw value would reject every write made from + // a freshly stat'ed file. + scanned := time.Date(2026, time.March, 4, 5, 6, 7, 123456789, time.UTC) + fileID := copySafetyTestRow(t, ctx, pool, 4096, &scanned) + + t.Run("matching generation writes", func(t *testing.T) { + if err := repo.UpdateMultiplePPS(ctx, fileID, true, 4096, &scanned); err != nil { + t.Fatalf("UpdateMultiplePPS() error = %v, want the write accepted despite sub-microsecond mtime drift", err) + } + multi, size, mtime := readCopySafetyRow(t, ctx, pool, fileID) + if multi == nil || !*multi { + t.Fatalf("multiple_pps = %v, want true", multi) + } + if size == nil || *size != 4096 { + t.Fatalf("multiple_pps_scan_size = %v, want 4096", size) + } + if mtime == nil { + t.Fatal("multiple_pps_scan_mtime = NULL, want the scanned mtime") + } + }) + + t.Run("superseded size is refused", func(t *testing.T) { + if _, err := pool.Exec(ctx, `UPDATE media_files SET file_size = 8192 WHERE id = $1`, fileID); err != nil { + t.Fatalf("rewrite the row: %v", err) + } + err := repo.UpdateMultiplePPS(ctx, fileID, false, 4096, &scanned) + if !errors.Is(err, ErrStaleCopySafetyScan) { + t.Fatalf("UpdateMultiplePPS() error = %v, want ErrStaleCopySafetyScan", err) + } + // The earlier verdict is untouched: a refused write changes nothing. + multi, size, _ := readCopySafetyRow(t, ctx, pool, fileID) + if multi == nil || !*multi || size == nil || *size != 4096 { + t.Fatalf("verdict = (%v, %v), want the refused write to have changed nothing", multi, size) + } + }) + + t.Run("superseded mtime is refused", func(t *testing.T) { + if _, err := pool.Exec(ctx, `UPDATE media_files SET file_size = 4096, file_modified_at = $2 WHERE id = $1`, + fileID, scanned.Add(time.Hour)); err != nil { + t.Fatalf("rewrite the row: %v", err) + } + if err := repo.UpdateMultiplePPS(ctx, fileID, false, 4096, &scanned); !errors.Is(err, ErrStaleCopySafetyScan) { + t.Fatalf("UpdateMultiplePPS() error = %v, want ErrStaleCopySafetyScan", err) + } + }) + + t.Run("row without an mtime", func(t *testing.T) { + bare := copySafetyTestRow(t, ctx, pool, 2048, nil) + if err := repo.UpdateMultiplePPS(ctx, bare, true, 2048, nil); err != nil { + t.Fatalf("UpdateMultiplePPS() error = %v, want a mtime-less row to accept a mtime-less verdict", err) + } + // The same row will not take a verdict claiming an mtime it does not + // have: that pairing is a different generation, not a match. + if err := repo.UpdateMultiplePPS(ctx, bare, true, 2048, &scanned); !errors.Is(err, ErrStaleCopySafetyScan) { + t.Fatalf("UpdateMultiplePPS() error = %v, want ErrStaleCopySafetyScan", err) + } + }) + + t.Run("missing row", func(t *testing.T) { + if err := repo.UpdateMultiplePPS(ctx, -1, true, 4096, &scanned); !errors.Is(err, ErrStaleCopySafetyScan) { + t.Fatalf("UpdateMultiplePPS() error = %v, want ErrStaleCopySafetyScan for a row that is gone", err) + } + }) +} diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index e64ad0f99..09c17e300 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -219,16 +219,43 @@ func (e *PlaybackProbeEnsurer) NeedsCopySafetyScan(file *models.MediaFile) bool // unknown, persisting and memoizing the result. Concurrent callers for one file // share a single scan, so a start, a replan and a watch-page load racing on the // same file spawn one ffmpeg between them. -func (e *PlaybackProbeEnsurer) ScanCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { +// +// The second return reports that the verdict is stale: the row moved to another +// generation of the file while the scan ran, so the answer is correct for bytes +// the server is no longer serving. It is not an error — nothing failed — but a +// caller must neither trust it nor act on it. +func (e *PlaybackProbeEnsurer) ScanCopySafety(ctx context.Context, file *models.MediaFile) (multi bool, stale bool, err error) { if e == nil || file == nil { - return false, nil + return false, false, nil } if strings.TrimSpace(e.ffmpegPath) == "" { - return false, errCopySafetyScanUnavailable + return false, false, errCopySafetyScanUnavailable } return e.scanAndPersistCopySafety(ctx, file) } +// KnownCopySafetyVerdict answers the copy-safety question for a file without +// ever running ffmpeg, from the process memo or from the persisted row, and +// re-attempts a write this process reached but never managed to store. +// +// It exists because "unknown" and "known but unpersisted" are different states +// that the media_files row cannot tell apart. A verdict whose write failed is +// authoritative on this replica and invisible everywhere else, so the paths +// that gate a revived stream-copy — and the race that withdraws one — have to +// be able to ask this process what it already knows rather than only asking the +// row. +func (e *PlaybackProbeEnsurer) KnownCopySafetyVerdict(ctx context.Context, file *models.MediaFile) (bool, bool) { + if e == nil || file == nil { + return false, false + } + multi, known := e.knownCopySafetyVerdict(file) + if !known { + return false, false + } + e.retryUnpersistedCopySafety(ctx, file) + return multi, true +} + var errCopySafetyScanUnavailable = errors.New("ffmpeg path not configured") // knownCopySafetyVerdict answers the copy-safety question from memory or from @@ -298,7 +325,7 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model return fileWithMultiplePPS(file, multi), nil } - multi, err := e.scanAndPersistCopySafety(ctx, file) + multi, stale, err := e.scanAndPersistCopySafety(ctx, file) if err != nil { // Unknown safety must not fail open to the video-copy path this probe is // intended to guard. Leave MultiplePPS unset and do not cache or persist @@ -311,6 +338,16 @@ func (e *PlaybackProbeEnsurer) ensureCopySafety(ctx context.Context, file *model ) return fileWithCopySafety(file, nil, true), nil } + if stale { + // The caller is holding a snapshot of a generation the row has moved + // past. Its verdict describes bytes this file no longer contains, so it + // is treated exactly like an unresolved scan rather than stamped on. + slog.InfoContext(ctx, "video copy-safety verdict superseded before it could be recorded", + "component", "scanner", + "file_id", file.ID, + ) + return fileWithCopySafety(file, nil, true), nil + } return fileWithMultiplePPS(file, multi), nil } @@ -349,13 +386,17 @@ func copySafetyFlightKey(file *models.MediaFile) string { // — is bound to the leader's own snapshot of the file, so a joiner never writes // another generation's facts. The key is what keeps a joiner from *reading* // them. -func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, error) { +// +// A write refused as stale is neither memoized nor reported as a verdict: the +// row has moved to a generation this scan never read, and both the memo and any +// downstream notification would be facts about bytes nobody is serving. +func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, file *models.MediaFile) (bool, bool, error) { fileID := file.ID filePath := file.FilePath fileSize := file.FileSize fileModifiedAt := file.FileModifiedAt - multi, err, _ := e.copySafetyFlight.Do(copySafetyFlightKey(file), func() (any, error) { + outcome, err, _ := e.copySafetyFlight.Do(copySafetyFlightKey(file), func() (any, error) { timeout := e.timeout if timeout < 30*time.Second { timeout = 30 * time.Second @@ -364,7 +405,7 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil multi, err := DetectMultiplePPSH264(scanCtx, e.ffmpegPath, filePath) cancel() if err != nil { - return false, err + return copySafetyOutcome{}, err } // With no writer there is nowhere for the verdict to land, so it is not @@ -372,6 +413,13 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil persisted := true if e.copySafetyRepo != nil { if writeErr := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, multi, fileSize, fileModifiedAt); writeErr != nil { + if errors.Is(writeErr, ErrStaleCopySafetyScan) { + slog.InfoContext(ctx, "discarding a video copy-safety verdict for a superseded generation of the file", + "component", "scanner", + "file_id", fileID, + ) + return copySafetyOutcome{stale: true}, nil + } persisted = false slog.WarnContext(ctx, "persisting video copy-safety verdict failed", "component", "scanner", @@ -381,13 +429,20 @@ func (e *PlaybackProbeEnsurer) scanAndPersistCopySafety(ctx context.Context, fil } } e.storeCopySafety(file, multi, persisted) - return multi, nil + return copySafetyOutcome{multi: multi}, nil }) if err != nil { - return false, err + return false, false, err } - result, _ := multi.(bool) - return result, nil + result, _ := outcome.(copySafetyOutcome) + return result.multi, result.stale, nil +} + +// copySafetyOutcome is what one shared scan produced, carried through the +// singleflight so joiners learn about a superseded write as well as the verdict. +type copySafetyOutcome struct { + multi bool + stale bool } // retryUnpersistedCopySafety re-attempts the media_files write for a verdict @@ -416,19 +471,30 @@ func (e *PlaybackProbeEnsurer) retryUnpersistedCopySafety(ctx context.Context, f // the write while this caller queued behind it. entry, ok := e.memoizedCopySafety(file) if !ok || entry.persisted { - return entry.multi, nil + return copySafetyOutcome{multi: entry.multi}, nil } if err := e.copySafetyRepo.UpdateMultiplePPS(ctx, fileID, entry.multi, entry.size, entry.mtime); err != nil { + if errors.Is(err, ErrStaleCopySafetyScan) { + // The row has moved on. The memo stays — it is still the right + // answer for the snapshot that produced it, and it can no longer + // match a caller holding the current generation — but there is + // nothing left to write, so this stops being a failure. + slog.InfoContext(ctx, "video copy-safety verdict is no longer writable; the row holds another generation", + "component", "scanner", + "file_id", fileID, + ) + return copySafetyOutcome{multi: entry.multi, stale: true}, nil + } slog.WarnContext(ctx, "retrying the video copy-safety verdict write failed", "component", "scanner", "file_id", fileID, "error", err, ) - return entry.multi, nil + return copySafetyOutcome{multi: entry.multi}, nil } entry.persisted = true e.copySafety.Store(fileID, entry) - return entry.multi, nil + return copySafetyOutcome{multi: entry.multi}, nil }) } diff --git a/internal/scanner/probe_repair_copy_safety_cached_test.go b/internal/scanner/probe_repair_copy_safety_cached_test.go index 2b52bff14..64e36ad29 100644 --- a/internal/scanner/probe_repair_copy_safety_cached_test.go +++ b/internal/scanner/probe_repair_copy_safety_cached_test.go @@ -138,10 +138,13 @@ func TestScanCopySafetyPersistsAndMemoizes(t *testing.T) { mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) file := copySafetyTestFile(mtime) - multi, err := ensurer.ScanCopySafety(context.Background(), file) + multi, stale, err := ensurer.ScanCopySafety(context.Background(), file) if err != nil { t.Fatalf("ScanCopySafety() error = %v", err) } + if stale { + t.Fatal("ScanCopySafety() stale = true, want false for a write the row accepted") + } if !multi { t.Fatal("ScanCopySafety() = false, want true for the conflicting-PPS stream") } @@ -181,12 +184,12 @@ func TestScanCopySafetyErrorRecordsNothing(t *testing.T) { file := copySafetyTestFile(time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC)) - multi, err := ensurer.ScanCopySafety(context.Background(), file) + multi, stale, err := ensurer.ScanCopySafety(context.Background(), file) if err == nil { t.Fatal("ScanCopySafety() error = nil, want the scan failure surfaced to the caller") } - if multi { - t.Fatal("ScanCopySafety() = true on error, want false") + if multi || stale { + t.Fatalf("ScanCopySafety() = (%t, %t) on error, want (false, false)", multi, stale) } if writes := writer.recorded(); len(writes) != 0 { t.Fatalf("failed scan recorded %d verdicts, want 0", len(writes)) diff --git a/internal/scanner/probe_repair_copy_safety_persist_test.go b/internal/scanner/probe_repair_copy_safety_persist_test.go index 790902888..20f2d5e6e 100644 --- a/internal/scanner/probe_repair_copy_safety_persist_test.go +++ b/internal/scanner/probe_repair_copy_safety_persist_test.go @@ -308,6 +308,79 @@ func TestEnsureCopySafetyRetriesAFailedPersistWithoutRescanning(t *testing.T) { } } +// A scan reads the opening seconds over storage that can be slow, so the file +// can be rewritten in place while it runs. The row then refuses the write, and +// the verdict is about bytes nobody is serving: it must not be memoized as +// though it described the file, and the caller has to be told, because a +// verdict that never reached the row must not be pushed at live sessions either. +func TestScanCopySafetyReportsASupersededWriteAsStale(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{err: ErrStaleCopySafetyScan} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + file := copySafetyTestFile(mtime) + + multi, stale, err := ensurer.ScanCopySafety(context.Background(), file) + if err != nil { + t.Fatalf("ScanCopySafety() error = %v, want a superseded write reported as stale, not failed", err) + } + if !stale { + t.Fatal("ScanCopySafety() stale = false, want true for a write the row refused") + } + if multi { + t.Fatal("ScanCopySafety() = true alongside stale, want no verdict claimed for a superseded generation") + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want 1", runs()) + } + // Nothing was memoized, so the file is still unresolved: the next request + // for whatever generation the row now holds scans it properly. + if !ensurer.NeedsCopySafetyScan(file) { + t.Fatal("NeedsCopySafetyScan() = false after a superseded write, want the file still unresolved") + } + if _, known := ensurer.KnownCopySafetyVerdict(context.Background(), file); known { + t.Fatal("KnownCopySafetyVerdict() reported a verdict for a superseded generation") + } +} + +// KnownCopySafetyVerdict is the question the serve gate and the race both ask. +// It has to see the memo — a verdict whose write failed is authoritative here +// and invisible on the row — and it must retry that write, without ffmpeg. +func TestKnownCopySafetyVerdictSeesTheMemoAndRetriesTheWrite(t *testing.T) { + ffmpegPath, runs := fakeFFmpeg(t, conflictingPPSAnnexB, 0) + writer := &fakeCopySafetyWriter{err: fmt.Errorf("database unavailable")} + ensurer := &PlaybackProbeEnsurer{ffmpegPath: ffmpegPath, copySafetyRepo: writer} + + mtime := time.Date(2026, time.March, 4, 5, 6, 7, 0, time.UTC) + file := copySafetyTestFile(mtime) + + if _, _, err := ensurer.ScanCopySafety(context.Background(), file); err != nil { + t.Fatalf("ScanCopySafety() error = %v", err) + } + if writes := writer.recorded(); len(writes) != 1 { + t.Fatalf("UpdateMultiplePPS called %d times for the first scan, want 1", len(writes)) + } + + multi, known := ensurer.KnownCopySafetyVerdict(context.Background(), file) + if !known || !multi { + t.Fatalf("KnownCopySafetyVerdict() = (%t, %t), want the unpersisted unsafe verdict", multi, known) + } + if runs() != 1 { + t.Fatalf("ffmpeg ran %d times, want the known-verdict lookup to never scan", runs()) + } + if writes := writer.recorded(); len(writes) != 2 { + t.Fatalf("UpdateMultiplePPS called %d times, want the unpersisted write retried", len(writes)) + } + + // An untouched file is simply unknown; the lookup never invents a verdict. + other := copySafetyTestFile(mtime) + other.ID = 43 + if _, known := ensurer.KnownCopySafetyVerdict(context.Background(), other); known { + t.Fatal("KnownCopySafetyVerdict() reported a verdict for a file nothing has scanned") + } +} + // Rows predating the file_modified_at column carry no mtime. Their verdict is // still persisted and still honored on read — refusing to write it would leave // them permanently unverdicted, so every replica would rescan the same file and diff --git a/web/src/player/hooks/usePlaybackSession.test.ts b/web/src/player/hooks/usePlaybackSession.test.ts index 1f4d9241c..519693f48 100644 --- a/web/src/player/hooks/usePlaybackSession.test.ts +++ b/web/src/player/hooks/usePlaybackSession.test.ts @@ -1757,3 +1757,160 @@ describe("usePlaybackSession server-invalidated plans", () => { unmount(); }); }); + +describe("usePlaybackSession server-invalidated plans", () => { + // An invalidation waits out whatever is still being adopted, so it decides + // against the plan that actually won rather than the one on screen. The wait + // has to be scoped to the request that can still own the session: a start + // abandoned by a version switch cannot install anything any more, and a hung + // one would otherwise hold the invalidation past the server's 8s deadline — + // which stops the very session that is playing fine. + it("does not wait on a superseded start that never settles", async () => { + let startCount = 0; + const replanBodies: Array<{ operation: string }> = []; + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/playback/start")) { + startCount += 1; + if (startCount === 1) { + // The abandoned request: it never settles, and nothing will ever + // count it out. + return new Promise(() => {}); + } + return jsonResponse( + { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-2", + playback_plan: fixturePlanV3({ session_id: "session-2" }), + }, + { status: 201 }, + ); + } + if (url.endsWith("/playback/session-2/replan")) { + replanBodies.push(JSON.parse(String(init?.body)) as { operation: string }); + return jsonResponse({ + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-2", + playback_plan: fixturePlanV3({ + session_id: "session-2", + plan_id: "plan:2222222222222222", + plan_attempt_key: "v3:2222222222222222", + }), + }); + } + if (url.endsWith("/playback/route-events")) return new Response(null, { status: 202 }); + if (init?.method === "DELETE") return new Response(null, { status: 204 }); + throw new Error(`Unexpected request: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const { result, rerender, unmount } = renderHook( + ({ requestKey, fileId }: { requestKey: string; fileId: number }) => + usePlaybackSession(requestKey, [], [], fileId, 0, false, "auto"), + { wrapper, initialProps: { requestKey: "episode-1", fileId: 7 } }, + ); + await waitFor(() => expect(startCount).toBe(1)); + + rerender({ requestKey: "episode-2", fileId: 8 }); + await waitFor(() => expect(result.current.plan).not.toBeNull()); + + const planId = result.current.plan?.plan_id; + if (!planId) throw new Error("expected an adopted plan"); + + const outcome = await act(async () => + Promise.race([ + result.current.invalidatePlan(planId, "video_copy_unsafe", 120), + new Promise<"blocked">((resolve) => { + setTimeout(() => resolve("blocked"), 500); + }), + ]), + ); + + expect(outcome).toBe(true); + expect(replanBodies.map(({ operation }) => operation)).toEqual(["failure_recovery"]); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:2222222222222222")); + + unmount(); + }); + + // The scoping must not weaken the guarantee it was built for: an invalidation + // that arrives while the *current* start is still in flight still waits, so + // it decides against the plan that response installs rather than no-opping + // against the one already on screen. + it("still waits for the start that currently owns the session", async () => { + let releaseStart: ((response: Response) => void) | undefined; + const pendingStart = new Promise((resolve) => { + releaseStart = resolve; + }); + let startCount = 0; + const replanBodies: Array<{ operation: string }> = []; + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/playback/start")) { + startCount += 1; + return pendingStart; + } + if (url.endsWith("/playback/session-1/replan")) { + replanBodies.push(JSON.parse(String(init?.body)) as { operation: string }); + return jsonResponse({ + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ + session_id: "session-1", + plan_id: "plan:3333333333333333", + plan_attempt_key: "v3:3333333333333333", + }), + }); + } + if (url.endsWith("/playback/route-events")) return new Response(null, { status: 202 }); + if (init?.method === "DELETE") return new Response(null, { status: 204 }); + throw new Error(`Unexpected request: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(startCount).toBe(1)); + + // The verdict names a plan this client has not read the response for yet. + let settled = false; + const invalidation = result.current + .invalidatePlan("plan:0123456789abcdef", "video_copy_unsafe", 120) + .then((adopted) => { + settled = true; + return adopted; + }); + await act(async () => { + await Promise.resolve(); + }); + expect(settled).toBe(false); + expect(replanBodies).toHaveLength(0); + + await act(async () => { + releaseStart?.( + jsonResponse( + { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ session_id: "session-1" }), + }, + { status: 201 }, + ), + ); + await expect(invalidation).resolves.toBe(true); + }); + expect(replanBodies.map(({ operation }) => operation)).toEqual(["failure_recovery"]); + + unmount(); + }); +}); diff --git a/web/src/player/hooks/usePlaybackSession.ts b/web/src/player/hooks/usePlaybackSession.ts index 4a53a1845..ebed49240 100644 --- a/web/src/player/hooks/usePlaybackSession.ts +++ b/web/src/player/hooks/usePlaybackSession.ts @@ -316,14 +316,23 @@ export function usePlaybackSession( const attemptedPlanKeysRef = useRef([]); const attemptCountRef = useRef(1); const replanInFlightRef = useRef(false); - // Adoptions in flight: a start or a replan whose decision has not been - // applied yet. The server commits a replacement plan — and starts the - // copy-safety scan behind it — before the client can read the response, so a - // `plan_invalidated` command can name a plan this client is still adopting. - // Waiters registered here are woken once nothing is in flight, which lets an - // invalidation decide against the plan that actually won. - const adoptionsInFlightRef = useRef(0); - const adoptionWaitersRef = useRef void>>([]); + // Adoptions in flight, counted per load sequence: a start or a replan whose + // decision has not been applied yet. The server commits a replacement plan — + // and starts the copy-safety scan behind it — before the client can read the + // response, so a `plan_invalidated` command can name a plan this client is + // still adopting. Waiters registered here are woken once their own sequence + // has nothing in flight, which lets an invalidation decide against the plan + // that actually won. + // + // The key is what makes the wait bounded. A superseded request — a version + // switch abandoned mid-flight, a start whose `fetch` never settles — is not a + // candidate to own the session any more, so waiting for it decides nothing + // and is worse than not waiting: the server's invalidation deadline is 8s, + // and a session that misses it is stopped outright. Only the sequence that + // currently owns the session can still change what the invalidation should + // decide against, so only it is waited on. + const adoptionsInFlightRef = useRef(new Map()); + const adoptionWaitersRef = useRef void }>>([]); const pendingReplanRef = useRef<{ options: ReplanOptions; loadSequence: number; @@ -340,34 +349,51 @@ export function usePlaybackSession( stateRef.current = state; }, [state]); - const beginAdoption = useCallback(() => { - adoptionsInFlightRef.current += 1; + const beginAdoption = useCallback((loadSequence: number) => { + const inFlight = adoptionsInFlightRef.current; + inFlight.set(loadSequence, (inFlight.get(loadSequence) ?? 0) + 1); }, []); /** - * Counts one in-flight adoption out. + * Counts one in-flight adoption out of its load sequence. * - * Waiters are woken only when nothing is left in flight: a queued replan is - * dispatched from its predecessor's `finally` before the predecessor is - * counted out, so the count tracks the whole chain rather than one request. + * A sequence's waiters are woken only when nothing is left in flight for it: + * a queued replan is dispatched from its predecessor's `finally` before the + * predecessor is counted out, so the count tracks the whole chain rather than + * one request. */ - const endAdoption = useCallback(() => { - adoptionsInFlightRef.current = Math.max(0, adoptionsInFlightRef.current - 1); - if (adoptionsInFlightRef.current > 0) return; + const endAdoption = useCallback((loadSequence: number) => { + const inFlight = adoptionsInFlightRef.current; + const remaining = (inFlight.get(loadSequence) ?? 0) - 1; + if (remaining > 0) { + inFlight.set(loadSequence, remaining); + return; + } + inFlight.delete(loadSequence); const waiters = adoptionWaitersRef.current; if (waiters.length === 0) return; - adoptionWaitersRef.current = []; - for (const wake of waiters) wake(); + const settled = waiters.filter((waiter) => !inFlight.has(waiter.loadSequence)); + if (settled.length === 0) return; + adoptionWaitersRef.current = waiters.filter((waiter) => inFlight.has(waiter.loadSequence)); + for (const waiter of settled) waiter.resolve(); }, []); /** - * Resolves once no start or replan is in flight, or null when none is — - * callers act synchronously in the common case rather than deferring a turn. + * Resolves once the sequence that currently owns the session has no start or + * replan in flight, or null when it has none — callers act synchronously in + * the common case rather than deferring a turn. + * + * A request from a superseded sequence is deliberately not waited for. It can + * no longer install a plan (every path re-checks the sequence before adopting + * one), so it has nothing left to say about what an invalidation should + * decide against, and a hung one would otherwise hold the wait open past the + * server's deadline and cost the live session its stream. */ const awaitAdoptionSettled = useCallback((): Promise | null => { - if (adoptionsInFlightRef.current === 0) return null; + const loadSequence = loadSequenceRef.current; + if (!adoptionsInFlightRef.current.has(loadSequence)) return null; return new Promise((resolve) => { - adoptionWaitersRef.current.push(resolve); + adoptionWaitersRef.current.push({ loadSequence, resolve }); }); }, []); @@ -619,7 +645,7 @@ export function usePlaybackSession( })); }; - beginAdoption(); + beginAdoption(loadSequence); try { const selectedFileId = selectFileId(preferredFileId); if (!selectedFileId) { @@ -683,7 +709,7 @@ export function usePlaybackSession( const nextError = describePlaybackSessionError(err, initialErrorMessage); retirePreviousSession(nextError); } finally { - endAdoption(); + endAdoption(loadSequence); } }, [adoptDecision, beginAdoption, endAdoption, requestStart, selectFileId, stopSession], @@ -860,7 +886,7 @@ export function usePlaybackSession( const loadSequence = loadSequenceRef.current; replanInFlightRef.current = true; - beginAdoption(); + beginAdoption(loadSequence); setState((current) => ({ ...current, replanning: true, @@ -950,7 +976,7 @@ export function usePlaybackSession( } // Last: a queued replan dispatched just above has already counted // itself in, so waiters are not woken between the two links of a chain. - endAdoption(); + endAdoption(loadSequence); } }, [ From b6a504d52fc594cde47af89a24f0e79d30b4ab2b Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:51:09 -0400 Subject: [PATCH 43/44] feat(playback): let original players manage HDR Accept delivery-scoped client claims for Aether-managed dynamic range and selected audio on original HTTP while retaining packaged-output gates and the existing behavior for clients that do not claim support. --- cmd/playbackfixtures/main.go | 27 +- docs/architecture/playback-protocol-v3.md | 50 ++- docs/feature-changelog.md | 3 + internal/playback/capabilities_v3.go | 27 +- internal/playback/plan_v3.go | 64 +++- internal/playback/protocol_v3.go | 26 +- internal/playback/protocol_v3_test.go | 333 +++++++++++++++++- .../protocol_v3/conformance_matrix.json | 237 +++++++++++++ 8 files changed, 710 insertions(+), 57 deletions(-) diff --git a/cmd/playbackfixtures/main.go b/cmd/playbackfixtures/main.go index a05e83b7c..d3d417743 100644 --- a/cmd/playbackfixtures/main.go +++ b/cmd/playbackfixtures/main.go @@ -555,6 +555,25 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 { hdr10Request.PlaybackAttemptID = "attempt-hdr10-direct" planner = append(planner, makePlannerScenario("hdr10_exact_direct", "hdr_dv_matrix", hdr10Request, conformanceHDRFile(), nil, settings, registry)) + clientManagedFile := conformanceHDRFile() + clientManagedFile.AudioTracks = append(clientManagedFile.AudioTracks, models.AudioTrack{Codec: codecAAC, Channels: 2, Layout: audioLayoutStereo}) + clientManagedRequest := conformanceHDRRequest() + clientManagedRequest.PlaybackAttemptID = "attempt-client-managed-original" + clientManagedRequest.Capabilities.HDR = false + clientManagedRequest.Capabilities.HDRDetails = &playback.HDRCapabilitiesV3{DolbyVisionProfiles: []int{}} + clientManagedRequest.ClientPlaybackContext.Output.HDRDetails = &playback.HDRCapabilitiesV3{DolbyVisionProfiles: []int{}} + clientManagedAudioIndex := 1 + clientManagedRequest.AudioTrackIndex = &clientManagedAudioIndex + clientManagedRequest.AudioTrackID = playback.TrackIDV3(clientManagedFile.ID, "audio", clientManagedAudioIndex) + clientManagedDelivery := clientManagedRequest.ClientPlaybackContext.Deliveries[playback.DeliveryClassOriginalHTTPV3] + clientManagedDelivery.HDRDetails = &playback.HDRCapabilitiesV3{DolbyVisionProfiles: []int{}} + clientManagedDelivery.ValidatedClaims = []string{playback.ClaimClientManagedDynamicRangeV3, playback.ClaimClientSelectedAudioTrackV3} + clientManagedRequest.ClientPlaybackContext.Deliveries[playback.DeliveryClassOriginalHTTPV3] = clientManagedDelivery + planner = append(planner, makePlannerScenarioWithAudioIndex( + "client_managed_hdr_selected_audio", "hdr_dv_matrix", clientManagedRequest, clientManagedFile, + clientManagedAudioIndex, nil, settings, registry, + )) + dv8File := conformanceHDRFile() dv8File.VideoTracks[0].DVProfile = 8 dv8File.VideoTracks[0].DVBLCompatID = 1 @@ -745,8 +764,12 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 { } func makePlannerScenario(name, category string, request playback.StartRequestV3, file *models.MediaFile, attempted []string, settings playback.PlannerSettingsV3, registry *playback.TransformationRegistryV3) playback.PlannerScenarioV3 { + return makePlannerScenarioWithAudioIndex(name, category, request, file, 0, attempted, settings, registry) +} + +func makePlannerScenarioWithAudioIndex(name, category string, request playback.StartRequestV3, file *models.MediaFile, audioTrackIndex int, attempted []string, settings playback.PlannerSettingsV3, registry *playback.TransformationRegistryV3) playback.PlannerScenarioV3 { result := playback.PlanPlaybackV3(playback.PlannerInputV3{ - Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: audioTrackIndex, Settings: settings, Registry: registry, AttemptedKeys: attempted, }) expected := playback.PlannerExpectationV3{Outcome: playback.OutcomeAdaptationUnavailableV3} @@ -771,7 +794,7 @@ func makePlannerScenario(name, category string, request playback.StartRequestV3, } return playback.PlannerScenarioV3{ Name: name, Category: category, Request: request, - Source: playback.SourceDescriptorFromFileV3(file, 0), + Source: playback.SourceDescriptorFromFileV3(file, audioTrackIndex), AttemptedKeys: append([]string(nil), attempted...), Expected: expected, } } diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index ea19473ed..3c6967409 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -401,14 +401,25 @@ HAL — can supply that. `platform_attested` and `declared` audio evidence still qualify for ordinary decode/copy routes; they simply cannot earn `claims.audio.passthrough = true`. -**HDR is decided against the output, not the decoder.** `output.hdr_details` (the -display or receiver actually attached) takes precedence over -`client_capabilities.hdr_details` (what the device could do in principle). A -source whose dynamic range is recorded as `hdr_unknown` — legacy rows that only -stored a file-level HDR boolean — is treated as HDR10 when the output supports -HDR10, and the plan carries the `hdr_range_assumed_hdr10` degradation warning. -Refusing to play those outright would be worse than an assumption the client is -told about. +**Native HDR presentation is decided against the output, not the decoder.** +`output.hdr_details` (the display or receiver actually attached) takes +precedence over `client_capabilities.hdr_details` (what the device could do in +principle). A source whose dynamic range is recorded as `hdr_unknown` — legacy +rows that only stored a file-level HDR boolean — is treated as HDR10 when the +output supports HDR10, and the plan carries the `hdr_range_assumed_hdr10` +degradation warning. Refusing to play those outright would be worse than an +assumption the client is told about. + +There is one delivery-scoped exception. An `original_http` capability carrying +the validated claim `client_managed_dynamic_range_v1` asserts that its executor +accepts the declared source range and resolves presentation against the live +output after receiving the original bytes. The planner may therefore deliver +HDR or Dolby Vision through that class even when the active sink does not +natively advertise the source range. The exception does not apply to +`progressive` or `hls`: those server-packaged streams remain output-gated. The +output snapshot is still retained for plan identity, diagnostics, output-change +replans, explicit Dolby Vision transformation selection, and future server +tone-map targeting. The web client does not promote the generic high-dynamic-range media query to a format claim, and it does not gate format claims on it either. Decoder capability @@ -443,6 +454,14 @@ client negotiates in three classes. | `server_remux_hls` | `hls` | Repackaged into HLS segments; codecs untouched | | `server_transcode_hls` | `hls` | Re-encoded and segmented | +Because `original_http` carries the complete source file, a client may put +`client_selected_audio_track_v1` in that delivery's `validated_claims`. The +claim says it maps `selected_tracks.audio.index` onto its probed source +inventory, so selecting a non-default audio track does not by itself require +the server to remux the file. Without the claim, the historical default-track +gate remains. A claiming client that cannot honor the identity reports a typed +playback failure so the bounded replan ladder can choose a packaged route. + `client_playback_context.deliveries` is keyed by **class**, because a client's answer to "can you play HLS" does not differ between a remux and a transcode — the same player component handles both. The server folds its four values into @@ -471,6 +490,14 @@ because "the user turned HLS off" and "this device has no HLS player" call for different degradation warnings and different diagnostics. A class the client omits entirely is unavailable — the server will not guess. +`client_managed_dynamic_range_v1` is valid only as a `validated_claims` entry +on `original_http`. It is not a selectable transformation: the server supplies +the source and the client executor probes and routes it internally. If that +executor later reports a typed load failure, normal attempted-plan-key +exclusion applies. Until a server tone-map recipe exists, an exhausted HDR +original route terminates honestly rather than pretending an ordinary video +transcode can produce a supported result. + `stream.header_refresh` tells the client what to do when the stream URL's auth expires: `none` means the URL is stable for the session, `session` means re-request headers from `header_refresh_url` rather than restarting playback. @@ -1123,6 +1150,13 @@ transformations participate in plan identity exactly like server ones, so a client that changes its transform version invalidates its prior attempt keys — which is the intent. +Automatic work wholly owned by an original-file executor is not enumerated as +a transformation merely because it can include demuxing, local repackaging, +audio bridging, or display adaptation. Those operations do not give the server +a distinct selectable output recipe. Use a delivery claim for an executor +property; reserve transformations for named outcomes the server deliberately +selects and can describe in the plan. + --- ## 12. Conformance diff --git a/docs/feature-changelog.md b/docs/feature-changelog.md index a5f21331f..24ee2efa4 100644 --- a/docs/feature-changelog.md +++ b/docs/feature-changelog.md @@ -16,6 +16,9 @@ Three things change. Media pages no longer trigger the analysis at all; it now h A file that changes on disk is re-checked automatically: the stored answer is only trusted while the file's size and modification time still match, so re-encoding or replacing a file in place invalidates it without any manual step. Nothing is recorded when an analysis fails, so a transient error never turns into a stale verdict — the next request simply retries. No configuration changes, and playback behavior is unchanged. +### Let capable original-file players manage HDR presentation +Playback protocol v3 now recognizes the delivery-scoped `client_managed_dynamic_range_v1` claim on `original_http`. A client with a runtime-probing original-file engine can receive a declared HDR or Dolby Vision source even when the connected display does not natively advertise that source range, then choose its local presentation path after loading the file. The companion `client_selected_audio_track_v1` claim lets original delivery keep the complete source when a non-default audio track is selected; the plan's selected-track ordinal tells the claiming client which probed stream to activate instead of forcing a server remux, while unclaimed clients retain the old gate. Progressive and HLS outputs remain display-gated, explicit server-selected Dolby Vision transformations remain preferred, and a typed failure excludes the attempted original plan instead of looping; until server tone mapping exists, an exhausted HDR route still reports that limitation honestly. + ### Serve tokenless playback from proxy nodes again Playback protocol v3 now advertises the engine-neutral `authorized_media_origins_v1` opt-in, which a client sends together with `header_authenticated_media_v1`. Plans for such an attempt may return absolute, still credential-free media URLs on server-designated proxy origins (`/stream/v3/...`), so direct play, progressive remux, and HLS egress from the node pool instead of the API server. The proxy validates the caller's own access token against the same live login session the API checks, so revoking a session stops proxy playback immediately; replans and every other control-plane call stay on the API. A client that sends only `header_authenticated_media_v1` keeps today's API-local behavior unchanged, and so does a deployment with no proxy pool. diff --git a/internal/playback/capabilities_v3.go b/internal/playback/capabilities_v3.go index c81151de8..ddabaf7f8 100644 --- a/internal/playback/capabilities_v3.go +++ b/internal/playback/capabilities_v3.go @@ -87,7 +87,7 @@ func SourceDescriptorFromFileV3(file *models.MediaFile, audioIndex int) SourceDe } if source.DynamicRange == "" { if file.HDR { - source.DynamicRange = "hdr_unknown" + source.DynamicRange = DynamicRangeHDRUnknownV3 } else { source.DynamicRange = DynamicRangeSDRV3 } @@ -188,7 +188,7 @@ func outputRangeEligibleV3(source SourceDescriptorV3, request StartRequestV3) (b case "hdr10": claims.HDR10 = hdr != nil && hdr.HDR10 return claims.HDR10, claims - case "hdr_unknown": + case DynamicRangeHDRUnknownV3: // Legacy rows only recorded a file-level HDR flag without per-track // range metadata. HDR10 is by far the most common static-HDR range, so // an HDR10-capable output treats the source as HDR10 instead of @@ -219,6 +219,29 @@ func outputRangeEligibleV3(source SourceDescriptorV3, request StartRequestV3) (b } } +// clientManagesOriginalDynamicRangeV3 is intentionally delivery-scoped. It +// says the original-file executor can inspect the source and choose its own +// display presentation after delivery; it does not make the same HDR source +// safe for a server-produced progressive or HLS stream. +func clientManagesOriginalDynamicRangeV3(source SourceDescriptorV3, request StartRequestV3) bool { + if source.DynamicRange == "" || source.DynamicRange == DynamicRangeSDRV3 { + return false + } + delivery, ok := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + return ok && delivery.Enabled && delivery.SupportedOnDevice && + containsFoldV3(delivery.ValidatedClaims, ClaimClientManagedDynamicRangeV3) +} + +// clientSelectsOriginalAudioTrackV3 is delivery-scoped because original HTTP +// carries every source stream unchanged. A client that makes this claim maps +// selected_tracks.audio.index onto its probed source inventory; clients that +// do not keep the historical server-remux requirement for non-default audio. +func clientSelectsOriginalAudioTrackV3(request StartRequestV3) bool { + delivery, ok := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + return ok && delivery.Enabled && delivery.SupportedOnDevice && + containsFoldV3(delivery.ValidatedClaims, ClaimClientSelectedAudioTrackV3) +} + func clientSupportsHDR10V3(request StartRequestV3) bool { hdr := request.ClientPlaybackContext.Output.HDRDetails if hdr == nil { diff --git a/internal/playback/plan_v3.go b/internal/playback/plan_v3.go index d08c8f9c4..c57dd5520 100644 --- a/internal/playback/plan_v3.go +++ b/internal/playback/plan_v3.go @@ -156,7 +156,11 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { } } rangeOK, videoClaims := outputRangeEligibleV3(source, input.Request) + clientManagedRange := clientManagesOriginalDynamicRangeV3(source, input.Request) + originalRangeOK := rangeOK || clientManagedRange audioOK, passthrough, audioClaims := audioEligibilityV3(source, input.Request) + originalAudioSelectionOK := audioSelectionUsesContainerDefaultV3(file, input.AudioTrackIndex) || + clientSelectsOriginalAudioTrackV3(input.Request) if !audioOK && source.AudioCodec == "" && (file == nil || len(file.AudioTracks) == 0) { // Video-only media has no audio stream to adapt: treating the absence // as an unsupported codec would force a pointless AAC conversion — or @@ -198,7 +202,7 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { // so the client is told the actual cause — a source whose Dolby Vision // metadata cannot be removed — rather than a generic HDR message that // sends the user looking for a missing encoder. - if dvStripUnsupportedBySource && !rangeOK && !clientDV81Eligible && !clientHDR10Eligible { + if dvStripUnsupportedBySource && !originalRangeOK && !clientDV81Eligible && !clientHDR10Eligible { return terminalPlannerResultV3(TerminalDVConversionUnsupportedV3, "This source's Dolby Vision metadata cannot be removed cleanly, and this device cannot play the source as it is.", false) } @@ -223,10 +227,10 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { base.AvailableQualities = availableQualitiesV3(input, source) base.Subtitle.Inventory = BuildSubtitleInventoryV3(file, input.AdditionalSubtitles) base.Claims.Audio.Passthrough = passthrough - if source.DynamicRange == "hdr_unknown" && rangeOK { + if source.DynamicRange == DynamicRangeHDRUnknownV3 && (rangeOK || clientManagedRange) { base.DegradationWarnings = append(base.DegradationWarnings, DegradationWarningV3{ Code: "hdr_range_assumed_hdr10", - Message: "The source is flagged HDR without precise range metadata and is delivered as HDR10.", + Message: "The source is flagged HDR without precise range metadata; playback treats it as HDR10 unless the client resolves a more precise presentation.", }) } if dvStripUnsupportedBySource { @@ -260,7 +264,7 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { // degradation warning instead of refusing playback. Explicit user-selected // rungs keep the existing terminals. if quality.RequiresTranscode && !quality.ExplicitRung && !subtitle.RequiresBurn && videoOK && - (rangeOK || dvStripEligible || clientDV81Eligible || clientHDR10Eligible) && + (originalRangeOK || dvStripEligible || clientDV81Eligible || clientHDR10Eligible) && !videoTranscodeExecutableV3(input, source) { warnings := append(quality.Warnings, DegradationWarningV3{ Code: "quality_reduction_unavailable", @@ -272,7 +276,7 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { base.DegradationWarnings = append(base.DegradationWarnings, quality.Warnings...) if quality.RequiresTranscode || !videoOK || - (!rangeOK && !dvStripEligible && !clientDV81Eligible && !clientHDR10Eligible) || + (!originalRangeOK && !dvStripEligible && !clientDV81Eligible && !clientHDR10Eligible) || (subtitle.RequiresBurn && !remuxSubtitleOK && !hlsRemuxSubtitleOK) { reasonOverride := "" if !quality.RequiresTranscode && !videoOK && videoEvidenceInsufficient { @@ -284,7 +288,7 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { // True when the burn requirement is the sole disjunct that fired: every // other route condition still permits a source-preserving delivery. subtitleForcedAdaptation := !quality.RequiresTranscode && videoOK && - (rangeOK || dvStripEligible || clientDV81Eligible || clientHDR10Eligible) && + (originalRangeOK || dvStripEligible || clientDV81Eligible || clientHDR10Eligible) && subtitle.RequiresBurn && !remuxSubtitleOK && !hlsRemuxSubtitleOK return planVideoTranscodeV3(input, base, source, quality, hlsSubtitle, reasonOverride, subtitleForcedAdaptation) } @@ -293,8 +297,7 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { // source. A decoder profile/max-instance claim alone is not proof of native // dual-layer output, so the default Android route mirrors Silo Apple: P8.1 // base-layer Dolby Vision first, then same-file HDR10. - if source.DVProfile == 7 && quality.PreservesSource && videoOK && containerOK && audioOK && - audioSelectionUsesContainerDefaultV3(file, input.AudioTrackIndex) && !subtitle.RequiresBurn { + if source.DVProfile == 7 && quality.PreservesSource && videoOK && containerOK && audioOK && originalAudioSelectionOK && !subtitle.RequiresBurn { if clientDV81Eligible { plan := base plan.Delivery = DeliveryOriginalHTTPV3 @@ -335,14 +338,26 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { return PlannerResultV3{Plan: &plan, PlayMethod: PlayDirect, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleCodec: subtitle.Codec, DownloadedSubtitleID: subtitle.DownloadedSubtitleID} } } + if clientManagedRange { + plan := base + plan.Delivery = DeliveryOriginalHTTPV3 + plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshNoneV3} + plan.DecisionReason = decisionReasonClientManagedDynamicRangeV3 + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID, input.Request.ClientPlaybackContext.Output.OutputContextID) + if deliverySupportsPlanV3(input.Request, DeliveryClassOriginalHTTPV3, plan) && !planAttemptedV3(plan, input.Request.ClientPlaybackContext.Output.OutputContextID, input.AttemptedKeys) { + return PlannerResultV3{Plan: &plan, PlayMethod: PlayDirect, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleCodec: subtitle.Codec, DownloadedSubtitleID: subtitle.DownloadedSubtitleID} + } + } } - if source.DVProfile != 7 && deliveryAvailableV3(input.Request, DeliveryClassOriginalHTTPV3) && containerOK && videoOK && rangeOK && audioOK && quality.PreservesSource && - audioSelectionUsesContainerDefaultV3(file, input.AudioTrackIndex) && !subtitle.RequiresBurn { + if source.DVProfile != 7 && deliveryAvailableV3(input.Request, DeliveryClassOriginalHTTPV3) && containerOK && videoOK && originalRangeOK && audioOK && originalAudioSelectionOK && quality.PreservesSource && !subtitle.RequiresBurn { plan := base plan.Delivery = DeliveryOriginalHTTPV3 plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshNoneV3} plan.DecisionReason = "validated_original_playback" + if !rangeOK && clientManagedRange { + plan.DecisionReason = decisionReasonClientManagedDynamicRangeV3 + } applyCopiedVideoQuirksV3(&plan, source, input.Request, high10Quirk) finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID, input.Request.ClientPlaybackContext.Output.OutputContextID) if deliverySupportsPlanV3(input.Request, DeliveryClassOriginalHTTPV3, plan) && !planAttemptedV3(plan, input.Request.ClientPlaybackContext.Output.OutputContextID, input.AttemptedKeys) { @@ -530,9 +545,15 @@ func audioAvailableQualitiesV3(source SourceDescriptorV3) []AvailableQualityV3 { return []AvailableQualityV3{{Label: QualityOriginalV3, BitrateKbps: source.BitrateKbps, PreservesSource: true}} } -// decisionReasonBandwidthCapV3 marks a plan whose recipe was constrained by -// the request's bandwidth cap rather than by decode capability. -const decisionReasonBandwidthCapV3 = "quality_bandwidth_cap" +const ( + // decisionReasonBandwidthCapV3 marks a plan whose recipe was constrained by + // the request's bandwidth cap rather than by decode capability. + decisionReasonBandwidthCapV3 = "quality_bandwidth_cap" + + // decisionReasonClientManagedDynamicRangeV3 marks an original-file plan + // whose executor owns source-to-output dynamic-range presentation. + decisionReasonClientManagedDynamicRangeV3 = "client_managed_dynamic_range" +) // planAudioOnlyV3 plans sources without a video track (audiobooks, music). // The route family is deliberately small: the original container over @@ -542,6 +563,8 @@ const decisionReasonBandwidthCapV3 = "quality_bandwidth_cap" func planAudioOnlyV3(input PlannerInputV3, file *models.MediaFile, source SourceDescriptorV3) PlannerResultV3 { request := input.Request audioOK, _, audioClaims := audioEligibilityV3(source, request) + originalAudioSelectionOK := audioSelectionUsesContainerDefaultV3(file, input.AudioTrackIndex) || + clientSelectsOriginalAudioTrackV3(request) bandwidthCapKbps := optionalValueV3(request.BandwidthCapKbps) bandwidthCapExceeded := bandwidthCapKbps > 0 && source.BitrateKbps > bandwidthCapKbps if source.AudioCodec == "" { @@ -570,7 +593,7 @@ func planAudioOnlyV3(input PlannerInputV3, file *models.MediaFile, source Source Timeline: TimelineV3{SourceStartSeconds: floatOrZeroV3(request.StartPosition), PlayerStartSeconds: floatOrZeroV3(request.StartPosition), CanSeekAnywhere: true, SeekRestoration: "player_position"}, } containerOK := containsFoldV3(request.Capabilities.Containers, source.Container) - if audioOK && containerOK && !bandwidthCapExceeded && audioSelectionUsesContainerDefaultV3(file, input.AudioTrackIndex) && deliveryAvailableV3(request, DeliveryClassOriginalHTTPV3) { + if audioOK && containerOK && !bandwidthCapExceeded && originalAudioSelectionOK && deliveryAvailableV3(request, DeliveryClassOriginalHTTPV3) { plan := base plan.Delivery = DeliveryOriginalHTTPV3 plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshNoneV3} @@ -1015,9 +1038,9 @@ func selectedTracksForPlanV3(file *models.MediaFile, audioIndex int, subtitle Su } // audioSelectionUsesContainerDefaultV3 reports whether an untouched source -// stream can realize the selected audio track. Original HTTP serves the file -// byte-for-byte, so any non-default selection must use a remux/transcode route -// that can map the requested stream explicitly. +// stream can realize the selected audio track without client-side selection. +// Clients that explicitly claim client_selected_audio_track_v1 on +// original_http may select another stream after probing the complete source. func audioSelectionUsesContainerDefaultV3(file *models.MediaFile, audioIndex int) bool { if file == nil || len(file.AudioTracks) == 0 { return true @@ -1196,7 +1219,10 @@ func deliverySupportsPlanV3(request StartRequestV3, deliveryClass string, plan P if capability.MaxChannels != nil && plan.EffectiveRecipe.AudioChannels != nil && *plan.EffectiveRecipe.AudioChannels > *capability.MaxChannels { return false } - if capability.HDRDetails != nil && !hdrDetailsSupportPlanV3(*capability.HDRDetails, plan) { + clientManagedOriginalRange := deliveryClass == DeliveryClassOriginalHTTPV3 && + len(plan.Transformations) == 0 && + containsFoldV3(capability.ValidatedClaims, ClaimClientManagedDynamicRangeV3) + if capability.HDRDetails != nil && !hdrDetailsSupportPlanV3(*capability.HDRDetails, plan) && !clientManagedOriginalRange { return false } return true @@ -1206,7 +1232,7 @@ func hdrDetailsSupportPlanV3(hdr HDRCapabilitiesV3, plan PlanV3) bool { switch plan.EffectiveRecipe.DynamicRange { case "", DynamicRangeSDRV3: return true - case DynamicRangeHDR10V3, "hdr_unknown": + case DynamicRangeHDR10V3, DynamicRangeHDRUnknownV3: return hdr.HDR10 && hdr10LimitsSupportPlanV3(hdr, plan) case DynamicRangeHDR10PlusV3: return hdr.HDR10Plus diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index f162fdfd9..785a40455 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -57,15 +57,22 @@ const ( // negotiate the token, or has no realtime connection, is stopped instead; // the client's ordinary recovery then mints a fresh attempt that plans // against the now-persisted verdict. - FeaturePlanInvalidatedV3 = "plan_invalidated_v1" - PlanRecipeVersionV3 = "v3.4" - ClientDV7ToDV81V3 = "client_dv7_to_dv81" - ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" - ClientDVTransformVersionV3 = "1" - ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" - ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" - ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" - DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" + FeaturePlanInvalidatedV3 = "plan_invalidated_v1" + PlanRecipeVersionV3 = "v3.4" + ClientDV7ToDV81V3 = "client_dv7_to_dv81" + ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" + ClientDVTransformVersionV3 = "1" + // ClaimClientManagedDynamicRangeV3 is scoped to the original_http + // delivery. A client that advertises it accepts responsibility for mapping + // any source dynamic range it declares decodable onto the active output; + // the server must not require native sink-HDR support before delivering the + // original bytes. Packaged server deliveries remain output-gated. + ClaimClientManagedDynamicRangeV3 = "client_managed_dynamic_range_v1" + ClaimClientSelectedAudioTrackV3 = "client_selected_audio_track_v1" + ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" + ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" + ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" + DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" ) // ServerFeaturesV3 returns the complete feature set advertised by protocol-v3 @@ -164,6 +171,7 @@ const ( DynamicRangeHDR10PlusV3 = "hdr10_plus" DynamicRangeHLGV3 = "hlg" DynamicRangeDolbyVisionV3 = "dolby_vision" + DynamicRangeHDRUnknownV3 = "hdr_unknown" ) // Server transformation names. A plan names the transformations its serving diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index ff63f3af5..2122fab6d 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -353,22 +353,23 @@ func TestProtocolV3ConformanceMatrixCoversReleaseTrain(t *testing.T) { } } for name, delivery := range map[string]DeliveryV3{ - "evidence_exact": DeliveryTranscodeHLSV3, - "evidence_platform_attested": DeliveryOriginalHTTPV3, - "evidence_declared": DeliveryOriginalHTTPV3, - "delivery_original": DeliveryOriginalHTTPV3, - "delivery_progressive": DeliveryRemuxProgressiveV3, - "delivery_hls": DeliveryRemuxHLSV3, - "delivery_transcode": DeliveryTranscodeHLSV3, - "audio_only_original": DeliveryOriginalHTTPV3, - "hdr10_exact_direct": DeliveryOriginalHTTPV3, - "dolby_vision_8_exact_direct": DeliveryOriginalHTTPV3, - "dolby_vision_7_hdr10_fallback": DeliveryRemuxProgressiveV3, - "truehd_audio_conversion": DeliveryRemuxProgressiveV3, - "truehd_exact_layout_passthrough": DeliveryOriginalHTTPV3, - "embedded_pgs_sidecar": DeliveryOriginalHTTPV3, - "embedded_ass_authored_render": DeliveryOriginalHTTPV3, - "embedded_dvd_burn_in": DeliveryTranscodeHLSV3, + "evidence_exact": DeliveryTranscodeHLSV3, + "evidence_platform_attested": DeliveryOriginalHTTPV3, + "evidence_declared": DeliveryOriginalHTTPV3, + "delivery_original": DeliveryOriginalHTTPV3, + "delivery_progressive": DeliveryRemuxProgressiveV3, + "delivery_hls": DeliveryRemuxHLSV3, + "delivery_transcode": DeliveryTranscodeHLSV3, + "audio_only_original": DeliveryOriginalHTTPV3, + "hdr10_exact_direct": DeliveryOriginalHTTPV3, + "client_managed_hdr_selected_audio": DeliveryOriginalHTTPV3, + "dolby_vision_8_exact_direct": DeliveryOriginalHTTPV3, + "dolby_vision_7_hdr10_fallback": DeliveryRemuxProgressiveV3, + "truehd_audio_conversion": DeliveryRemuxProgressiveV3, + "truehd_exact_layout_passthrough": DeliveryOriginalHTTPV3, + "embedded_pgs_sidecar": DeliveryOriginalHTTPV3, + "embedded_ass_authored_render": DeliveryOriginalHTTPV3, + "embedded_dvd_burn_in": DeliveryTranscodeHLSV3, } { value, ok := plannerByName[name] if !ok || value.Expected.Outcome != OutcomePlayableV3 || value.Expected.Delivery != delivery { @@ -390,6 +391,11 @@ func TestProtocolV3ConformanceMatrixCoversReleaseTrain(t *testing.T) { if value := plannerByName["hdr10_exact_direct"]; value.Source.DynamicRange != DynamicRangeHDR10V3 || value.Source.BitDepth != 10 { t.Errorf("HDR10 scenario source = %#v", value.Source) } + if value := plannerByName["client_managed_hdr_selected_audio"]; value.Expected.DecisionReason != decisionReasonClientManagedDynamicRangeV3 || + value.Expected.SelectedTracks == nil || value.Expected.SelectedTracks.Audio == nil || value.Expected.SelectedTracks.Audio.Index == nil || + *value.Expected.SelectedTracks.Audio.Index != 1 { + t.Errorf("client-managed HDR selected-audio scenario = %#v", value) + } if value := plannerByName["dolby_vision_8_exact_direct"]; value.Source.DynamicRange != DynamicRangeDolbyVisionV3 || value.Source.DVProfile != 8 || len(value.Expected.Transformations) != 0 { t.Errorf("Dolby Vision 8 scenario = %#v", value) } @@ -2047,6 +2053,230 @@ func TestPlanPlaybackV3AppliesDeliverySpecificHDRDetails(t *testing.T) { } } +func TestPlanPlaybackV3ClientManagedDynamicRangeUsesOriginalOnSDROutput(t *testing.T) { + file := detailedFixtureFileV3() + request := validStartRequestV3() + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDR = false + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{} + request.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{} + direct := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"aac"} + direct.HDRDetails = &HDRCapabilitiesV3{} + direct.ValidatedClaims = []string{ClaimClientManagedDynamicRangeV3} + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + + result := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.PlayMethod != PlayDirect { + t.Fatalf("client-managed HDR source did not reach original_http: %s", ExplainPlannerResultV3(result)) + } + if result.Plan.DecisionReason != decisionReasonClientManagedDynamicRangeV3 { + t.Fatalf("decision reason = %q, want client_managed_dynamic_range", result.Plan.DecisionReason) + } + if len(result.Plan.Transformations) != 0 { + t.Fatalf("Aether-owned routing must not be represented as a selectable transformation: %#v", result.Plan.Transformations) + } + if result.Plan.Claims.Video.HDR10 || result.Plan.Claims.Video.HDR10Plus || result.Plan.Claims.Video.HLG || result.Plan.Claims.Video.DolbyVision { + t.Fatalf("server must not invent the client's runtime output mode: %#v", result.Plan.Claims.Video) + } +} + +func TestPlanPlaybackV3ClientManagedDynamicRangeClaimDoesNotTransferToPackagedDeliveries(t *testing.T) { + for _, deliveryClass := range []string{DeliveryClassProgressiveV3, DeliveryClassHLSV3} { + t.Run(deliveryClass, func(t *testing.T) { + file := detailedFixtureFileV3() + request := validStartRequestV3() + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDR = false + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{} + request.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{} + + original := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + original.ValidatedClaims = nil + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = original + packaged := request.ClientPlaybackContext.Deliveries[deliveryClass] + packaged.ValidatedClaims = append(packaged.ValidatedClaims, ClaimClientManagedDynamicRangeV3) + request.ClientPlaybackContext.Deliveries[deliveryClass] = packaged + + result := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + }) + if result.Plan != nil { + t.Fatalf("packaged delivery inherited the original-file claim: %s", ExplainPlannerResultV3(result)) + } + if result.Terminal == nil || result.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("result = %s, want honest HDR terminal", ExplainPlannerResultV3(result)) + } + }) + } +} + +func TestPlanPlaybackV3ClientManagedDynamicRangeFailureDoesNotLoop(t *testing.T) { + file := detailedFixtureFileV3() + request := validStartRequestV3() + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{} + request.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{} + direct := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"aac"} + direct.HDRDetails = &HDRCapabilitiesV3{} + direct.ValidatedClaims = []string{ClaimClientManagedDynamicRangeV3} + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + input := PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + } + + first := PlanPlaybackV3(input) + if first.Plan == nil || first.Plan.Delivery != DeliveryOriginalHTTPV3 { + t.Fatalf("first = %s", ExplainPlannerResultV3(first)) + } + input.AttemptedKeys = []string{PlanAttemptKeyV3(*first.Plan, request.ClientPlaybackContext.Output.OutputContextID, nil)} + second := PlanPlaybackV3(input) + if second.Terminal == nil || second.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("failed original route must terminate honestly until server tone mapping exists: %s", ExplainPlannerResultV3(second)) + } +} + +func TestPlanPlaybackV3ClientManagedDynamicRangeCanHandDV7ToEngine(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].DVELPresent = true + file.VideoTracks[0].DVEnhancementLayer = "unknown" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + request := validStartRequestV3() + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{} + request.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{} + direct := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"aac"} + direct.HDRDetails = &HDRCapabilitiesV3{} + direct.ValidatedClaims = []string{ClaimClientManagedDynamicRangeV3} + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + + result := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.Plan.DecisionReason != decisionReasonClientManagedDynamicRangeV3 { + t.Fatalf("client-managed DV7 source did not reach the engine: %s", ExplainPlannerResultV3(result)) + } + if len(result.Plan.Transformations) != 0 { + t.Fatalf("engine-managed DV7 fallback unexpectedly selected a V3 transformation: %#v", result.Plan.Transformations) + } +} + +func TestPlanPlaybackV3ClientManagedDynamicRangeFollowsDV7TransformationLadder(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].DVELPresent = true + file.VideoTracks[0].DVEnhancementLayer = "unknown" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + request := validStartRequestV3() + request.ClientFeatures = append(request.ClientFeatures, FeatureClientVideoTransforms) + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{8}} + request.ClientPlaybackContext.Output.HDRDetails = request.Capabilities.HDRDetails + direct := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"aac"} + direct.HDRDetails = request.Capabilities.HDRDetails + direct.ValidatedClaims = append(direct.ValidatedClaims, ClaimClientManagedDynamicRangeV3) + direct.Transformations = []TransformationV3{ + {Name: ClientDV7ToDV81V3, Executor: ExecutorClientV3, RecipeVersion: ClientDVTransformVersionV3}, + {Name: ClientDV7ToHDR10V3, Executor: ExecutorClientV3, RecipeVersion: ClientDVTransformVersionV3}, + } + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + input := PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: NewTransformationRegistryV3(nil), + } + + first := PlanPlaybackV3(input) + if first.Plan == nil || first.Plan.DecisionReason != "client_dv7_to_dv81" { + t.Fatalf("first = %s", ExplainPlannerResultV3(first)) + } + input.AttemptedKeys = append(input.AttemptedKeys, first.Plan.PlanAttemptKey) + + second := PlanPlaybackV3(input) + if second.Plan == nil || second.Plan.DecisionReason != "client_dv7_to_hdr10" { + t.Fatalf("second = %s", ExplainPlannerResultV3(second)) + } + input.AttemptedKeys = append(input.AttemptedKeys, second.Plan.PlanAttemptKey) + + third := PlanPlaybackV3(input) + if third.Plan == nil || third.Plan.DecisionReason != decisionReasonClientManagedDynamicRangeV3 || len(third.Plan.Transformations) != 0 { + t.Fatalf("third = %s", ExplainPlannerResultV3(third)) + } + input.AttemptedKeys = append(input.AttemptedKeys, third.Plan.PlanAttemptKey) + + fourth := PlanPlaybackV3(input) + if fourth.Terminal == nil || fourth.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("fourth = %s, want exhausted HDR terminal", ExplainPlannerResultV3(fourth)) + } +} + +func TestPlanPlaybackV3ClientManagedDynamicRangeDoesNotBypassTransformationOutputLimits(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].DVELPresent = true + file.VideoTracks[0].DVEnhancementLayer = "unknown" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + request := validStartRequestV3() + request.ClientFeatures = append(request.ClientFeatures, FeatureClientVideoTransforms) + request.Capabilities.VideoEvidence = EvidenceDeclaredV3 + request.Capabilities.AudioEvidence = EvidenceDeclaredV3 + request.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + request.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + direct := request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"aac"} + direct.HDRDetails = &HDRCapabilitiesV3{HDR10: true, HDR10MaxWidth: 1920, HDR10MaxHeight: 1080} + direct.ValidatedClaims = append(direct.ValidatedClaims, ClaimClientManagedDynamicRangeV3) + direct.Transformations = []TransformationV3{{Name: ClientDV7ToHDR10V3, Executor: ExecutorClientV3, RecipeVersion: ClientDVTransformVersionV3}} + request.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + + result := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: NewTransformationRegistryV3(nil), + }) + if result.Plan == nil || result.Plan.DecisionReason != decisionReasonClientManagedDynamicRangeV3 { + t.Fatalf("delivery-level HDR limits did not reject the explicit HDR10 transformation: %s", ExplainPlannerResultV3(result)) + } + if len(result.Plan.Transformations) != 0 { + t.Fatalf("client-managed fallback unexpectedly retained a rejected transformation: %#v", result.Plan.Transformations) + } +} + func validStartRequestV3() StartRequestV3 { return StartRequestV3{ ProtocolVersion: ProtocolV3, @@ -2300,7 +2530,7 @@ func TestPlanPlaybackV3AudioOnlyHonorsBandwidthCap(t *testing.T) { } } -func TestPlanPlaybackV3NonDefaultAudioSelectionCannotUseOriginalHTTP(t *testing.T) { +func TestPlanPlaybackV3NonDefaultAudioSelectionRequiresScopedOriginalClaim(t *testing.T) { file := detailedFixtureFileV3() file.VideoTracks[0].VideoRange = "SDR" file.VideoTracks[0].VideoRangeType = "SDR" @@ -2315,11 +2545,80 @@ func TestPlanPlaybackV3NonDefaultAudioSelectionCannotUseOriginalHTTP(t *testing. if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 || result.PlayMethod != PlayRemux || result.TranscodeAudio { t.Fatalf("result = %s", ExplainPlannerResultV3(result)) } + packaged := req.ClientPlaybackContext.Deliveries[DeliveryClassProgressiveV3] + packaged.ValidatedClaims = append(packaged.ValidatedClaims, ClaimClientSelectedAudioTrackV3) + req.ClientPlaybackContext.Deliveries[DeliveryClassProgressiveV3] = packaged + result = PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 1, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 { + t.Fatalf("packaged claim leaked into original eligibility: %s", ExplainPlannerResultV3(result)) + } + direct := req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.ValidatedClaims = append(direct.ValidatedClaims, ClaimClientSelectedAudioTrackV3) + req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + result = PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 1, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.PlayMethod != PlayDirect || result.TranscodeAudio { + t.Fatalf("claimed original selection = %s", ExplainPlannerResultV3(result)) + } if result.Plan.SelectedTracks.Audio == nil || result.Plan.SelectedTracks.Audio.Index == nil || *result.Plan.SelectedTracks.Audio.Index != 1 { t.Fatalf("selected audio = %#v", result.Plan.SelectedTracks.Audio) } } +func TestPlanPlaybackV3AetherManagedHDRWithNonDefaultAudioAndPGSUsesOriginalHTTP(t *testing.T) { + file := detailedFixtureFileV3() + file.Container = "mkv" + file.Resolution = "2160p" + file.Bitrate = 77_930 + file.VideoTracks[0] = models.VideoTrack{Codec: "hevc", Profile: "Main 10", Level: 153, Width: 3840, Height: 2160, FrameRate: "23.976", BitDepth: 10, VideoRange: "HDR", VideoRangeType: "HDR10"} + file.AudioTracks = []models.AudioTrack{ + {Codec: "truehd", Channels: 6, Layout: "5.1", Default: true}, + {Codec: "ac3", Channels: 6, Layout: "5.1"}, + {Codec: "truehd", Channels: 6, Layout: "5.1"}, + } + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "hdmv_pgs_subtitle", Language: "en"}} + + req := validStartRequestV3() + req.QualityPreference = QualityOriginalV3 + req.Capabilities.VideoEvidence = EvidenceDeclaredV3 + req.Capabilities.AudioEvidence = EvidenceDeclaredV3 + req.Capabilities.CodecsVideo = []string{"hevc"} + req.Capabilities.CodecsAudio = []string{"truehd"} + req.Capabilities.Containers = []string{"mkv"} + req.Capabilities.VideoDecode = nil + req.Capabilities.HDR = false + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{} + req.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{} + direct := req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] + direct.Containers = []string{"mkv"} + direct.VideoCodecs = []string{"hevc"} + direct.AudioDecodeCodecs = []string{"truehd"} + direct.HDRDetails = &HDRCapabilitiesV3{} + direct.Subtitles.EmbeddedBitmap = true + direct.ValidatedClaims = []string{ClaimClientManagedDynamicRangeV3, ClaimClientSelectedAudioTrackV3} + req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = direct + audioIndex := 2 + subtitleIndex := 0 + req.AudioTrackIndex = &audioIndex + req.AudioTrackID = TrackIDV3(file.ID, "audio", audioIndex) + req.SubtitleTrackIndex = &subtitleIndex + req.SubtitleTrackID = TrackIDV3(file.ID, "subtitle", subtitleIndex) + + result := PlanPlaybackV3(PlannerInputV3{ + Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: audioIndex, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.PlayMethod != PlayDirect { + t.Fatalf("Aether-managed HDR regression source did not reach original HTTP: %s", ExplainPlannerResultV3(result)) + } + if result.Plan.SelectedTracks.Audio == nil || result.Plan.SelectedTracks.Audio.Index == nil || *result.Plan.SelectedTracks.Audio.Index != audioIndex { + t.Fatalf("selected audio = %#v", result.Plan.SelectedTracks.Audio) + } + if result.Plan.Subtitle.Mode != SubtitleRenderV3 || !result.Plan.Claims.Subtitles.BitmapSidecar { + t.Fatalf("selected PGS subtitle = decision %#v claims %#v", result.Plan.Subtitle, result.Plan.Claims.Subtitles) + } +} + func TestPlanPlaybackV3HLSAudioConversionHonorsChannelCeiling(t *testing.T) { file := detailedFixtureFileV3() file.VideoTracks[0].VideoRange = "SDR" diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index d0b9b3943..b2e55fb63 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -1973,6 +1973,243 @@ ] } }, + { + "name": "client_managed_hdr_selected_audio", + "category": "hdr_dv_matrix", + "request": { + "protocol_version": 3, + "client_features": [ + "playback_plan_v3" + ], + "file_id": 42, + "profile_id": "profile-1", + "playback_attempt_id": "attempt-client-managed-original", + "quality_preference": "original", + "subtitle_fidelity_preference": "compatible", + "start_position": 12.5, + "progress_persistence": "client", + "audio_track_id": "file:42:audio:1", + "audio_track_index": 1, + "metered": false, + "client_capabilities": { + "video_evidence": "exact", + "audio_evidence": "exact", + "codecs_video": [ + "hevc" + ], + "codecs_video_hardware": [ + "hevc" + ], + "codecs_audio": [ + "aac" + ], + "containers": [ + "mkv" + ], + "max_resolution": "2160p", + "hdr": false, + "hdr_details": { + "hdr10": false, + "hdr10_plus": false, + "hlg": false, + "dolby_vision_profiles": [] + }, + "video_decode": [ + { + "codec": "hevc", + "profiles": [ + "main 10" + ], + "levels": [ + 153 + ], + "bit_depths": [ + 10 + ], + "max_width": 3840, + "max_height": 2160, + "max_frame_rate": 60, + "max_bitrate_kbps": 80000, + "hardware": true + } + ] + }, + "client_playback_context": { + "protocol_version": 3, + "form_factor": "tv", + "app_version": "3.0-test", + "device": { + "platform": "fixture" + }, + "output": { + "hdr_details": { + "hdr10": false, + "hdr10_plus": false, + "hlg": false, + "dolby_vision_profiles": [] + }, + "output_context_id": "output-a" + }, + "deliveries": { + "hls": { + "enabled": true, + "supported_on_device": true, + "containers": [ + "hls" + ], + "video_codecs": [ + "hevc", + "h264" + ], + "audio_decode_codecs": [ + "aac" + ], + "audio_passthrough_codecs": [], + "subtitles": { + "embedded_text": false, + "sidecar_text": false, + "ass_styling": false, + "embedded_bitmap": false, + "sidecar_bitmap": false, + "font_attachments": false + }, + "features": [], + "auth_header_refresh": false, + "validated_claims": [], + "transformations": [] + }, + "original_http": { + "enabled": true, + "supported_on_device": true, + "containers": [ + "mkv", + "mp4" + ], + "video_codecs": [ + "hevc", + "h264" + ], + "audio_decode_codecs": [ + "aac" + ], + "audio_passthrough_codecs": [], + "hdr_details": { + "hdr10": false, + "hdr10_plus": false, + "hlg": false, + "dolby_vision_profiles": [] + }, + "subtitles": { + "embedded_text": false, + "sidecar_text": false, + "ass_styling": false, + "embedded_bitmap": false, + "sidecar_bitmap": false, + "font_attachments": false + }, + "features": [], + "auth_header_refresh": false, + "validated_claims": [ + "client_managed_dynamic_range_v1", + "client_selected_audio_track_v1" + ], + "transformations": [] + }, + "progressive": { + "enabled": true, + "supported_on_device": true, + "containers": [ + "mp4" + ], + "video_codecs": [ + "hevc", + "h264" + ], + "audio_decode_codecs": [ + "aac" + ], + "audio_passthrough_codecs": [], + "subtitles": { + "embedded_text": false, + "sidecar_text": false, + "ass_styling": false, + "embedded_bitmap": false, + "sidecar_bitmap": false, + "font_attachments": false + }, + "features": [], + "auth_header_refresh": false, + "validated_claims": [], + "transformations": [] + } + } + } + }, + "source": { + "media_file_id": 42, + "duration_seconds": 7200, + "container": "mkv", + "video_codec": "hevc", + "video_profile": "main 10", + "video_level": 153, + "bit_depth": 10, + "color_range": "tv", + "width": 3840, + "height": 2160, + "frame_rate": 23.976023976023978, + "bitrate_kbps": 60000, + "dynamic_range": "hdr10", + "hdr10_plus": false, + "dv_enhancement_layer": "none", + "audio_codec": "aac", + "audio_channels": 2, + "audio_layout": "stereo" + }, + "expected": { + "outcome": "playable", + "delivery": "original_http", + "decision_reason": "client_managed_dynamic_range", + "plan_id": "plan:880cfe7954be7aff16c5d89d2b2a09bd", + "plan_attempt_key": "v3:b020c19644d2bb36", + "selected_tracks": { + "audio": { + "id": "file:42:audio:1", + "index": 1 + } + }, + "subtitle": { + "mode": "off", + "inventory": [] + }, + "claims": { + "video": { + "hdr10": false, + "hdr10_plus": false, + "hlg": false, + "dolby_vision": false + }, + "audio": { + "codec": "aac", + "passthrough": false, + "atmos_preserved": false, + "reason": "client_decode_supported" + }, + "subtitles": { + "ass_styling_preserved": false, + "bitmap_overlay": false, + "bitmap_sidecar": false + } + }, + "available_qualities": [ + { + "label": "original", + "height": 2160, + "bitrate_kbps": 60000, + "preserves_source": true + } + ] + } + }, { "name": "dolby_vision_8_exact_direct", "category": "hdr_dv_matrix", From 0b98026b81584a7ed4c72c351863619d3415bbca Mon Sep 17 00:00:00 2001 From: blurbery <271863866+blurbery@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:59:41 +1000 Subject: [PATCH 44/44] chore: satisfy envutil lint --- internal/envutil/bool.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/envutil/bool.go b/internal/envutil/bool.go index 0cab8de15..f023db07c 100644 --- a/internal/envutil/bool.go +++ b/internal/envutil/bool.go @@ -13,12 +13,14 @@ import ( "strings" ) +const trueValue = "true" + // Truthy reports whether a raw environment value means "on". Case and // surrounding whitespace are ignored. Anything else, including an empty or // unset value, is false — a flag has to be turned on deliberately. func Truthy(value string) bool { switch strings.ToLower(strings.TrimSpace(value)) { - case "1", "true", "yes", "on", "enabled": + case "1", trueValue, "yes", "on", "enabled": return true default: return false @@ -32,7 +34,7 @@ func Bool(name string) bool { return Truthy(os.Getenv(name)) } // to def when the variable is unset or empty — whitespace-only counts as empty, // since a value that survives a shell only as spaces was never really supplied. // -// A value that IS present but unrecognised ("flase", "no", "0") reads as false +// A value that IS present but unrecognized ("flase", "no", "0") reads as false // rather than as def. For a flag that defaults on, that means a typo in the kill // switch turns the flag OFF, which is the fail-safe direction: the operator was // reaching for "off", and a mistyped disable that silently left the feature @@ -47,6 +49,6 @@ func BoolDefault(name string, def bool) bool { // IsSet reports whether the named environment variable carries a non-empty value // once surrounding whitespace is trimmed. It answers "did the operator touch this // knob?", which a default-on flag has to ask separately from "is it on?" — an -// unset knob and one explicitly set to false want different behaviour when +// unset knob and one explicitly set to false want different behavior when // something else would otherwise derive the value. func IsSet(name string) bool { return strings.TrimSpace(os.Getenv(name)) != "" }