Fix Firefox crackling during remuxed and transcoded playback

docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json

@@ -28,7 +28,7 @@
     {
       "name": "audio_to_aac",
       "executor": "server",
-      "recipe_version": "3",
+      "recipe_version": "4",
       "validated_claims": [
         "audio_decode"
       ]

internal/playback/device_quirks_v3.go

@@ -8,6 +8,7 @@ const (
 	QuirkAndroidMobileEAC3BluetoothV3 = "android.mobile.eac3_bluetooth_hls_audio_adapt_v1"
 	QuirkFireTVDV8HDR10PlusV3         = "android.fire_tv.dv8_hdr10plus_sei_v1"
 	QuirkFirefoxHEVCOpenGOPV3         = "web.firefox.hevc_open_gop_resume_v1"
+	QuirkFirefoxMatroskaAACTimingV3   = "web.firefox.matroska_aac_timestamps_v1"
 )
 
 func high10DecodeOverrideV3(source SourceDescriptorV3, request StartRequestV3) (*AppliedQuirkV3, bool) {
@@ -92,12 +93,7 @@ func dv8HDR10PlusRuntimeCorrectionV3(source SourceDescriptorV3, request StartReq
 // for a non-zero seek, but the plan freezes the quirk from the first start so a
 // later seek reanchor cannot lose the byte recipe.
 func firefoxHEVCOpenGOPQuirkV3(source SourceDescriptorV3, request StartRequestV3) (*AppliedQuirkV3, bool) {
-	device := request.ClientPlaybackContext.Device
-	if !strings.EqualFold(device.Platform, "web") || !strings.EqualFold(source.VideoCodec, "hevc") {
-		return nil, false
-	}
-	userAgent := strings.ToLower(strings.TrimSpace(device.PlatformDetails["user_agent"]))
-	if !strings.Contains(userAgent, "firefox/") || strings.Contains(userAgent, "seamonkey/") {
+	if !isFirefoxWebV3(request) || !strings.EqualFold(source.VideoCodec, "hevc") {
 		return nil, false
 	}
 	quirk := AppliedQuirkV3{
@@ -109,6 +105,35 @@ func firefoxHEVCOpenGOPQuirkV3(source SourceDescriptorV3, request StartRequestV3
 	return &quirk, true
 }
 
+// firefoxMatroskaAACTimingQuirkV3 prevents millisecond-rounded Matroska AAC
+// packet timestamps from being copied into MP4/fMP4. Firefox treats those
+// sub-frame gaps as missing audio and inserts silence, which is heard as
+// crackling. Other clients keep codec-copy remuxing, and Firefox direct play
+// remains available when its native container claim is valid.
+func firefoxMatroskaAACTimingQuirkV3(source SourceDescriptorV3, request StartRequestV3) (*AppliedQuirkV3, bool) {
+	container := strings.ToLower(strings.TrimSpace(source.Container))
+	if !isFirefoxWebV3(request) || (container != containerMKVV3 && container != "matroska") ||
+		!strings.EqualFold(strings.TrimSpace(source.AudioCodec), audioCodecAACV3) {
+		return nil, false
+	}
+	quirk := AppliedQuirkV3{
+		ID:               QuirkFirefoxMatroskaAACTimingV3,
+		RegistryRevision: DeviceQuirkRegistryRevisionV3,
+		Action:           "audio_only_transcode",
+		Reason:           "Firefox requires Matroska AAC timestamps to be normalized before MP4 or HLS packaging.",
+	}
+	return &quirk, true
+}
+
+func isFirefoxWebV3(request StartRequestV3) bool {
+	device := request.ClientPlaybackContext.Device
+	if !strings.EqualFold(device.Platform, "web") {
+		return false
+	}
+	userAgent := strings.ToLower(strings.TrimSpace(device.PlatformDetails["user_agent"]))
+	return strings.Contains(userAgent, "firefox/") && !strings.Contains(userAgent, "seamonkey/")
+}
+
 func applyFirefoxHEVCOpenGOPQuirkV3(plan *PlanV3, source SourceDescriptorV3, request StartRequestV3) bool {
 	quirk, ok := firefoxHEVCOpenGOPQuirkV3(source, request)
 	if !ok {

internal/playback/device_quirks_v3_test.go

@@ -217,6 +217,44 @@ func TestFirefoxHEVCOpenGOPQuirkIsExact(t *testing.T) {
 	}
 }
 
+func TestFirefoxMatroskaAACTimingQuirkIsExact(t *testing.T) {
+	request := validStartRequestV3()
+	request.ClientPlaybackContext.Device = DeviceContextV3{
+		Platform: "web",
+		PlatformDetails: map[string]string{
+			"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:154.0) Gecko/20100101 Firefox/154.0",
+		},
+	}
+	source := SourceDescriptorV3{Container: "mkv", AudioCodec: "aac"}
+	quirk, ok := firefoxMatroskaAACTimingQuirkV3(source, request)
+	if !ok || quirk == nil || quirk.ID != QuirkFirefoxMatroskaAACTimingV3 || quirk.Action != "audio_only_transcode" {
+		t.Fatalf("Firefox Matroska AAC quirk = %#v, ok=%v", quirk, ok)
+	}
+
+	for _, test := range []struct {
+		name      string
+		platform  string
+		userAgent string
+		container string
+		codec     string
+	}{
+		{name: "MP4 AAC", platform: "web", userAgent: "Mozilla/5.0 Firefox/154.0", container: "mp4", codec: "aac"},
+		{name: "Matroska Opus", platform: "web", userAgent: "Mozilla/5.0 Firefox/154.0", container: "mkv", codec: "opus"},
+		{name: "Chrome", platform: "web", userAgent: "Mozilla/5.0 Chrome/140.0", container: "mkv", codec: "aac"},
+		{name: "SeaMonkey", platform: "web", userAgent: "Mozilla/5.0 Firefox/128.0 SeaMonkey/2.53", container: "mkv", codec: "aac"},
+		{name: "non-web", platform: "android", userAgent: "Mozilla/5.0 Firefox/154.0", container: "mkv", codec: "aac"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			candidate := request
+			candidate.ClientPlaybackContext.Device.Platform = test.platform
+			candidate.ClientPlaybackContext.Device.PlatformDetails = map[string]string{"user_agent": test.userAgent}
+			if got, eligible := firefoxMatroskaAACTimingQuirkV3(SourceDescriptorV3{Container: test.container, AudioCodec: test.codec}, candidate); eligible || got != nil {
+				t.Fatalf("unexpected quirk = %#v, eligible=%v", got, eligible)
+			}
+		})
+	}
+}
+
 func TestPlanAttemptKeyV3DeviceQuirkIsStable(t *testing.T) {
 	width, height, bitrate := 3840, 2160, 60_000
 	plan := PlanV3{

internal/playback/plan_v3.go

@@ -21,6 +21,7 @@ type PlannerSettingsV3 struct {
 const (
 	TerminalMessage4KTranscodeDisabledV3   = "A lower-resolution source is required because 4K transcoding is disabled."
 	containerMP4V3                         = "mp4"
+	containerMKVV3                         = "mkv"
 	mimeVideoMP4V3                         = "video/mp4"
 	degradationAudioConvertedV3            = "audio_converted"
 	audioCodecAACV3                        = "aac"
@@ -454,8 +455,15 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 {
 		progressiveAudioOK := noAudioTrack || deliverySupportsAudioClaimV3(input.Request, DeliveryClassProgressiveV3, source.AudioCodec, audioClaims, audioOK)
 		hlsAudioOK := noAudioTrack || hlsNativeAudioCodecV3(source.AudioCodec) &&
 			deliverySupportsAudioClaimV3(input.Request, DeliveryClassHLSV3, source.AudioCodec, audioClaims, audioOK)
-		progressiveTranscodeAudio := !progressiveAudioOK
-		hlsTranscodeAudio := !hlsAudioOK
+		// AAC frames in Matroska use a millisecond packet clock while each frame
+		// contains 1024 samples. Copying those rounded timestamps into MP4/fMP4
+		// produces real sub-frame gaps and overlaps that Firefox renders as
+		// crackle. Keep video-copy remuxing, but re-encode the selected AAC track
+		// through the versioned timestamp-normalization recipe. Native original
+		// playback above remains byte-for-byte direct play.
+		firefoxAACTimingQuirk, normalizeMatroskaAAC := firefoxMatroskaAACTimingQuirkV3(source, input.Request)
+		progressiveTranscodeAudio := !progressiveAudioOK || normalizeMatroskaAAC
+		hlsTranscodeAudio := !hlsAudioOK || normalizeMatroskaAAC
 		hlsAudioQuirk, hlsAudioQuirkOK := hlsEAC3AudioCorrectionV3(source, input.Request)
 		localAudioConvertOK := input.Registry.Available(TransformationAudioToAACV3)
 		if progressiveTranscodeAudio && hlsTranscodeAudio {
@@ -496,6 +504,9 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 {
 			progressivePlan.DegradationWarnings = append(progressivePlan.DegradationWarnings, DegradationWarningV3{Code: degradationAudioConvertedV3, Message: fmt.Sprintf("The selected audio track is converted to AAC %s.", audioLayoutForChannelsV3(progressiveAudioChannels))})
 			progressivePlan.DecisionReason = decisionReasonAudioAdaptationV3
 		}
+		if normalizeMatroskaAAC {
+			appendAppliedQuirkV3(&progressivePlan, *firefoxAACTimingQuirk, "")
+		}
 		if !dvStrip {
 			applyCopiedVideoQuirksV3(&progressivePlan, source, input.Request, high10Quirk)
 		}
@@ -548,6 +559,9 @@ func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 {
 				plan.Transformations = append(plan.Transformations, TransformationV3{Name: TransformationAudioToAACV3, Executor: ExecutorServerV3, RecipeVersion: TransformationAudioToAACRecipeVersionV3, ValidatedClaims: []string{ClaimAudioDecodeV3}})
 				plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: degradationAudioConvertedV3, Message: "The selected audio track is converted to AAC for HLS delivery."})
 			}
+			if normalizeMatroskaAAC {
+				appendAppliedQuirkV3(&plan, *firefoxAACTimingQuirk, "")
+			}
 			if hlsAudioQuirkOK && !hlsTranscodeAudio {
 				if !input.hlsRegistry().Available(TransformationAudioToAACV3) {
 					return terminalPlannerResultV3(TerminalAudioConversionUnsupportedV3, "The device-specific HLS route requires the validated AAC conversion toolchain.", true)
@@ -986,7 +1000,7 @@ func applySubtitleDecisionV3(plan *PlanV3, decision SubtitleDecisionV3) {
 }
 
 func prefersWebHLSForMKVDolbyV3(source SourceDescriptorV3, request StartRequestV3) bool {
-	if !strings.EqualFold(source.Container, "mkv") && !strings.EqualFold(source.Container, "matroska") {
+	if !strings.EqualFold(source.Container, containerMKVV3) && !strings.EqualFold(source.Container, "matroska") {
 		return false
 	}
 	if !strings.EqualFold(strings.TrimSpace(request.ClientPlaybackContext.Device.Platform), "web") || source.DVProfile <= 0 {

internal/playback/protocol_v3.go

@@ -188,7 +188,7 @@ const (
 	TransformationHDRToSDRToneMapV3 = "hdr_to_sdr_tonemap"
 
 	TransformationVideoToH264RecipeVersionV3     = "2"
-	TransformationAudioToAACRecipeVersionV3      = "3"
+	TransformationAudioToAACRecipeVersionV3      = "4"
 	TransformationServerDV7HDR10RecipeVersionV3  = "3"
 	TransformationServerDV8BaseRecipeVersionV3   = "2"
 	TransformationHDRToSDRToneMapRecipeVersionV3 = "1"

internal/playback/protocol_v3_test.go

@@ -1045,8 +1045,8 @@ func TestPlanPlaybackV3WebHLSJSKeepsProgressiveDolbyRouteFirst(t *testing.T) {
 			if result.PlayMethod != PlayRemux || result.Plan.EffectiveRecipe.VideoCodec != "hevc" || result.Plan.EffectiveMediaFileID != file.ID {
 				t.Fatalf("the first browser route did not keep the 4K HEVC remux: %#v", result)
 			}
-			if !result.DropInitialLeadingPictures || len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFirefoxHEVCOpenGOPV3 {
-				t.Fatalf("Firefox progressive resume recipe was not frozen: %#v", result)
+			if !result.DropInitialLeadingPictures || len(result.Plan.AppliedQuirks) != 2 || result.Plan.AppliedQuirks[0].ID != QuirkFirefoxMatroskaAACTimingV3 || result.Plan.AppliedQuirks[1].ID != QuirkFirefoxHEVCOpenGOPV3 {
+				t.Fatalf("Firefox progressive audio and resume recipes were not frozen: %#v", result)
 			}
 
 			input.AttemptedKeys = []string{PlanAttemptKeyV3(*result.Plan, req.ClientPlaybackContext.Output.OutputContextID, nil)}
@@ -1057,8 +1057,8 @@ func TestPlanPlaybackV3WebHLSJSKeepsProgressiveDolbyRouteFirst(t *testing.T) {
 			if fallback.Plan.EffectiveRecipe.VideoSampleEntry != VideoSampleEntryHEV1V3 {
 				t.Fatalf("hls.js recovery sample entry = %q, want hev1", fallback.Plan.EffectiveRecipe.VideoSampleEntry)
 			}
-			if !fallback.DropInitialLeadingPictures || len(fallback.Plan.AppliedQuirks) != 1 || fallback.Plan.AppliedQuirks[0].ID != QuirkFirefoxHEVCOpenGOPV3 {
-				t.Fatalf("Firefox HLS resume recipe was not frozen: %#v", fallback)
+			if !fallback.DropInitialLeadingPictures || len(fallback.Plan.AppliedQuirks) != 2 || fallback.Plan.AppliedQuirks[0].ID != QuirkFirefoxMatroskaAACTimingV3 || fallback.Plan.AppliedQuirks[1].ID != QuirkFirefoxHEVCOpenGOPV3 {
+				t.Fatalf("Firefox HLS audio and resume recipes were not frozen: %#v", fallback)
 			}
 		})
 	}
@@ -3526,6 +3526,125 @@ func TestPlanPlaybackV3VideoRemuxAdaptsProgressiveWhenHLSVideoUnsupported(t *tes
 	}
 }
 
+func TestPlanPlaybackV3MatroskaAACRemuxUsesTimestampNormalizedAudio(t *testing.T) {
+	file := detailedFixtureFileV3()
+	file.CodecVideo = "h264"
+	file.Resolution = "1080p"
+	file.Bitrate = 4_244
+	file.VideoTracks[0] = models.VideoTrack{Codec: "h264", Profile: "High", Level: 40, Width: 1920, Height: 804, FrameRate: "25", BitDepth: 8, VideoRange: "SDR", VideoRangeType: "SDR"}
+	file.AudioTracks[0] = models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", SampleRate: 48_000}
+
+	req := validStartRequestV3()
+	req.Capabilities.VideoEvidence = EvidenceDeclaredV3
+	req.Capabilities.AudioEvidence = EvidenceDeclaredV3
+	req.ClientPlaybackContext.FormFactor = "desktop"
+	req.ClientPlaybackContext.Device = DeviceContextV3{Platform: "web", PlatformDetails: map[string]string{"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:154.0) Gecko/20100101 Firefox/154.0"}}
+	req.Capabilities.CodecsVideo = []string{"h264"}
+	req.Capabilities.CodecsAudio = []string{"aac"}
+	req.Capabilities.Containers = []string{"mp4"}
+	req.Capabilities.MaxResolution = "1080p"
+	delete(req.ClientPlaybackContext.Deliveries, DeliveryClassOriginalHTTPV3)
+	for _, delivery := range []string{DeliveryClassProgressiveV3, DeliveryClassHLSV3} {
+		capability := req.ClientPlaybackContext.Deliveries[delivery]
+		if delivery == DeliveryClassProgressiveV3 {
+			capability.Containers = []string{"mp4"}
+		} else {
+			capability.Containers = []string{"hls"}
+		}
+		capability.VideoCodecs = []string{"h264"}
+		capability.AudioDecodeCodecs = []string{"aac"}
+		req.ClientPlaybackContext.Deliveries[delivery] = capability
+	}
+
+	result := PlanPlaybackV3(PlannerInputV3{
+		Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0,
+		Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3(),
+	})
+	if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 || result.PlayMethod != PlayRemux || !result.TranscodeAudio || result.TargetAudioCodec != "aac" {
+		t.Fatalf("result = %s", ExplainPlannerResultV3(result))
+	}
+	if result.Plan.DecisionReason != decisionReasonAudioAdaptationV3 || len(result.Plan.Transformations) != 1 || result.Plan.Transformations[0].Name != TransformationAudioToAACV3 || result.Plan.Transformations[0].RecipeVersion != TransformationAudioToAACRecipeVersionV3 || len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFirefoxMatroskaAACTimingV3 {
+		t.Fatalf("normalized AAC plan = %#v", result.Plan)
+	}
+}
+
+func TestPlanPlaybackV3NativeMatroskaAACDirectPlayRemainsUnchanged(t *testing.T) {
+	file := detailedFixtureFileV3()
+	file.VideoTracks[0].VideoRange = "SDR"
+	file.VideoTracks[0].VideoRangeType = "SDR"
+	req := validStartRequestV3()
+	req.Capabilities.VideoEvidence = EvidenceDeclaredV3
+	req.Capabilities.AudioEvidence = EvidenceDeclaredV3
+	req.ClientPlaybackContext.FormFactor = "desktop"
+	req.ClientPlaybackContext.Device = DeviceContextV3{Platform: "web", PlatformDetails: map[string]string{"user_agent": "Mozilla/5.0 Firefox/154.0"}}
+	original := req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3]
+	original.Containers = []string{"mkv"}
+	original.VideoCodecs = []string{"hevc"}
+	original.AudioDecodeCodecs = []string{"aac"}
+	req.ClientPlaybackContext.Deliveries[DeliveryClassOriginalHTTPV3] = original
+
+	result := PlanPlaybackV3(PlannerInputV3{
+		Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0,
+		Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3(),
+	})
+	if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.PlayMethod != PlayDirect || result.TranscodeAudio || len(result.Plan.Transformations) != 0 {
+		t.Fatalf("direct play changed = %s", ExplainPlannerResultV3(result))
+	}
+}
+
+func TestPlanPlaybackV3FirefoxIncompatibleAudioCodecsUseNormalizedAACRecipe(t *testing.T) {
+	for _, test := range []struct {
+		codec    string
+		channels int
+	}{
+		{codec: "dts", channels: 6},
+		{codec: "eac3", channels: 6},
+		{codec: "ac3", channels: 6},
+		{codec: "truehd", channels: 8},
+		{codec: "opus", channels: 2},
+		{codec: "vorbis", channels: 2},
+		{codec: "flac", channels: 2},
+	} {
+		t.Run(test.codec, func(t *testing.T) {
+			file := detailedFixtureFileV3()
+			file.CodecVideo = "h264"
+			file.CodecAudio = test.codec
+			file.Resolution = "1080p"
+			file.VideoTracks[0] = models.VideoTrack{Codec: "h264", Profile: "High", Level: 40, Width: 1920, Height: 1080, FrameRate: "24", BitDepth: 8, VideoRange: "SDR", VideoRangeType: "SDR"}
+			file.AudioTracks[0] = models.AudioTrack{Codec: test.codec, Channels: test.channels}
+
+			req := validStartRequestV3()
+			req.Capabilities.VideoEvidence = EvidenceDeclaredV3
+			req.Capabilities.AudioEvidence = EvidenceDeclaredV3
+			req.ClientPlaybackContext.FormFactor = "desktop"
+			req.ClientPlaybackContext.Device = DeviceContextV3{Platform: "web", PlatformDetails: map[string]string{"user_agent": "Mozilla/5.0 Firefox/154.0"}}
+			req.Capabilities.CodecsVideo = []string{"h264"}
+			req.Capabilities.CodecsAudio = []string{"aac"}
+			req.Capabilities.Containers = []string{"mp4"}
+			delete(req.ClientPlaybackContext.Deliveries, DeliveryClassOriginalHTTPV3)
+			for _, delivery := range []string{DeliveryClassProgressiveV3, DeliveryClassHLSV3} {
+				capability := req.ClientPlaybackContext.Deliveries[delivery]
+				if delivery == DeliveryClassProgressiveV3 {
+					capability.Containers = []string{"mp4"}
+				} else {
+					capability.Containers = []string{"hls"}
+				}
+				capability.VideoCodecs = []string{"h264"}
+				capability.AudioDecodeCodecs = []string{"aac"}
+				req.ClientPlaybackContext.Deliveries[delivery] = capability
+			}
+
+			result := PlanPlaybackV3(PlannerInputV3{
+				Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0,
+				Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3(),
+			})
+			if result.Plan == nil || result.PlayMethod != PlayRemux || !result.TranscodeAudio || result.TargetAudioCodec != "aac" || len(result.Plan.Transformations) != 1 || result.Plan.Transformations[0].RecipeVersion != TransformationAudioToAACRecipeVersionV3 {
+				t.Fatalf("result = %s", ExplainPlannerResultV3(result))
+			}
+		})
+	}
+}
+
 func TestPlanPlaybackV3VideoRemuxHonorsProgressiveAudioPassthrough(t *testing.T) {
 	file := detailedFixtureFileV3()
 	file.CodecAudio = "ac3"

internal/playback/remux.go

@@ -260,9 +260,7 @@ func buildRemuxArgsWithAudioV3(filePath, outputFormat string, seekSeconds float6
 			"-ac", strconv.Itoa(channels),
 			"-b:a", strconv.Itoa(bitrateKbps)+"k",
 		)
-		if IsAudioToAACStereoDownmixV3(sourceAudioChannels, "aac", targetAudioChannels) {
-			args = appendStereoDownmixBoostArgs(args, sourceAudioChannels, channels)
-		}
+		args = appendAACEncodeFilterArgs(args, sourceAudioChannels, "aac", targetAudioChannels, channels)
 	} else {
 		args = append(args, "-c", "copy")
 	}