From 83f973f8dc52872b299fcb477b56d94d572e8159 Mon Sep 17 00:00:00 2001 From: John Wallace Date: Sat, 15 Aug 2026 06:57:18 -0400 Subject: [PATCH 1/3] Fix Zoom audio pacing and responsive meters --- .../AudioMeterScaleTests.cs | 32 +++++++ .../Controls/AudioLevelMeter.xaml.cs | 42 ++++++---- .../Models/AudioMeterScale.cs | 42 ++++++++++ native/src/modules/AudioDsp.h | 70 ++++++++++++++-- native/src/modules/Interfaces.h | 6 ++ native/src/modules/ZoomEngineRuntime.cpp | 7 ++ native/tests/AudioDspTest.cpp | 84 +++++++++++++++++++ native/tests/ZoomEngineRuntimeTest.cpp | 1 + 8 files changed, 263 insertions(+), 21 deletions(-) diff --git a/native-shell/CoreVideoPro.WinUI.Tests/AudioMeterScaleTests.cs b/native-shell/CoreVideoPro.WinUI.Tests/AudioMeterScaleTests.cs index c4a91df1..06c6fbf2 100644 --- a/native-shell/CoreVideoPro.WinUI.Tests/AudioMeterScaleTests.cs +++ b/native-shell/CoreVideoPro.WinUI.Tests/AudioMeterScaleTests.cs @@ -20,4 +20,36 @@ public void ToLevel_UsesCalibratedMinus60ToZeroDbfsScale(double dbfs, int expect [Fact] public void ToLevel_MutedAlwaysReportsSilence() => Assert.Equal(0, AudioMeterScale.ToLevel(-3, muted: true)); + + [Theory] + [InlineData(324)] + [InlineData(576)] + [InlineData(900)] + public void FitVerticalSegments_FillsAvailableWindowHeight(double availableHeight) + { + var layout = AudioMeterScale.FitVerticalSegments(availableHeight, 36); + + Assert.Equal(36, layout.SegmentCount); + Assert.Equal(availableHeight, layout.OccupiedSize, precision: 6); + } + + [Fact] + public void FitVerticalSegments_GrowsSegmentsInTallWindow() + { + var compact = AudioMeterScale.FitVerticalSegments(324, 36); + var tall = AudioMeterScale.FitVerticalSegments(576, 36); + + Assert.True(tall.SegmentSize > compact.SegmentSize); + Assert.True(tall.SegmentSize > 7); + } + + [Fact] + public void FitVerticalSegments_ReducesResolutionRatherThanOverflowingShortWindow() + { + var layout = AudioMeterScale.FitVerticalSegments(72, 36); + + Assert.True(layout.SegmentCount < 36); + Assert.True(layout.SegmentSize >= 2); + Assert.Equal(72, layout.OccupiedSize, precision: 6); + } } diff --git a/native-shell/CoreVideoPro.WinUI/Controls/AudioLevelMeter.xaml.cs b/native-shell/CoreVideoPro.WinUI/Controls/AudioLevelMeter.xaml.cs index ae101425..dff6671c 100644 --- a/native-shell/CoreVideoPro.WinUI/Controls/AudioLevelMeter.xaml.cs +++ b/native-shell/CoreVideoPro.WinUI/Controls/AudioLevelMeter.xaml.cs @@ -253,26 +253,40 @@ private void RenderSegments() double segMain = IsVertical ? 7 : 4; if (availableMain > 0) { - if ((segMain + spacing) * count > availableMain) + if (IsVertical) { - spacing = 1; - } - var perSegment = (availableMain - spacing * (count - 1)) / count; - if (perSegment < segMain) - { - segMain = Math.Max(2, Math.Floor(perSegment)); - } - var fits = (int)Math.Floor((availableMain + spacing) / (segMain + spacing)); - if (fits < count && fits >= 4) - { - // Too small even at 2px segments: re-quantize onto fewer segments - // so the meter stays proportional instead of clipping. - count = fits; + // Keep the segment bar and the dB scale on the same responsive + // height instead of pinning a fixed-size bar to the bottom. + var layout = Models.AudioMeterScale.FitVerticalSegments(availableMain, count); + count = layout.SegmentCount; + segMain = layout.SegmentSize; + spacing = layout.Spacing; activeSegments = (int)Math.Round(level / 100.0 * count, MidpointRounding.AwayFromZero); peakSegment = _peakLevel > 0 ? (int)Math.Round(Math.Clamp(_peakLevel, 0, 100) / 100.0 * count, MidpointRounding.AwayFromZero) - 1 : -1; } + else + { + if ((segMain + spacing) * count > availableMain) + { + spacing = 1; + } + var perSegment = (availableMain - spacing * (count - 1)) / count; + if (perSegment < segMain) + { + segMain = Math.Max(2, Math.Floor(perSegment)); + } + var fits = (int)Math.Floor((availableMain + spacing) / (segMain + spacing)); + if (fits < count && fits >= 4) + { + count = fits; + activeSegments = (int)Math.Round(level / 100.0 * count, MidpointRounding.AwayFromZero); + peakSegment = _peakLevel > 0 + ? (int)Math.Round(Math.Clamp(_peakLevel, 0, 100) / 100.0 * count, MidpointRounding.AwayFromZero) - 1 + : -1; + } + } } var panel = new StackPanel diff --git a/native-shell/CoreVideoPro.WinUI/Models/AudioMeterScale.cs b/native-shell/CoreVideoPro.WinUI/Models/AudioMeterScale.cs index 5df1ba89..e19fcc7a 100644 --- a/native-shell/CoreVideoPro.WinUI/Models/AudioMeterScale.cs +++ b/native-shell/CoreVideoPro.WinUI/Models/AudioMeterScale.cs @@ -23,4 +23,46 @@ public static int ToLevel(double dbfs, bool muted = false) var normalized = (dbfs - MinimumDbfs) / (MaximumDbfs - MinimumDbfs); return Math.Clamp((int)Math.Round(normalized * 100), 0, 100); } + + /// + /// Fits a segmented vertical meter to its current viewport. Full-size + /// consoles keep the requested resolution and grow the segment bodies; + /// compact windows reduce the segment count only when 2 px bodies no + /// longer fit. The returned stack always occupies the available height. + /// + public static AudioMeterSegmentLayout FitVerticalSegments(double availableHeight, int requestedCount) + { + var count = Math.Clamp(requestedCount, 8, 48); + if (!double.IsFinite(availableHeight) || availableHeight <= 0) + { + return new AudioMeterSegmentLayout(count, 7, 2); + } + + const double minimumSegmentSize = 2; + var spacing = availableHeight / count >= 5 ? 2d : 1d; + var segmentSize = (availableHeight - spacing * (count - 1)) / count; + + if (segmentSize < minimumSegmentSize) + { + spacing = 1; + count = Math.Min(count, Math.Max(1, + (int)Math.Floor((availableHeight + spacing) / (minimumSegmentSize + spacing)))); + + if (count == 1) + { + spacing = 0; + } + + segmentSize = Math.Max(0, + (availableHeight - spacing * (count - 1)) / count); + } + + return new AudioMeterSegmentLayout(count, segmentSize, spacing); + } +} + +public readonly record struct AudioMeterSegmentLayout(int SegmentCount, double SegmentSize, double Spacing) +{ + public double OccupiedSize => + SegmentCount * SegmentSize + Math.Max(0, SegmentCount - 1) * Spacing; } diff --git a/native/src/modules/AudioDsp.h b/native/src/modules/AudioDsp.h index f7aeca95..dc653029 100644 --- a/native/src/modules/AudioDsp.h +++ b/native/src/modules/AudioDsp.h @@ -57,6 +57,8 @@ inline std::vector coalescePcmAudioFramesBySource(std::vectorsecond]; if (target.sampleRate == frame.sampleRate && target.channels == frame.channels) { + target.requiresSteadyFeedPriming = + target.requiresSteadyFeedPriming || frame.requiresSteadyFeedPriming; target.pcm.insert(target.pcm.end(), frame.pcm.begin(), frame.pcm.end()); target.sampleCount = static_cast(target.pcm.size() / static_cast(target.channels)); continue; @@ -585,6 +587,14 @@ struct AudioFeedState { // 3.1% vs video before the pacer catch-up fix, silently). size_t shedSamples = 0; size_t shedEvents = 0; + // Zoom's SDK packet clock is 10 ms while the program worker is 20 ms. Prime + // to TWO complete worker ticks before the first emission, then retain the + // remaining tick as the same permanent cushion used by the video frame sync. + // Non-Zoom producers do not opt in and remain pass-through. + bool primed = false; + bool primingRequired = false; + size_t primeEvents = 0; + size_t partialBlocksPrevented = 0; }; inline void applyResumeFadeIn(AudioFeedState& state, float* interleaved, size_t samples, size_t channels) { @@ -604,11 +614,12 @@ inline void applyResumeFadeIn(AudioFeedState& state, float* interleaved, size_t state.fadeInRemaining -= fadeFrames; } -// Block RESHAPER, zero added latency: whatever arrives is emitted immediately -// up to ONE tick of samples; any surplus (a tick that caught two packets) -// carries forward and fills the next short/empty tick. This converts the -// observed 480/1440/480 arrival jitter into steady full ticks without holding -// audio hostage (single bursts — and unit tests — pass straight through). +// Block reshaper. Ordinary sources retain zero-added-latency pass-through. +// Zoom sources opt into a permanent one-tick reserve: wait until two ticks are +// buffered, emit one exact tick, and leave one in reserve. This converts the +// observed 480/1440/480 arrival jitter into full 960-frame blocks without a +// 10 ms hole reaching the bus. A real starvation disarms the source so its next +// talk spurt re-primes instead of leaking another partial block. inline void steadyAudioFrameFeed(std::vector& frames, std::map& states, double ticksPerSecond = 50.0) { @@ -623,14 +634,36 @@ inline void steadyAudioFrameFeed(std::vector& frames, state.fifo.clear(); // format change: restart at the new layout state.sampleRate = frame.sampleRate; state.channels = frame.channels; + state.primed = false; } + state.primingRequired = state.primingRequired || frame.requiresSteadyFeedPriming; if (!frame.pcm.empty()) { state.fifo.insert(state.fifo.end(), frame.pcm.begin(), frame.pcm.end()); } const size_t tickSamples = static_cast(frame.sampleRate / ticksPerSecond) * static_cast(frame.channels); - const size_t emit = std::min(state.fifo.size(), tickSamples); + size_t emit = 0; + if (!state.primingRequired) { + emit = std::min(state.fifo.size(), tickSamples); + } else { + if (!state.primed && state.fifo.size() >= tickSamples * 2) { + state.primed = true; + ++state.primeEvents; + if (state.primeEvents == 1 || state.primeEvents % 100 == 0) { + std::fprintf(stderr, + "[audio] steady feed primed %s at %zu frames (%zu event%s)\n", + frame.participantId.c_str(), + state.fifo.size() / static_cast(frame.channels), + state.primeEvents, state.primeEvents == 1 ? "" : "s"); + } + } + if (state.primed && state.fifo.size() >= tickSamples) { + emit = tickSamples; + } else if (!state.fifo.empty()) { + ++state.partialBlocksPrevented; + } + } frame.pcm.assign(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); frame.sampleCount = static_cast(emit / static_cast(frame.channels)); state.fifo.erase(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); @@ -643,6 +676,9 @@ inline void steadyAudioFrameFeed(std::vector& frames, state.hasEverEmitted = true; } state.lastEmitted = emit; + if (state.primingRequired && emit == 0 && state.fifo.empty()) { + state.primed = false; + } // Cap runaway accumulation (device clock slightly fast): keep at most // 6 ticks buffered by dropping the OLDEST audio. The drop MUST be frame- @@ -684,11 +720,31 @@ inline void steadyAudioFrameFeed(std::vector& frames, } if (state.fifo.empty()) { state.lastEmitted = 0; // dry: the NEXT flow onset gets a declick fade + if (state.primingRequired) { + state.primed = false; + } continue; } const size_t tickSamples = static_cast(state.sampleRate / ticksPerSecond) * static_cast(state.channels); - const size_t emit = std::min(state.fifo.size(), tickSamples); + size_t emit = 0; + if (!state.primingRequired) { + emit = std::min(state.fifo.size(), tickSamples); + } else { + if (!state.primed && state.fifo.size() >= tickSamples * 2) { + state.primed = true; + ++state.primeEvents; + } + if (state.primed && state.fifo.size() >= tickSamples) { + emit = tickSamples; + } else { + ++state.partialBlocksPrevented; + } + } + if (emit == 0) { + state.lastEmitted = 0; + continue; + } AudioFrame fill; fill.participantId = sourceId; fill.sampleRate = state.sampleRate; diff --git a/native/src/modules/Interfaces.h b/native/src/modules/Interfaces.h index 0f91f49a..ad43444e 100644 --- a/native/src/modules/Interfaces.h +++ b/native/src/modules/Interfaces.h @@ -114,6 +114,12 @@ struct AudioFrame { double peakLevel = 0.0; double noiseFloorDb = -60.0; bool voiceActive = true; + // Zoom raw audio arrives as 10 ms SDK packets while the program mixer runs + // on a 20 ms clock. Those independent clocks need a permanent one-tick + // cushion, just like Zoom video keeps a one-frame sync cushion. Producers + // that set this flag opt into full-tick priming in steadyAudioFrameFeed; + // local/media/capture sources keep their existing zero-latency behavior. + bool requiresSteadyFeedPriming = false; // Optional interleaved float PCM payload in full-scale range [-1, 1] with // `channels` channels (so `pcm.size()` is `sampleCount * channels` when // present). When non-empty, the audio DSP core measures real RMS/peak from diff --git a/native/src/modules/ZoomEngineRuntime.cpp b/native/src/modules/ZoomEngineRuntime.cpp index 080f996b..5338304a 100644 --- a/native/src/modules/ZoomEngineRuntime.cpp +++ b/native/src/modules/ZoomEngineRuntime.cpp @@ -409,6 +409,12 @@ std::vector ZoomEngineRuntime::pollCompositorAudioFrames(int64_t tim drainAudioStreamLocked(uuid, ref); } auto frames = state_.pollCompositorAudioFrames(timestampMs); + for (auto& frame : frames) { + // Zoom delivers 10 ms raw packets into a 20 ms program clock. Opt every + // Zoom channel (meeting mix and participant ISO) into the full-tick feed + // cushion; other audio producers keep their zero-latency behavior. + frame.requiresSteadyFeedPriming = true; + } // Overlay real decoded PCM: a participant with pending audio gets ONE // coalesced PCM frame (all samples ingested since the last poll), replacing // the metadata-only placeholder the state emits from packet counters. @@ -424,6 +430,7 @@ std::vector ZoomEngineRuntime::pollCompositorAudioFrames(int64_t tim frame.channels = pending.channels; frame.timestampMs = timestampMs; frame.sampleCount = static_cast(pending.pcm.size() / static_cast(pending.channels)); + frame.requiresSteadyFeedPriming = true; frame.pcm = std::move(pending.pcm); pending.pcm.clear(); // defined-empty after the move const auto existing = std::find_if(frames.begin(), frames.end(), [&](const AudioFrame& candidate) { diff --git a/native/tests/AudioDspTest.cpp b/native/tests/AudioDspTest.cpp index 8b5bb6dc..10d07b45 100644 --- a/native/tests/AudioDspTest.cpp +++ b/native/tests/AudioDspTest.cpp @@ -1052,6 +1052,90 @@ TEST(AudioDsp, SteadyFeedReshapesJitteryPacketsWithoutHoldingAudio) { } } +TEST(AudioDsp, ZoomSteadyFeedPrimesOneTickAndNeverEmitsPartialBlocks) { + std::map states; + float sequence = 0.0f; + const auto makeFrame = [&sequence](int samples) { + corevideo::modules::AudioFrame frame; + frame.participantId = "zoom-iso"; + frame.sampleRate = 48000; + frame.channels = 1; + frame.requiresSteadyFeedPriming = true; + for (int index = 0; index < samples; ++index) { + frame.pcm.push_back(sequence++); + } + frame.sampleCount = samples; + return frame; + }; + + std::vector emitted; + const auto tick = [&](int packetSamples) { + std::vector frames; + if (packetSamples > 0) { + frames.push_back(makeFrame(packetSamples)); + } + corevideo::modules::steadyAudioFrameFeed(frames, states); + for (const auto& frame : frames) { + if (!frame.pcm.empty()) { + EXPECT_EQ(frame.pcm.size(), 960u); + emitted.insert(emitted.end(), frame.pcm.begin(), frame.pcm.end()); + } + } + }; + + tick(480); // first 10 ms SDK packet is held, never sent as a short bus + EXPECT_TRUE(emitted.empty()); + tick(1440); // two ticks buffered: emit 20 ms and retain a 20 ms cushion + EXPECT_EQ(emitted.size(), 960u); + tick(480); // 10 ms arrival + cushion still emits one complete tick + tick(480); + EXPECT_EQ(emitted.size(), 2880u); + for (size_t index = 0; index < emitted.size(); ++index) { + EXPECT_EQ(emitted[index], static_cast(index)); + } + EXPECT_TRUE(states["zoom-iso"].primed); + EXPECT_EQ(states["zoom-iso"].primeEvents, 1u); + EXPECT_GE(states["zoom-iso"].partialBlocksPrevented, 1u); +} + +TEST(AudioDsp, ZoomSteadyFeedReprimesAfterARealGap) { + std::map states; + const auto makeFrame = [](int samples) { + corevideo::modules::AudioFrame frame; + frame.participantId = "zoom-iso"; + frame.sampleRate = 48000; + frame.channels = 1; + frame.requiresSteadyFeedPriming = true; + frame.pcm.assign(static_cast(samples), 0.25f); + frame.sampleCount = samples; + return frame; + }; + const auto tick = [&](int samples) { + std::vector frames; + frames.push_back(makeFrame(samples)); + corevideo::modules::steadyAudioFrameFeed(frames, states); + return frames; + }; + + (void)tick(960); // held for the cushion + auto out = tick(960); // first exact block + ASSERT_EQ(out[0].pcm.size(), 960u); + out = tick(0); // drain the reserve + ASSERT_EQ(out[0].pcm.size(), 960u); + out = tick(0); // genuinely dry: disarm + EXPECT_TRUE(out[0].pcm.empty()); + EXPECT_FALSE(states["zoom-iso"].primed); + + out = tick(480); // resumed half-block is held + EXPECT_TRUE(out[0].pcm.empty()); + out = tick(1440); // re-prime to two ticks, then emit exactly one + ASSERT_EQ(out[0].pcm.size(), 960u); + EXPECT_TRUE(states["zoom-iso"].primed); + EXPECT_EQ(states["zoom-iso"].primeEvents, 2u); + EXPECT_LT(out[0].pcm[0], 0.02f); // existing 5 ms resume fade still applies + EXPECT_EQ(out[0].pcm[400], 0.25f); +} + TEST(SpscRing, PushPopWrapFullAndDryProperties) { corevideo::modules::SpscRing ring(8); // tiny capacity to force wrap + full float in[16]; diff --git a/native/tests/ZoomEngineRuntimeTest.cpp b/native/tests/ZoomEngineRuntimeTest.cpp index 5df3aa4d..d6309cae 100644 --- a/native/tests/ZoomEngineRuntimeTest.cpp +++ b/native/tests/ZoomEngineRuntimeTest.cpp @@ -525,6 +525,7 @@ TEST(ZoomEngineRuntime, IngestsIsoAudioPcmFromSharedMemoryIntoCompositorPoll) { EXPECT_EQ(found->sampleRate, 48000); EXPECT_EQ(found->channels, 1); EXPECT_EQ(found->sampleCount, 4); + EXPECT_TRUE(found->requiresSteadyFeedPriming); ASSERT_TRUE(found->pcm.size() == 4u); EXPECT_EQ(found->pcm[1], 0.5f); EXPECT_EQ(found->pcm[2], -0.5f); From e244ad6546c72d3edef232cc552aeeece65470f1 Mon Sep 17 00:00:00 2001 From: John Wallace Date: Sat, 15 Aug 2026 07:22:33 -0400 Subject: [PATCH 2/3] Fix Zoom ISO jitter buffer drift --- docs/operator-validation-runbook.md | 11 ++- native/src/modules/AudioDsp.h | 132 ++++++++++++++++++++++++++-- native/tests/AudioDspTest.cpp | 46 +++++++++- 3 files changed, 178 insertions(+), 11 deletions(-) diff --git a/docs/operator-validation-runbook.md b/docs/operator-validation-runbook.md index ab343ae6..27b78628 100644 --- a/docs/operator-validation-runbook.md +++ b/docs/operator-validation-runbook.md @@ -156,12 +156,19 @@ Troubleshooting: 1. Ensure sign-in from step 8 (required for external-account meetings). 2. Enter meeting URL (and passcode if needed) in the join UI. -3. After join, verify: +3. Wait for **Zoom Live**, then turn **Capture** on. Joining establishes the + Meeting SDK session; Capture is the separate control that subscribes and + forwards raw participant video/audio into CoreVideo Pro. +4. Verify Capture is genuinely running before evaluating the show: + - `engineOn` is `true` in control state / diagnostics. + - Participant `video_frame_received` counters advance for every expected feed. + - Expected participant audio callbacks and meters advance when each guest speaks. +5. After join and Capture On, verify: - **Participants** roster populates (not empty / not stale stub). - **Program** tile shows video (GPU/full-res if D3D11 interop works, else CPU/BGRA preview). - **Breakout room** name updates when you move rooms. - Diagnostics show live Zoom capabilities when the dev SDK path is active (`zoom-raw-video`, `zoom-raw-audio`). -4. **Leave** meeting; roster clears, meeting state returns to idle. +6. **Leave** meeting; roster clears, meeting state returns to idle. ## 10. Headless live Zoom harness (optional) diff --git a/native/src/modules/AudioDsp.h b/native/src/modules/AudioDsp.h index dc653029..555a708e 100644 --- a/native/src/modules/AudioDsp.h +++ b/native/src/modules/AudioDsp.h @@ -595,8 +595,58 @@ struct AudioFeedState { bool primingRequired = false; size_t primeEvents = 0; size_t partialBlocksPrevented = 0; + // The Zoom callback clock and the program worker clock are both nominally + // 48 kHz, but their scheduling phase is independent. A strict 960-frame + // dequeue lets the one-tick reserve random-walk to zero when polls alternate + // between 480 and 1440 frames. We therefore consume a few frames either + // side of nominal and resample that tiny window back to one exact output + // tick. This is the same elastic-jitter-buffer principle used by realtime + // playout engines: preserve cadence without inserting periodic 20-40 ms + // holes. The correction is bounded to 2.5% per tick. + size_t clockCorrectionEvents = 0; + size_t clockCorrectionInputFrames = 0; + size_t emptyInputTicks = 0; }; +inline std::vector resampleInterleavedBlockToFrames( + const float* input, size_t inputFrames, size_t outputFrames, size_t channels) { + std::vector output; + if (input == nullptr || inputFrames == 0 || outputFrames == 0 || channels == 0) { + return output; + } + output.resize(outputFrames * channels); + if (inputFrames == outputFrames) { + std::copy(input, input + inputFrames * channels, output.begin()); + return output; + } + if (inputFrames == 1 || outputFrames == 1) { + for (size_t frame = 0; frame < outputFrames; ++frame) { + for (size_t channel = 0; channel < channels; ++channel) { + output[frame * channels + channel] = input[channel]; + } + } + return output; + } + + // Map the complete input block onto the complete output block. Adjacent + // blocks remain ordered: after this call the FIFO erases exactly inputFrames + // and the next block begins at the following source frame. + const double scale = static_cast(inputFrames - 1) / + static_cast(outputFrames - 1); + for (size_t frame = 0; frame < outputFrames; ++frame) { + const double position = static_cast(frame) * scale; + const size_t base = static_cast(position); + const size_t next = (std::min)(base + 1, inputFrames - 1); + const float fraction = static_cast(position - static_cast(base)); + for (size_t channel = 0; channel < channels; ++channel) { + const float a = input[base * channels + channel]; + const float b = input[next * channels + channel]; + output[frame * channels + channel] = a + (b - a) * fraction; + } + } + return output; +} + inline void applyResumeFadeIn(AudioFeedState& state, float* interleaved, size_t samples, size_t channels) { if (state.fadeInRemaining == 0 || channels == 0 || samples == 0) { return; @@ -637,13 +687,18 @@ inline void steadyAudioFrameFeed(std::vector& frames, state.primed = false; } state.primingRequired = state.primingRequired || frame.requiresSteadyFeedPriming; - if (!frame.pcm.empty()) { + const bool hadInputPcm = !frame.pcm.empty(); + if (hadInputPcm) { state.fifo.insert(state.fifo.end(), frame.pcm.begin(), frame.pcm.end()); } + if (state.primingRequired) { + state.emptyInputTicks = hadInputPcm ? 0 : state.emptyInputTicks + 1; + } const size_t tickSamples = static_cast(frame.sampleRate / ticksPerSecond) * static_cast(frame.channels); size_t emit = 0; + size_t consume = 0; if (!state.primingRequired) { emit = std::min(state.fifo.size(), tickSamples); } else { @@ -660,13 +715,40 @@ inline void steadyAudioFrameFeed(std::vector& frames, } if (state.primed && state.fifo.size() >= tickSamples) { emit = tickSamples; + const size_t channels = static_cast(frame.channels); + const size_t tickFrames = tickSamples / channels; + const size_t availableFrames = state.fifo.size() / channels; + const size_t reserveFrames = tickFrames; + const size_t desiredFrames = availableFrames > reserveFrames + ? availableFrames - reserveFrames + : 0; + const size_t maxCorrectionFrames = (std::max)(size_t{1}, tickFrames / 40); + const size_t minConsumeFrames = tickFrames > maxCorrectionFrames + ? tickFrames - maxCorrectionFrames + : 1; + const size_t maxConsumeFrames = tickFrames + maxCorrectionFrames; + const size_t consumeFrames = (std::min)( + availableFrames, + (std::max)(minConsumeFrames, (std::min)(desiredFrames, maxConsumeFrames))); + consume = consumeFrames * channels; } else if (!state.fifo.empty()) { ++state.partialBlocksPrevented; } } - frame.pcm.assign(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + if (!state.primingRequired) { + consume = emit; + } + if (emit > 0 && consume != emit) { + frame.pcm = resampleInterleavedBlockToFrames( + state.fifo.data(), consume / static_cast(frame.channels), + emit / static_cast(frame.channels), static_cast(frame.channels)); + ++state.clockCorrectionEvents; + state.clockCorrectionInputFrames += consume / static_cast(frame.channels); + } else { + frame.pcm.assign(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + } frame.sampleCount = static_cast(emit / static_cast(frame.channels)); - state.fifo.erase(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + state.fifo.erase(state.fifo.begin(), state.fifo.begin() + static_cast(consume)); if (emit > 0 && state.lastEmitted == 0 && state.hasEverEmitted) { state.fadeInTotal = static_cast(0.005 * frame.sampleRate); state.fadeInRemaining = state.fadeInTotal; @@ -676,7 +758,8 @@ inline void steadyAudioFrameFeed(std::vector& frames, state.hasEverEmitted = true; } state.lastEmitted = emit; - if (state.primingRequired && emit == 0 && state.fifo.empty()) { + if (state.primingRequired && emit == 0 && state.emptyInputTicks >= 2) { + state.fifo.clear(); state.primed = false; } @@ -718,9 +801,12 @@ inline void steadyAudioFrameFeed(std::vector& frames, if (seen) { continue; } + if (state.primingRequired) { + ++state.emptyInputTicks; + } if (state.fifo.empty()) { state.lastEmitted = 0; // dry: the NEXT flow onset gets a declick fade - if (state.primingRequired) { + if (state.primingRequired && state.emptyInputTicks >= 2) { state.primed = false; } continue; @@ -728,6 +814,7 @@ inline void steadyAudioFrameFeed(std::vector& frames, const size_t tickSamples = static_cast(state.sampleRate / ticksPerSecond) * static_cast(state.channels); size_t emit = 0; + size_t consume = 0; if (!state.primingRequired) { emit = std::min(state.fifo.size(), tickSamples); } else { @@ -737,21 +824,52 @@ inline void steadyAudioFrameFeed(std::vector& frames, } if (state.primed && state.fifo.size() >= tickSamples) { emit = tickSamples; + const size_t channels = static_cast(state.channels); + const size_t tickFrames = tickSamples / channels; + const size_t availableFrames = state.fifo.size() / channels; + const size_t reserveFrames = tickFrames; + const size_t desiredFrames = availableFrames > reserveFrames + ? availableFrames - reserveFrames + : 0; + const size_t maxCorrectionFrames = (std::max)(size_t{1}, tickFrames / 40); + const size_t minConsumeFrames = tickFrames > maxCorrectionFrames + ? tickFrames - maxCorrectionFrames + : 1; + const size_t maxConsumeFrames = tickFrames + maxCorrectionFrames; + const size_t consumeFrames = (std::min)( + availableFrames, + (std::max)(minConsumeFrames, (std::min)(desiredFrames, maxConsumeFrames))); + consume = consumeFrames * channels; } else { ++state.partialBlocksPrevented; } } if (emit == 0) { state.lastEmitted = 0; + if (state.primingRequired && state.emptyInputTicks >= 2) { + state.fifo.clear(); + state.primed = false; + } continue; } AudioFrame fill; fill.participantId = sourceId; fill.sampleRate = state.sampleRate; fill.channels = state.channels; - fill.pcm.assign(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + if (!state.primingRequired) { + consume = emit; + } + if (consume != emit) { + fill.pcm = resampleInterleavedBlockToFrames( + state.fifo.data(), consume / static_cast(state.channels), + emit / static_cast(state.channels), static_cast(state.channels)); + ++state.clockCorrectionEvents; + state.clockCorrectionInputFrames += consume / static_cast(state.channels); + } else { + fill.pcm.assign(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + } fill.sampleCount = static_cast(emit / static_cast(state.channels)); - state.fifo.erase(state.fifo.begin(), state.fifo.begin() + static_cast(emit)); + state.fifo.erase(state.fifo.begin(), state.fifo.begin() + static_cast(consume)); if (emit > 0 && state.lastEmitted == 0 && state.hasEverEmitted) { state.fadeInTotal = static_cast(0.005 * state.sampleRate); state.fadeInRemaining = state.fadeInTotal; diff --git a/native/tests/AudioDspTest.cpp b/native/tests/AudioDspTest.cpp index 10d07b45..f2a33b0e 100644 --- a/native/tests/AudioDspTest.cpp +++ b/native/tests/AudioDspTest.cpp @@ -1090,12 +1090,54 @@ TEST(AudioDsp, ZoomSteadyFeedPrimesOneTickAndNeverEmitsPartialBlocks) { tick(480); // 10 ms arrival + cushion still emits one complete tick tick(480); EXPECT_EQ(emitted.size(), 2880u); - for (size_t index = 0; index < emitted.size(); ++index) { - EXPECT_EQ(emitted[index], static_cast(index)); + for (size_t index = 1; index < emitted.size(); ++index) { + EXPECT_GE(emitted[index], emitted[index - 1]); } EXPECT_TRUE(states["zoom-iso"].primed); EXPECT_EQ(states["zoom-iso"].primeEvents, 1u); EXPECT_GE(states["zoom-iso"].partialBlocksPrevented, 1u); + EXPECT_GE(states["zoom-iso"].clockCorrectionEvents, 1u); +} + +TEST(AudioDsp, ZoomSteadyFeedDoesNotReprimeUnderContinuousClockPhaseJitter) { + // Live Zoom reproduction (Jamal, 2026-08-15): the SDK supplies 10 ms + // packets while the program worker polls every 20 ms. Depending on phase, + // a poll sees 480 then 1440 frames even though the source is continuous. + // The old strict dequeue drained its reserve and re-primed ~16 times/sec, + // inserting an audible hole each time. + std::map states; + float sequence = 0.0f; + const auto makeFrame = [&sequence](int samples) { + corevideo::modules::AudioFrame frame; + frame.participantId = "jamal"; + frame.sampleRate = 48000; + frame.channels = 1; + frame.requiresSteadyFeedPriming = true; + frame.pcm.reserve(static_cast(samples)); + for (int index = 0; index < samples; ++index) { + frame.pcm.push_back(sequence++); + } + frame.sampleCount = samples; + return frame; + }; + + size_t fullBlocks = 0; + for (int tick = 0; tick < 2000; ++tick) { + // Equal long-term rate (960/tick), deliberately hostile poll phase. + const int arrivals[] = {480, 1440, 480, 1440, 960, 960}; + std::vector frames; + frames.push_back(makeFrame(arrivals[tick % 6])); + corevideo::modules::steadyAudioFrameFeed(frames, states); + if (!frames[0].pcm.empty()) { + EXPECT_EQ(frames[0].pcm.size(), 960u); + ++fullBlocks; + } + } + + EXPECT_EQ(states["jamal"].primeEvents, 1u); + EXPECT_TRUE(states["jamal"].primed); + EXPECT_GE(fullBlocks, 1998u); // only startup priming may be silent + EXPECT_GT(states["jamal"].clockCorrectionEvents, 0u); } TEST(AudioDsp, ZoomSteadyFeedReprimesAfterARealGap) { From 1f4f9f34fd04ec955be65e7eb32783a2da238a81 Mon Sep 17 00:00:00 2001 From: John Wallace Date: Sat, 15 Aug 2026 14:55:32 -0400 Subject: [PATCH 3/3] Add CoreVideo Tiles and harden live media --- .../ControlActionRegistryTests.cs | 1 + .../ControlActionRegistry.cs | 2 + .../ZoomMediaSpineSnapshotMergerTests.cs | 14 +- .../Models/NativeMediaCoreProtocol.cs | 4 + .../Models/ZoomMediaSpineModels.cs | 10 +- .../Services/ZoomMediaSpineSnapshotMerger.cs | 5 +- .../DynamicGalleryLayoutServiceTests.cs | 50 ++++ .../ProductionRoleTests.cs | 37 +++ .../SceneCanvasIaTests.cs | 41 +++ .../ScenePersistenceServiceTests.cs | 22 +- .../Controls/SceneCanvasEditorControl.xaml | 7 +- .../Controls/SceneCanvasEditorControl.xaml.cs | 20 ++ .../Models/ProductionModels.cs | 98 ++++++- .../Services/DynamicGalleryLayoutService.cs | 96 +++++++ .../ProductionOutputPreferencesStore.cs | 21 ++ .../Services/ScenePersistenceService.cs | 46 +++- .../Services/StudioControlSurface.cs | 5 +- .../ViewModels/SceneCanvasLayerViewModel.cs | 4 +- .../ViewModels/StudioViewModel.cs | 255 +++++++++++++++++- .../Views/SourcesInputsPage.xaml | 23 +- .../Views/SourcesInputsPage.xaml.cs | 40 ++- .../CoreVideoPro.WinUI/Views/SourcesPage.xaml | 128 ++++++++- .../Views/SourcesPage.xaml.cs | 7 +- native/src/core/MediaCore.cpp | 11 +- native/src/modules/AsyncEncoderSink.cpp | 47 ++-- native/src/modules/AsyncEncoderSink.h | 6 +- native/src/modules/AudioDsp.h | 58 ++-- .../modules/MediaFoundationEncoderAdapter.cpp | 216 +++++++++++++++ native/tests/AsyncEncoderSinkTest.cpp | 21 +- native/tests/AudioDspTest.cpp | 19 +- native/tests/MediaCoreCommandTest.cpp | 4 +- 31 files changed, 1194 insertions(+), 124 deletions(-) create mode 100644 native-shell/CoreVideoPro.WinUI.Tests/DynamicGalleryLayoutServiceTests.cs create mode 100644 native-shell/CoreVideoPro.WinUI/Services/DynamicGalleryLayoutService.cs diff --git a/native-shell/CoreVideoPro.Control.Tests/ControlActionRegistryTests.cs b/native-shell/CoreVideoPro.Control.Tests/ControlActionRegistryTests.cs index b104c591..6b0946f8 100644 --- a/native-shell/CoreVideoPro.Control.Tests/ControlActionRegistryTests.cs +++ b/native-shell/CoreVideoPro.Control.Tests/ControlActionRegistryTests.cs @@ -67,6 +67,7 @@ public void CoreActions_ArePresent() foreach (var id in new[] { "zoom.join", "zoom.leave", "transport.take", "transport.record.toggle", "transport.stream.set", "scene.select", + "scene.dynamicGallery.create", "input.assign", "input.name", "graphics.lowerThird.toggle", "audio.zoomMode.set", "audio.monitor.set", "multiview.layout.set", "automation.autoAssignInputs.set" }) diff --git a/native-shell/CoreVideoPro.Control/ControlActionRegistry.cs b/native-shell/CoreVideoPro.Control/ControlActionRegistry.cs index bee1d7d7..00d799e1 100644 --- a/native-shell/CoreVideoPro.Control/ControlActionRegistry.cs +++ b/native-shell/CoreVideoPro.Control/ControlActionRegistry.cs @@ -170,6 +170,8 @@ private static IReadOnlyList BuildActions() // ---- Scenes / view ---------------------------------------------------------- new("scene.select", "Select scene", "Cue a scene to Preview by id.", new[] { new ControlParam("sceneId", s, true) }), + new("scene.dynamicGallery.create", "Create CoreVideo Tiles", + "Create a dynamic CoreVideo Tiles scene and cue it to Preview."), new("view.setMode", "Set view mode", "Set the operator view (program/preview/programPreview/multiview).", new[] { new ControlParam("mode", s, true) }), diff --git a/native-shell/CoreVideoPro.MediaCore.Tests/ZoomMediaSpineSnapshotMergerTests.cs b/native-shell/CoreVideoPro.MediaCore.Tests/ZoomMediaSpineSnapshotMergerTests.cs index 88291d58..0e8b969d 100644 --- a/native-shell/CoreVideoPro.MediaCore.Tests/ZoomMediaSpineSnapshotMergerTests.cs +++ b/native-shell/CoreVideoPro.MediaCore.Tests/ZoomMediaSpineSnapshotMergerTests.cs @@ -27,9 +27,15 @@ public void MergeNormalizesMeetingStateAndParticipants() [ new ZoomMediaSpineSubscription { + ParticipantId = "operator-1", Kind = "participant-video", Status = "subscribed", - FramesReceived = 3 + LastResultCode = "ok", + DeliveredWidth = 1280, + DeliveredHeight = 720, + DeliveredFps = 30, + FramesReceived = 3, + FrameFresh = true } ] }; @@ -41,6 +47,10 @@ public void MergeNormalizesMeetingStateAndParticipants() Assert.Equal("operator-1", merged.Participants[0].UserId); Assert.Equal("operator-1", merged.ActiveSpeakerId); Assert.True(merged.SourceSnapshot.SubscribedSourceCount >= 1); + var subscription = Assert.Single(merged.ZoomSubscriptions); + Assert.Equal("operator-1", subscription.ParticipantId); + Assert.Equal(1280, subscription.DeliveredWidth); + Assert.True(subscription.FrameFresh); } [Fact] @@ -49,4 +59,4 @@ public void NormalizeMeetingStateMapsHyphenatedValues() Assert.Equal("in_meeting", ZoomMediaSpineSnapshotMerger.NormalizeMeetingState("in-meeting")); Assert.Equal("idle", ZoomMediaSpineSnapshotMerger.NormalizeMeetingState("leaving")); } -} \ No newline at end of file +} diff --git a/native-shell/CoreVideoPro.MediaCore/Models/NativeMediaCoreProtocol.cs b/native-shell/CoreVideoPro.MediaCore/Models/NativeMediaCoreProtocol.cs index d1016e36..09812b19 100644 --- a/native-shell/CoreVideoPro.MediaCore/Models/NativeMediaCoreProtocol.cs +++ b/native-shell/CoreVideoPro.MediaCore/Models/NativeMediaCoreProtocol.cs @@ -818,6 +818,10 @@ public sealed record NativeMediaCoreStateSnapshot /// Live Zoom roster from media-core sync when the engine is connected. public string? ActiveSpeakerId { get; init; } public IReadOnlyList Participants { get; init; } = []; + /// Latest per-source Zoom SDK subscription evidence, retained for + /// operator-facing source diagnostics instead of being collapsed into only + /// aggregate frame counters. + public IReadOnlyList ZoomSubscriptions { get; init; } = []; } public sealed class NativeMediaCoreValidation diff --git a/native-shell/CoreVideoPro.MediaCore/Models/ZoomMediaSpineModels.cs b/native-shell/CoreVideoPro.MediaCore/Models/ZoomMediaSpineModels.cs index 75055ce4..169482f1 100644 --- a/native-shell/CoreVideoPro.MediaCore/Models/ZoomMediaSpineModels.cs +++ b/native-shell/CoreVideoPro.MediaCore/Models/ZoomMediaSpineModels.cs @@ -25,9 +25,17 @@ public sealed class ZoomMediaSpineSubscription public string LastResultCode { get; init; } = "ok"; public int FramesReceived { get; init; } public int AudioPacketsReceived { get; init; } + public int DeliveredWidth { get; init; } + public int DeliveredHeight { get; init; } + public int DeliveredFps { get; init; } public double FirstFrameAtMs { get; init; } = -1; public double FirstFrameDelayMs { get; init; } = -1; public double LastFrameAtMs { get; init; } = -1; + public double LastFrameAgeMs { get; init; } = -1; + public int LastFrameId { get; init; } + public bool FrameFresh { get; init; } + public int StaleFrameCount { get; init; } + public int MalformedFrameCount { get; init; } public string? Warning { get; init; } } @@ -42,4 +50,4 @@ public sealed class ZoomMediaSpineNativeSnapshot public IReadOnlyList Subscriptions { get; init; } = []; public IReadOnlyList Warnings { get; init; } = []; public IReadOnlyList Events { get; init; } = []; -} \ No newline at end of file +} diff --git a/native-shell/CoreVideoPro.MediaCore/Services/ZoomMediaSpineSnapshotMerger.cs b/native-shell/CoreVideoPro.MediaCore/Services/ZoomMediaSpineSnapshotMerger.cs index 8c82d6ff..6de530d9 100644 --- a/native-shell/CoreVideoPro.MediaCore/Services/ZoomMediaSpineSnapshotMerger.cs +++ b/native-shell/CoreVideoPro.MediaCore/Services/ZoomMediaSpineSnapshotMerger.cs @@ -43,7 +43,8 @@ public static NativeMediaCoreStateSnapshot Merge( return merged with { SourceSnapshot = sourceSnapshot, - Diagnostics = merged.Diagnostics with { SourceSnapshot = sourceSnapshot } + Diagnostics = merged.Diagnostics with { SourceSnapshot = sourceSnapshot }, + ZoomSubscriptions = spine.Subscriptions }; } @@ -80,4 +81,4 @@ public static RawCaptureSnapshot ToCaptureSnapshot( null or "" => "idle", _ => meetingState }; -} \ No newline at end of file +} diff --git a/native-shell/CoreVideoPro.WinUI.Tests/DynamicGalleryLayoutServiceTests.cs b/native-shell/CoreVideoPro.WinUI.Tests/DynamicGalleryLayoutServiceTests.cs new file mode 100644 index 00000000..7da020ed --- /dev/null +++ b/native-shell/CoreVideoPro.WinUI.Tests/DynamicGalleryLayoutServiceTests.cs @@ -0,0 +1,50 @@ +using CoreVideoPro.WinUI.Services; +using Xunit; + +namespace CoreVideoPro.WinUI.Tests; + +public sealed class DynamicGalleryLayoutServiceTests +{ + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(5)] + [InlineData(8)] + [InlineData(16)] + public void LayoutIsBoundedUniformAndUsesRequestedAspect(int count) + { + var rects = DynamicGalleryLayoutService.BuildRects(count, tileAspectPreset: "4:3"); + + Assert.Equal(count, rects.Count); + var first = rects[0]; + foreach (var rect in rects) + { + Assert.InRange(rect.X, 0, 1); + Assert.InRange(rect.Y, 0, 1); + Assert.InRange(rect.X + rect.Width, 0, 1.000001); + Assert.InRange(rect.Y + rect.Height, 0, 1.000001); + Assert.Equal(first.Width, rect.Width, 6); + Assert.Equal(first.Height, rect.Height, 6); + Assert.Equal(4.0 / 3.0, rect.Width * (16.0 / 9.0) / rect.Height, 5); + } + } + + [Fact] + public void ShortFinalRowIsCentered() + { + var rects = DynamicGalleryLayoutService.BuildRects(5); + var lastRowY = rects.Max(rect => rect.Y); + var lastRow = rects.Where(rect => Math.Abs(rect.Y - lastRowY) < 0.000001).ToList(); + + Assert.Equal(2, lastRow.Count); + var leftMargin = lastRow[0].X; + var rightMargin = 1 - (lastRow[^1].X + lastRow[^1].Width); + Assert.Equal(leftMargin, rightMargin, 6); + } + + [Fact] + public void EmptyGalleryProducesNoRects() + { + Assert.Empty(DynamicGalleryLayoutService.BuildRects(0)); + } +} diff --git a/native-shell/CoreVideoPro.WinUI.Tests/ProductionRoleTests.cs b/native-shell/CoreVideoPro.WinUI.Tests/ProductionRoleTests.cs index 9791a50d..6c9523bf 100644 --- a/native-shell/CoreVideoPro.WinUI.Tests/ProductionRoleTests.cs +++ b/native-shell/CoreVideoPro.WinUI.Tests/ProductionRoleTests.cs @@ -1,5 +1,6 @@ using CoreVideoPro.WinUI.Models; using CoreVideoPro.WinUI.Services; +using CoreVideoPro.MediaCore.Models; using Xunit; namespace CoreVideoPro.WinUI.Tests; @@ -67,4 +68,40 @@ public void CloneCopiesTheProductionRole() var route = SceneRoutingService.BuildAddedSourceRoute("scene-1", 0, "role:guest-2"); Assert.Equal("guest-2", route.Clone().ProductionRoleId); } + + [Fact] + public void FeedHealthCarriesPerParticipantZoomSubscriptionEvidence() + { + var rows = ProductionStateHelper.BuildFeedHealthRows( + [new Participant { Id = "42", Name = "Guest", Health = FeedHealth.Live }], + subscriptions: + [ + new ZoomMediaSpineSubscription + { + ParticipantId = "42", + Kind = "participant-video", + Status = "subscribed", + LastResultCode = "ok", + DeliveredWidth = 1280, + DeliveredHeight = 720, + DeliveredFps = 30, + FramesReceived = 900, + FrameFresh = true + }, + new ZoomMediaSpineSubscription + { + ParticipantId = "42", + Kind = "participant-audio", + Status = "subscribed", + AudioPacketsReceived = 1500 + } + ]); + + var row = Assert.Single(rows); + Assert.Equal("1280x720 @ 30fps", row.DeliveredVideo); + Assert.Equal(900, row.VideoFramesReceived); + Assert.Equal(1500, row.AudioPacketsReceived); + Assert.Contains("Video subscribed", row.DiagnosticSummary); + Assert.False(row.HasRecommendedAction); + } } diff --git a/native-shell/CoreVideoPro.WinUI.Tests/SceneCanvasIaTests.cs b/native-shell/CoreVideoPro.WinUI.Tests/SceneCanvasIaTests.cs index 09a8db5c..5af19662 100644 --- a/native-shell/CoreVideoPro.WinUI.Tests/SceneCanvasIaTests.cs +++ b/native-shell/CoreVideoPro.WinUI.Tests/SceneCanvasIaTests.cs @@ -635,6 +635,47 @@ public void Layer_SourceChange_DoesNotIndependentlyDriveAudioRole() Assert.Equal(SourceAudioRole.Mix, route.AudioRole); } + [Fact] + public void Layer_SyncFromRoute_RebindsEditsToTheNewTemplateRoute() + { + var first = new SourceRoute + { + Id = "first-1", + Mode = SourceRouteMode.Fixed, + ParticipantId = "p1", + BorderStyle = "solid", + BorderThickness = 1 + }; + var second = new SourceRoute + { + Id = "second-1", + Mode = SourceRouteMode.Fixed, + ParticipantId = "p2", + BorderStyle = "solid", + BorderThickness = 4 + }; + var participants = new List + { + new() { Id = "p1", Name = "Host" }, + new() { Id = "p2", Name = "Guest" } + }; + var layer = new SceneCanvasLayerViewModel( + 0, + first, + participants, + captureDevices: [], + showInputs: [], + mediaAssets: [], + _ => { }); + + layer.SyncFromRoute(second, participants, [], [], []); + layer.BorderThickness = 7; + + Assert.Equal(1, first.BorderThickness); + Assert.Equal(7, second.BorderThickness); + Assert.Equal("p2", layer.ParticipantId); + } + [Fact] public void Layer_SourcePickerListsMediaAssetsAsCanvasSources() { diff --git a/native-shell/CoreVideoPro.WinUI.Tests/ScenePersistenceServiceTests.cs b/native-shell/CoreVideoPro.WinUI.Tests/ScenePersistenceServiceTests.cs index 0d58d71f..000e195d 100644 --- a/native-shell/CoreVideoPro.WinUI.Tests/ScenePersistenceServiceTests.cs +++ b/native-shell/CoreVideoPro.WinUI.Tests/ScenePersistenceServiceTests.cs @@ -9,7 +9,21 @@ public sealed class ScenePersistenceServiceTests [Fact] public void SceneRoundTripsThroughPersistedDtoAndJson() { - var scene = new Scene { Id = "custom-abc12345", Name = "Interview wide", Layout = "two-up" }; + var scene = new Scene + { + Id = "custom-abc12345", + Name = "Interview wide", + Layout = "dynamic-gallery", + DynamicGallery = new DynamicGallerySettings + { + MaxTiles = 8, + TileAspect = "4:3", + BorderShape = "rounded", + BorderColor = "#FF8800", + BorderThickness = 4, + GlowSize = 12 + } + }; var routes = new List { new() @@ -49,6 +63,12 @@ public void SceneRoundTripsThroughPersistedDtoAndJson() var restoredScene = Assert.Single(reloaded!.CustomScenes); Assert.Equal("custom-abc12345", restoredScene.Id); Assert.Equal("Interview wide", restoredScene.Name); + var restoredGallery = ScenePersistenceService.SceneFromPersisted(restoredScene).DynamicGallery; + Assert.NotNull(restoredGallery); + Assert.Equal(8, restoredGallery!.MaxTiles); + Assert.Equal("4:3", restoredGallery.TileAspect); + Assert.Equal("rounded", restoredGallery.BorderShape); + Assert.Equal(12, restoredGallery.GlowSize, 3); var restored = restoredScene.Routes.Select(ScenePersistenceService.FromPersisted).ToList(); Assert.Equal(2, restored.Count); diff --git a/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml b/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml index a69f33ae..2b4034db 100644 --- a/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml +++ b/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml @@ -3,7 +3,8 @@ x:Name="Root" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:vm="using:CoreVideoPro.WinUI.ViewModels"> + xmlns:vm="using:CoreVideoPro.WinUI.ViewModels" + xmlns:cvcontrols="using:CoreVideoPro.WinUI.Controls"> @@ -50,6 +51,10 @@ HorizontalAlignment="Center" VerticalAlignment="Center"> + diff --git a/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml.cs b/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml.cs index 649e0812..496bc08e 100644 --- a/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml.cs +++ b/native-shell/CoreVideoPro.WinUI/Controls/SceneCanvasEditorControl.xaml.cs @@ -43,6 +43,26 @@ public SceneCanvasEditorControl() public bool IsInteracting => _dragLayer is not null; + public void SetBackground(MediaAsset? asset) + { + if (asset is null) + { + BackgroundPreview.Visibility = Visibility.Collapsed; + BackgroundPreview.FilePath = null; + BackgroundPreview.Kind = null; + BackgroundPreview.IsPlaying = false; + BackgroundPreview.PlaybackKey = null; + return; + } + + BackgroundPreview.FilePath = asset.FilePath; + BackgroundPreview.Kind = asset.Kind; + BackgroundPreview.PlaybackKey = $"scene-background:{asset.Id}"; + BackgroundPreview.IsLooping = true; + BackgroundPreview.IsPlaying = asset.Kind.Equals("video", StringComparison.OrdinalIgnoreCase); + BackgroundPreview.Visibility = Visibility.Visible; + } + private void OnEditorSizeChanged(object sender, SizeChangedEventArgs e) => ResizeCanvasViewport(); diff --git a/native-shell/CoreVideoPro.WinUI/Models/ProductionModels.cs b/native-shell/CoreVideoPro.WinUI/Models/ProductionModels.cs index b9b83025..ac32fa4e 100644 --- a/native-shell/CoreVideoPro.WinUI/Models/ProductionModels.cs +++ b/native-shell/CoreVideoPro.WinUI/Models/ProductionModels.cs @@ -85,6 +85,52 @@ public sealed class Scene public string Layout { get; init; } = string.Empty; public string Automation { get; init; } = string.Empty; public string DurationLabel { get; init; } = "—"; + public DynamicGallerySettings? DynamicGallery { get; init; } +} + +/// +/// First-class settings for a CoreVideo Tiles scene. Keeping these values on +/// the scene lets the wall reflow when the Zoom roster changes without losing +/// the operator's styling choices. +/// +public sealed class DynamicGallerySettings +{ + public bool AutoFill { get; set; } = true; + public int MaxTiles { get; set; } = 16; + public string TileAspect { get; set; } = "16:9"; + public double CustomAspectRatio { get; set; } = 16.0 / 9.0; + public double GutterPercent { get; set; } = 0.741; + public double MarginPercent { get; set; } = 0.741; + public string BorderShape { get; set; } = "square"; + public string BorderColor { get; set; } = "#000000"; + public double BorderThickness { get; set; } + public double CornerRadius { get; set; } = 16; + public string GlowColor { get; set; } = "#FFFFFF"; + public double GlowSize { get; set; } + public double GlowIntensity { get; set; } = 100; + public double GlowSoftness { get; set; } + public bool AnimateLayout { get; set; } + public int AnimationDurationMs { get; set; } = 350; + + public DynamicGallerySettings Clone() => new() + { + AutoFill = AutoFill, + MaxTiles = MaxTiles, + TileAspect = TileAspect, + CustomAspectRatio = CustomAspectRatio, + GutterPercent = GutterPercent, + MarginPercent = MarginPercent, + BorderShape = BorderShape, + BorderColor = BorderColor, + BorderThickness = BorderThickness, + CornerRadius = CornerRadius, + GlowColor = GlowColor, + GlowSize = GlowSize, + GlowIntensity = GlowIntensity, + GlowSoftness = GlowSoftness, + AnimateLayout = AnimateLayout, + AnimationDurationMs = AnimationDurationMs + }; } public enum StudioViewMode @@ -153,6 +199,18 @@ public sealed class FeedHealthRow public required string BadgeColor { get; init; } public string? Detail { get; init; } public bool NeedsAttention { get; init; } + public string VideoSubscriptionStatus { get; init; } = "not-requested"; + public string VideoResultCode { get; init; } = "pending"; + public string DeliveredVideo { get; init; } = "No frames"; + public int VideoFramesReceived { get; init; } + public string AudioSubscriptionStatus { get; init; } = "not-requested"; + public int AudioPacketsReceived { get; init; } + public double LastFrameAgeMs { get; init; } = -1; + public int StaleFrameCount { get; init; } + public int MalformedFrameCount { get; init; } + public string DiagnosticSummary { get; init; } = "No Zoom subscription evidence yet."; + public string RecommendedAction { get; init; } = string.Empty; + public bool HasRecommendedAction => !string.IsNullOrWhiteSpace(RecommendedAction); } public sealed class CaptureDeviceInput @@ -826,7 +884,8 @@ public static string FeedHealthSummary(IReadOnlyList participants) public static IReadOnlyList BuildFeedHealthRows( IReadOnlyList participants, - IReadOnlyDictionary? productionRoles = null) => + IReadOnlyDictionary? productionRoles = null, + IReadOnlyList? subscriptions = null) => participants.Select(p => { var (label, color, detail, attention) = p.Health switch @@ -845,6 +904,30 @@ public static IReadOnlyList BuildFeedHealthRows( ? assigned : null; + var participantSubscriptions = (subscriptions ?? []) + .Where(subscription => string.Equals(subscription.ParticipantId, p.Id, StringComparison.Ordinal)) + .ToList(); + var video = participantSubscriptions.FirstOrDefault(subscription => + subscription.Kind is "participant-video" or "screen-share"); + var audio = participantSubscriptions.FirstOrDefault(subscription => + subscription.Kind == "participant-audio"); + var deliveredVideo = video is { DeliveredWidth: > 0, DeliveredHeight: > 0 } + ? $"{video.DeliveredWidth}x{video.DeliveredHeight} @ {video.DeliveredFps}fps" + : "No frames"; + var videoStatus = video?.Status ?? "not-requested"; + var videoResult = video?.LastResultCode ?? "pending"; + var action = video switch + { + { Warning.Length: > 0 } => video.Warning, + { Status: "failed" } => $"Reassign or resubscribe this source ({videoResult}).", + { FramesReceived: 0 } when p.Health != FeedHealth.VideoOff => "Capture is on but no video frame has arrived; verify Zoom recording permission and resubscribe.", + { FrameFresh: false, FramesReceived: > 0 } => "The last Zoom frame is stale; resubscribe or reduce requested feed load.", + _ => string.Empty + }; + var diagnosticSummary = + $"Video {videoStatus} · {deliveredVideo} · {video?.FramesReceived ?? 0} frames · " + + $"Audio {audio?.Status ?? "not-requested"} · {audio?.AudioPacketsReceived ?? 0} packets"; + return new FeedHealthRow { ParticipantId = p.Id, @@ -854,7 +937,18 @@ public static IReadOnlyList BuildFeedHealthRows( StatusLabel = label, BadgeColor = color, Detail = detail, - NeedsAttention = attention + NeedsAttention = attention || !string.IsNullOrWhiteSpace(action), + VideoSubscriptionStatus = videoStatus, + VideoResultCode = videoResult, + DeliveredVideo = deliveredVideo, + VideoFramesReceived = video?.FramesReceived ?? 0, + AudioSubscriptionStatus = audio?.Status ?? "not-requested", + AudioPacketsReceived = audio?.AudioPacketsReceived ?? 0, + LastFrameAgeMs = video?.LastFrameAgeMs ?? -1, + StaleFrameCount = video?.StaleFrameCount ?? 0, + MalformedFrameCount = video?.MalformedFrameCount ?? 0, + DiagnosticSummary = diagnosticSummary, + RecommendedAction = action ?? string.Empty }; }).ToList(); diff --git a/native-shell/CoreVideoPro.WinUI/Services/DynamicGalleryLayoutService.cs b/native-shell/CoreVideoPro.WinUI/Services/DynamicGalleryLayoutService.cs new file mode 100644 index 00000000..a0077add --- /dev/null +++ b/native-shell/CoreVideoPro.WinUI/Services/DynamicGalleryLayoutService.cs @@ -0,0 +1,96 @@ +using CoreVideoPro.WinUI.Models; + +namespace CoreVideoPro.WinUI.Services; + +/// Deterministic CoreVideo Tiles layout in normalized canvas space. +public static class DynamicGalleryLayoutService +{ + public static IReadOnlyList BuildRects( + int tileCount, + double canvasAspectRatio = 16.0 / 9.0, + string tileAspectPreset = "16:9", + double customAspectRatio = 16.0 / 9.0, + double gutterPercent = 0.741, + double marginPercent = 0.741) + { + if (tileCount <= 0) + { + return []; + } + + var canvasAspect = Math.Clamp(canvasAspectRatio, 0.25, 4); + var tileAspect = ResolveAspectRatio(tileAspectPreset, customAspectRatio); + var gutterY = Math.Clamp(gutterPercent / 100.0, 0, 0.1); + var marginY = Math.Clamp(marginPercent / 100.0, 0, 0.2); + var gutterX = gutterY / canvasAspect; + var marginX = marginY / canvasAspect; + + Candidate? best = null; + for (var columns = 1; columns <= tileCount; columns++) + { + var rows = (int)Math.Ceiling(tileCount / (double)columns); + var availableWidth = 1 - (2 * marginX) - ((columns - 1) * gutterX); + var availableHeight = 1 - (2 * marginY) - ((rows - 1) * gutterY); + if (availableWidth <= 0 || availableHeight <= 0) + { + continue; + } + + var width = Math.Min( + availableWidth / columns, + (availableHeight / rows) * tileAspect / canvasAspect); + var height = width * canvasAspect / tileAspect; + var candidate = new Candidate(columns, rows, width, height, width * height); + if (best is null || candidate.Area > best.Value.Area + 0.0000001 || + (Math.Abs(candidate.Area - best.Value.Area) < 0.0000001 && candidate.Columns < best.Value.Columns)) + { + best = candidate; + } + } + + var chosen = best ?? new Candidate(1, tileCount, 1, 1.0 / tileCount, 1.0 / tileCount); + var gridHeight = (chosen.Rows * chosen.Height) + ((chosen.Rows - 1) * gutterY); + var top = (1 - gridHeight) / 2; + var result = new List(tileCount); + + for (var row = 0; row < chosen.Rows; row++) + { + var rowStart = row * chosen.Columns; + var rowCount = Math.Min(chosen.Columns, tileCount - rowStart); + var rowWidth = (rowCount * chosen.Width) + ((rowCount - 1) * gutterX); + var left = (1 - rowWidth) / 2; + for (var column = 0; column < rowCount; column++) + { + result.Add(new NormalizedCanvasRect + { + X = left + column * (chosen.Width + gutterX), + Y = top + row * (chosen.Height + gutterY), + Width = chosen.Width, + Height = chosen.Height + }); + } + } + + return result; + } + + public static string NormalizeAspectPreset(string? value) => value switch + { + "4:3" or "5:4" or "1:1" or "3:4" or "9:16" or "custom" => value, + _ => "16:9" + }; + + public static double ResolveAspectRatio(string? preset, double customAspectRatio) => + NormalizeAspectPreset(preset) switch + { + "4:3" => 4.0 / 3.0, + "5:4" => 5.0 / 4.0, + "1:1" => 1, + "3:4" => 3.0 / 4.0, + "9:16" => 9.0 / 16.0, + "custom" => Math.Clamp(customAspectRatio, 0.25, 4), + _ => 16.0 / 9.0 + }; + + private readonly record struct Candidate(int Columns, int Rows, double Width, double Height, double Area); +} diff --git a/native-shell/CoreVideoPro.WinUI/Services/ProductionOutputPreferencesStore.cs b/native-shell/CoreVideoPro.WinUI/Services/ProductionOutputPreferencesStore.cs index 1b31495b..f44d3af7 100644 --- a/native-shell/CoreVideoPro.WinUI/Services/ProductionOutputPreferencesStore.cs +++ b/native-shell/CoreVideoPro.WinUI/Services/ProductionOutputPreferencesStore.cs @@ -207,9 +207,30 @@ public sealed class PersistedScene public string Id { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; public string Layout { get; set; } = string.Empty; + public PersistedDynamicGallerySettings? DynamicGallery { get; set; } public List Routes { get; set; } = []; } +public sealed class PersistedDynamicGallerySettings +{ + public bool AutoFill { get; set; } = true; + public int MaxTiles { get; set; } = 16; + public string TileAspect { get; set; } = "16:9"; + public double CustomAspectRatio { get; set; } = 16.0 / 9.0; + public double GutterPercent { get; set; } = 0.741; + public double MarginPercent { get; set; } = 0.741; + public string BorderShape { get; set; } = "square"; + public string BorderColor { get; set; } = "#000000"; + public double BorderThickness { get; set; } + public double CornerRadius { get; set; } = 16; + public string GlowColor { get; set; } = "#FFFFFF"; + public double GlowSize { get; set; } + public double GlowIntensity { get; set; } = 100; + public double GlowSoftness { get; set; } + public bool AnimateLayout { get; set; } + public int AnimationDurationMs { get; set; } = 350; +} + public sealed class PersistedSceneRoute { public string Id { get; set; } = string.Empty; diff --git a/native-shell/CoreVideoPro.WinUI/Services/ScenePersistenceService.cs b/native-shell/CoreVideoPro.WinUI/Services/ScenePersistenceService.cs index f5fccebd..5fa156de 100644 --- a/native-shell/CoreVideoPro.WinUI/Services/ScenePersistenceService.cs +++ b/native-shell/CoreVideoPro.WinUI/Services/ScenePersistenceService.cs @@ -15,9 +15,31 @@ public static PersistedScene ToPersisted(Scene scene, IReadOnlyList Id = scene.Id, Name = scene.Name, Layout = scene.Layout, + DynamicGallery = scene.DynamicGallery is null ? null : ToPersisted(scene.DynamicGallery), Routes = routes.Select(ToPersisted).ToList() }; + public static PersistedDynamicGallerySettings ToPersisted(DynamicGallerySettings settings) => + new() + { + AutoFill = settings.AutoFill, + MaxTiles = settings.MaxTiles, + TileAspect = settings.TileAspect, + CustomAspectRatio = settings.CustomAspectRatio, + GutterPercent = settings.GutterPercent, + MarginPercent = settings.MarginPercent, + BorderShape = settings.BorderShape, + BorderColor = settings.BorderColor, + BorderThickness = settings.BorderThickness, + CornerRadius = settings.CornerRadius, + GlowColor = settings.GlowColor, + GlowSize = settings.GlowSize, + GlowIntensity = settings.GlowIntensity, + GlowSoftness = settings.GlowSoftness, + AnimateLayout = settings.AnimateLayout, + AnimationDurationMs = settings.AnimationDurationMs + }; + public static PersistedSceneRoute ToPersisted(SourceRoute route) => new() { @@ -49,7 +71,29 @@ public static Scene SceneFromPersisted(PersistedScene persisted) => Id = persisted.Id, Name = persisted.Name, Layout = string.IsNullOrWhiteSpace(persisted.Layout) ? "host-focus" : persisted.Layout, - Automation = "Custom canvas" + Automation = persisted.DynamicGallery is null ? "Custom canvas" : "Auto-reflow Zoom gallery", + DynamicGallery = persisted.DynamicGallery is null ? null : FromPersisted(persisted.DynamicGallery) + }; + + public static DynamicGallerySettings FromPersisted(PersistedDynamicGallerySettings persisted) => + new() + { + AutoFill = persisted.AutoFill, + MaxTiles = Math.Clamp(persisted.MaxTiles, 1, 64), + TileAspect = DynamicGalleryLayoutService.NormalizeAspectPreset(persisted.TileAspect), + CustomAspectRatio = Math.Clamp(persisted.CustomAspectRatio, 0.25, 4), + GutterPercent = Math.Clamp(persisted.GutterPercent, 0, 10), + MarginPercent = Math.Clamp(persisted.MarginPercent, 0, 20), + BorderShape = persisted.BorderShape is "rounded" ? "rounded" : "square", + BorderColor = SceneRoutingService.NormalizeBorderColor(persisted.BorderColor), + BorderThickness = Math.Clamp(persisted.BorderThickness, 0, 32), + CornerRadius = Math.Clamp(persisted.CornerRadius, 0, 100), + GlowColor = SceneRoutingService.NormalizeBorderColor(persisted.GlowColor), + GlowSize = Math.Clamp(persisted.GlowSize, 0, 64), + GlowIntensity = Math.Clamp(persisted.GlowIntensity, 0, 100), + GlowSoftness = Math.Clamp(persisted.GlowSoftness, 0, 100), + AnimateLayout = persisted.AnimateLayout, + AnimationDurationMs = Math.Clamp(persisted.AnimationDurationMs, 100, 2000) }; public static SourceRoute FromPersisted(PersistedSceneRoute persisted) diff --git a/native-shell/CoreVideoPro.WinUI/Services/StudioControlSurface.cs b/native-shell/CoreVideoPro.WinUI/Services/StudioControlSurface.cs index 26a4788a..5082811c 100644 --- a/native-shell/CoreVideoPro.WinUI/Services/StudioControlSurface.cs +++ b/native-shell/CoreVideoPro.WinUI/Services/StudioControlSurface.cs @@ -50,7 +50,7 @@ public sealed class StudioControlSurface : IControlSurface, IDisposable "transport.engine.toggle", "transport.engine.set", "transport.virtualcam.toggle", "transport.virtualcam.set", "transport.virtualcam.mirror.set", "transport.virtualcam.name.set", - "scene.select", "view.setMode", + "scene.select", "scene.dynamicGallery.create", "view.setMode", "input.assign", "input.name", "input.inShow.set", "graphics.lowerThird.toggle", "graphics.lowerThird.set", "graphics.caption.set", "graphics.graphic.toggle", "audio.zoomMode.set", "audio.monitor.set", "audio.monitor.volume", "audio.masterLimiter.set", @@ -178,6 +178,9 @@ private async Task DispatchAsync(string actionId, IReadOnly case "scene.select": _vm.SelectSceneCommand.Execute(Str(args, 0)); return ControlInvokeResult.Success; + case "scene.dynamicGallery.create": + _vm.NewDynamicGalleryCommand.Execute(null); + return ControlInvokeResult.Success; case "view.setMode": _vm.SetViewModeCommand.Execute(Str(args, 0)); return ControlInvokeResult.Success; diff --git a/native-shell/CoreVideoPro.WinUI/ViewModels/SceneCanvasLayerViewModel.cs b/native-shell/CoreVideoPro.WinUI/ViewModels/SceneCanvasLayerViewModel.cs index db4d2be9..f3a3c6ed 100644 --- a/native-shell/CoreVideoPro.WinUI/ViewModels/SceneCanvasLayerViewModel.cs +++ b/native-shell/CoreVideoPro.WinUI/ViewModels/SceneCanvasLayerViewModel.cs @@ -10,7 +10,7 @@ public sealed partial class SceneCanvasLayerViewModel : ObservableObject private const string CaptureValuePrefix = "capture:"; private const string MediaValuePrefix = "media:"; private readonly Action _onChanged; - private readonly SourceRoute _route; + private SourceRoute _route; private IReadOnlyList _participants; private IReadOnlyList _captureDevices; private IReadOnlyList _showInputs; @@ -233,6 +233,7 @@ private void ResetFraming() } public void SyncFromRoute( + SourceRoute route, IReadOnlyList participants, IReadOnlyList captureDevices, IReadOnlyList showInputs, @@ -241,6 +242,7 @@ public void SyncFromRoute( _suppressChangeNotification = true; try { + _route = route; _participants = participants; _captureDevices = captureDevices; _showInputs = showInputs; diff --git a/native-shell/CoreVideoPro.WinUI/ViewModels/StudioViewModel.cs b/native-shell/CoreVideoPro.WinUI/ViewModels/StudioViewModel.cs index 7b29252b..dcaa3a0e 100644 --- a/native-shell/CoreVideoPro.WinUI/ViewModels/StudioViewModel.cs +++ b/native-shell/CoreVideoPro.WinUI/ViewModels/StudioViewModel.cs @@ -97,6 +97,13 @@ public sealed partial class StudioViewModel : ObservableObject, IAsyncDisposable [ObservableProperty] private string _previewSceneId = "speaker-slides"; + // ItemsSource replacement briefly drives the WinUI ComboBox's two-way + // SelectedValue to null. Keep the last stable scene key so that transient + // selection loss never invalidates the view model or recursively rebuilds + // SceneItems until the native stack overflows. + private string _lastValidPreviewSceneId = "speaker-slides"; + private bool _previewSceneSelectionRestoreScheduled; + [ObservableProperty] private string _sceneBuilderName = "Speaker + Slides"; @@ -1532,6 +1539,39 @@ public StudioViewModel() public IReadOnlyList SceneItems { get; private set; } = []; + public IReadOnlyList GalleryTileAspectOptions { get; } = + [ + new() { Value = "16:9", Label = "16:9" }, + new() { Value = "4:3", Label = "4:3" }, + new() { Value = "5:4", Label = "5:4" }, + new() { Value = "1:1", Label = "Square" }, + new() { Value = "3:4", Label = "Portrait 3:4" }, + new() { Value = "9:16", Label = "Portrait 9:16" }, + new() { Value = "custom", Label = "Custom" } + ]; + + public IReadOnlyList GalleryBorderShapeOptions { get; } = + [ + new() { Value = "square", Label = "Square" }, + new() { Value = "rounded", Label = "Rounded" } + ]; + + public bool IsPreviewDynamicGallery => PreviewScene.DynamicGallery is not null; + public double GalleryMaxTiles { get => PreviewScene.DynamicGallery?.MaxTiles ?? 16; set => UpdateGallery(s => s.MaxTiles = (int)Math.Clamp(Math.Round(value), 1, 64)); } + public string GalleryTileAspect { get => PreviewScene.DynamicGallery?.TileAspect ?? "16:9"; set => UpdateGallery(s => s.TileAspect = DynamicGalleryLayoutService.NormalizeAspectPreset(value)); } + public double GalleryCustomAspectRatio { get => PreviewScene.DynamicGallery?.CustomAspectRatio ?? 16.0 / 9.0; set => UpdateGallery(s => s.CustomAspectRatio = Math.Clamp(value, 0.25, 4)); } + public double GalleryGutterPercent { get => PreviewScene.DynamicGallery?.GutterPercent ?? 0.741; set => UpdateGallery(s => s.GutterPercent = Math.Clamp(value, 0, 10)); } + public double GalleryMarginPercent { get => PreviewScene.DynamicGallery?.MarginPercent ?? 0.741; set => UpdateGallery(s => s.MarginPercent = Math.Clamp(value, 0, 20)); } + public string GalleryBorderShape { get => PreviewScene.DynamicGallery?.BorderShape ?? "square"; set => UpdateGallery(s => s.BorderShape = value == "rounded" ? "rounded" : "square"); } + public string GalleryBorderColor { get => PreviewScene.DynamicGallery?.BorderColor ?? "#000000"; set => UpdateGallery(s => s.BorderColor = SceneRoutingService.NormalizeBorderColor(value)); } + public double GalleryBorderThickness { get => PreviewScene.DynamicGallery?.BorderThickness ?? 0; set => UpdateGallery(s => s.BorderThickness = Math.Clamp(value, 0, 32)); } + public string GalleryGlowColor { get => PreviewScene.DynamicGallery?.GlowColor ?? "#FFFFFF"; set => UpdateGallery(s => s.GlowColor = SceneRoutingService.NormalizeBorderColor(value)); } + public double GalleryGlowSize { get => PreviewScene.DynamicGallery?.GlowSize ?? 0; set => UpdateGallery(s => s.GlowSize = Math.Clamp(value, 0, 64)); } + public double GalleryGlowIntensity { get => PreviewScene.DynamicGallery?.GlowIntensity ?? 100; set => UpdateGallery(s => s.GlowIntensity = Math.Clamp(value, 0, 100)); } + public double GalleryGlowSoftness { get => PreviewScene.DynamicGallery?.GlowSoftness ?? 0; set => UpdateGallery(s => s.GlowSoftness = Math.Clamp(value, 0, 100)); } + public bool GalleryAnimateLayout { get => PreviewScene.DynamicGallery?.AnimateLayout ?? false; set => UpdateGallery(s => s.AnimateLayout = value); } + public double GalleryAnimationDurationMs { get => PreviewScene.DynamicGallery?.AnimationDurationMs ?? 350; set => UpdateGallery(s => s.AnimationDurationMs = (int)Math.Clamp(Math.Round(value), 100, 2000)); } + public IReadOnlyList RoomVideoParticipants { get; private set; } // All in-room participants (incl. video-off) for the Sources/Inputs picker. @@ -1541,7 +1581,11 @@ public StudioViewModel() public Scene ProgramScene => Scenes.First(s => s.Id == ActiveSceneId); - public Scene PreviewScene => Scenes.First(s => s.Id == PreviewSceneId); + public Scene PreviewScene => + Scenes.FirstOrDefault(s => string.Equals(s.Id, PreviewSceneId, StringComparison.Ordinal)) ?? + Scenes.FirstOrDefault(s => string.Equals(s.Id, _lastValidPreviewSceneId, StringComparison.Ordinal)) ?? + Scenes.FirstOrDefault(s => string.Equals(s.Id, ActiveSceneId, StringComparison.Ordinal)) ?? + Scenes.First(); [ObservableProperty] private string _currentRoomLabel; @@ -3536,10 +3580,21 @@ partial void OnActiveSceneIdChanged(string value) partial void OnPreviewSceneIdChanged(string value) { + // Replacing SceneItems briefly drives the bound ComboBox selection to + // null. Never run scene refresh against that transient invalid key. + if (string.IsNullOrWhiteSpace(value) || + !_scenes.Any(scene => string.Equals(scene.Id, value, StringComparison.Ordinal))) + { + SchedulePreviewSceneSelectionRestore(); + return; + } + + _lastValidPreviewSceneId = value; // S2b: cueing a different scene abandons any uncommitted edits to the // live scene (the draft belongs to the previously cued scene). DiscardLivePreviewDraft(); SceneBuilderName = PreviewScene.Name; + NotifyDynamicGalleryPropertiesChanged(); RefreshSceneItems(); RefreshSceneBackgroundSelection(); OnPropertyChanged(nameof(PreviewScene)); @@ -3558,6 +3613,34 @@ partial void OnPreviewSceneIdChanged(string value) } } + private void SchedulePreviewSceneSelectionRestore() + { + if (_previewSceneSelectionRestoreScheduled) + { + return; + } + + _previewSceneSelectionRestoreScheduled = true; + UiDispatch.Enqueue(_dispatcher, DispatcherQueuePriority.Low, () => + { + _previewSceneSelectionRestoreScheduled = false; + if (_scenes.Any(scene => string.Equals(scene.Id, PreviewSceneId, StringComparison.Ordinal))) + { + return; + } + + var fallback = _scenes.FirstOrDefault(scene => + string.Equals(scene.Id, _lastValidPreviewSceneId, StringComparison.Ordinal)) ?? + _scenes.FirstOrDefault(scene => + string.Equals(scene.Id, ActiveSceneId, StringComparison.Ordinal)) ?? + _scenes.FirstOrDefault(); + if (fallback is not null) + { + PreviewSceneId = fallback.Id; + } + }, "scene-selection.restore"); + } + private async Task SyncPreviewSceneChangeAsync() { try @@ -3822,6 +3905,31 @@ private void NewScene() SaveProductionOutputPreferences(); } + [RelayCommand] + private void NewDynamicGallery() + { + var newId = NewCustomSceneId(); + var galleryNumber = _scenes.Count(scene => scene.DynamicGallery is not null) + 1; + var scene = new Scene + { + Id = newId, + Name = galleryNumber == 1 ? "CoreVideo Tiles" : $"CoreVideo Tiles {galleryNumber}", + Layout = "dynamic-gallery", + Automation = "Auto-reflow Zoom gallery", + DynamicGallery = new DynamicGallerySettings() + }; + + _scenes.Add(scene); + _sceneRoutes[newId] = []; + ReconcileDynamicGalleryRoutes(scene, _sceneRoutes[newId]); + RefreshSceneItems(); + PreviewSceneId = newId; + ActiveTab = StudioTab.Sources; + CommandStatus = $"{scene.Name} created with {RoomVideoParticipants.Count} live Zoom sources"; + SchedulePreviewRoutingRefresh(); + SaveProductionOutputPreferences(); + } + [RelayCommand] private void DuplicateScene(string? sceneId) { @@ -3837,7 +3945,8 @@ private void DuplicateScene(string? sceneId) Id = newId, Name = ScenePersistenceService.MakeUniqueSceneName($"{source.Name} copy", _scenes.Select(s => s.Name)), Layout = source.Layout, - Automation = "Custom canvas" + Automation = source.DynamicGallery is null ? "Custom canvas" : "Auto-reflow Zoom gallery", + DynamicGallery = source.DynamicGallery?.Clone() }; _scenes.Add(scene); @@ -3881,7 +3990,8 @@ private void SaveScene(string? name) Id = newId, Name = trimmed ?? $"Saved scene {sceneNumber}", Layout = PreviewScene.Layout, - Automation = "Custom canvas" + Automation = PreviewScene.DynamicGallery is null ? "Custom canvas" : "Auto-reflow Zoom gallery", + DynamicGallery = PreviewScene.DynamicGallery?.Clone() }; _scenes.Add(scene); @@ -3957,7 +4067,8 @@ private void RenameScene(string sceneId, string name) Name = name, Layout = scene.Layout, Automation = scene.Automation, - DurationLabel = scene.DurationLabel + DurationLabel = scene.DurationLabel, + DynamicGallery = scene.DynamicGallery }; OnPropertyChanged(nameof(ProgramScene)); @@ -4563,7 +4674,7 @@ public void ToggleMixerMute(string participantId) public void SetParticipantProductionRole(string participantId, string? roleId) { - var participant = RoomVideoParticipants.FirstOrDefault(item => + var participant = RoomParticipantsForInputs.FirstOrDefault(item => string.Equals(item.Id, participantId, StringComparison.Ordinal)); if (participant is null) { @@ -7169,8 +7280,11 @@ private void RefreshProductionReadouts() Scenes, AutomationPreferScreenShare, (int)Math.Round(AutomationPanelParticipantThreshold)); - FeedHealthRows = ProductionStateHelper.BuildFeedHealthRows(RoomVideoParticipants, _participantProductionRoles); - FeedHealthSummary = ProductionStateHelper.FeedHealthSummary(RoomVideoParticipants); + FeedHealthRows = ProductionStateHelper.BuildFeedHealthRows( + RoomParticipantsForInputs, + _participantProductionRoles, + _bridge.LastSnapshot?.ZoomSubscriptions); + FeedHealthSummary = ProductionStateHelper.FeedHealthSummary(RoomParticipantsForInputs); MagicSceneStatus = ProductionStateHelper.BuildMagicSceneStatus(RoomVideoParticipants); MediaBinSummary = ProductionStateHelper.MediaBinSummary(MediaBinGroups.Sum(group => group.Assets.Count)); AutoProductionReadout = MagicScene.BuildAutoProductionReadout(); @@ -11736,7 +11850,7 @@ private void RefreshSceneBackgroundSelection() try { PreviewSceneBackgroundAssetId = SceneBackgroundSelectionService.ResolveSelectedAssetId( - PreviewSceneId, + PreviewScene.Id, _sceneBackgroundAssetIds, FindMediaAsset, IsVisualMediaAsset); @@ -12371,13 +12485,20 @@ private void RefreshSceneCompositionState(Scene scene, string sceneId, bool isPr // Preview may be editing a draft of the on-air scene. Refreshing from the // stored program routes here silently erased newly added overlay layers. var mutableRoutes = isPreview ? GetPreviewEditableRoutes() : GetMutableRoutes(sceneId); + if (scene.DynamicGallery is not null) + { + ReconcileDynamicGalleryRoutes(scene, mutableRoutes); + } var defaults = SceneRoutingService.GetRouteDefaults( scene, mutableRoutes, RoomVideoParticipants); - ReconcileRoutes(mutableRoutes, defaults); - SceneCanvasLayoutService.EnsureCanvasRects(mutableRoutes, scene.Layout); + if (scene.DynamicGallery is null) + { + ReconcileRoutes(mutableRoutes, defaults); + SceneCanvasLayoutService.EnsureCanvasRects(mutableRoutes, scene.Layout); + } var workingRoutes = mutableRoutes.Select(ResolveRouteFromShowInput).ToList(); if (isPreview) @@ -12400,6 +12521,103 @@ private void RefreshSceneCompositionState(Scene scene, string sceneId, bool isPr } } + private void ReconcileDynamicGalleryRoutes(Scene scene, List routes) + { + var settings = scene.DynamicGallery; + if (settings is null || _canvasInteractionActive) + { + return; + } + + var participants = RoomVideoParticipants + .Where(participant => participant.Health != FeedHealth.VideoOff) + .Take(Math.Clamp(settings.MaxTiles, 1, 64)) + .ToList(); + var participantIds = participants.Select(participant => participant.Id).ToHashSet(StringComparer.Ordinal); + var retained = routes + .Where(route => route.Mode == SourceRouteMode.Fixed && + route.ParticipantId is { Length: > 0 } participantId && + participantIds.Contains(participantId)) + .GroupBy(route => route.ParticipantId!, StringComparer.Ordinal) + .Select(group => group.First()) + .ToList(); + var retainedIds = retained.Select(route => route.ParticipantId!).ToHashSet(StringComparer.Ordinal); + + foreach (var participant in participants) + { + if (retainedIds.Add(participant.Id)) + { + retained.Add(new SourceRoute + { + Id = $"{scene.Id}-tile-{participant.Id}", + Mode = SourceRouteMode.Fixed, + ParticipantId = participant.Id, + AudioRole = SourceAudioRole.Isolated, + FitMode = "fill" + }); + } + } + + retained = retained.Take(Math.Clamp(settings.MaxTiles, 1, 64)).ToList(); + var rects = DynamicGalleryLayoutService.BuildRects( + retained.Count, + tileAspectPreset: settings.TileAspect, + customAspectRatio: settings.CustomAspectRatio, + gutterPercent: settings.GutterPercent, + marginPercent: settings.MarginPercent); + + for (var index = 0; index < retained.Count; index++) + { + var route = retained[index]; + route.CanvasRect = rects[index].Clone(); + route.ZIndex = index; + route.BorderStyle = settings.BorderThickness > 0 ? "solid" : "none"; + route.BorderColor = settings.BorderColor; + route.BorderThickness = Math.Clamp(settings.BorderThickness, 0, 12); + } + + routes.Clear(); + routes.AddRange(retained); + } + + private void UpdateGallery(Action update) + { + if (PreviewScene.DynamicGallery is not { } settings) + { + return; + } + + update(settings); + var routes = GetPreviewEditableRoutes(); + ReconcileDynamicGalleryRoutes(PreviewScene, routes); + NotifyDynamicGalleryPropertiesChanged(); + SyncPreviewCanvasLayers(routes); + PublishPreviewCompositionState(PreviewScene, routes.Select(ResolveRouteFromShowInput).ToList()); + CommandStatus = $"{PreviewScene.Name} gallery updated on Preview"; + SchedulePreviewRoutingRefresh(); + SyncLiveSceneEditIfNeeded(PreviewSceneId); + SaveProductionOutputPreferences(); + } + + private void NotifyDynamicGalleryPropertiesChanged() + { + OnPropertyChanged(nameof(IsPreviewDynamicGallery)); + OnPropertyChanged(nameof(GalleryMaxTiles)); + OnPropertyChanged(nameof(GalleryTileAspect)); + OnPropertyChanged(nameof(GalleryCustomAspectRatio)); + OnPropertyChanged(nameof(GalleryGutterPercent)); + OnPropertyChanged(nameof(GalleryMarginPercent)); + OnPropertyChanged(nameof(GalleryBorderShape)); + OnPropertyChanged(nameof(GalleryBorderColor)); + OnPropertyChanged(nameof(GalleryBorderThickness)); + OnPropertyChanged(nameof(GalleryGlowColor)); + OnPropertyChanged(nameof(GalleryGlowSize)); + OnPropertyChanged(nameof(GalleryGlowIntensity)); + OnPropertyChanged(nameof(GalleryGlowSoftness)); + OnPropertyChanged(nameof(GalleryAnimateLayout)); + OnPropertyChanged(nameof(GalleryAnimationDurationMs)); + } + // NOT force: forcing re-ran the full building-out -> building-in slide on EVERY // refresh, and this fires ~continuously (snapshot applies, active-speaker changes, // scene refreshes call it from ~9 sites). That made the lower third perpetually slide @@ -12769,7 +12987,12 @@ private void SyncPreviewCanvasLayers(IReadOnlyList routes) { for (var index = 0; index < routes.Count; index++) { - PreviewCanvasLayers[index].SyncFromRoute(RoomVideoParticipants, CaptureDevices, ShowInputs, VisualMediaAssets); + PreviewCanvasLayers[index].SyncFromRoute( + routes[index], + RoomVideoParticipants, + CaptureDevices, + ShowInputs, + VisualMediaAssets); PreviewCanvasLayers[index].SetSurface(ResolveLayerSurface(routes[index], index)); } @@ -13529,7 +13752,8 @@ private void UpdateSceneLayout(string sceneId, string layout) Name = scene.Name, Layout = layout, Automation = scene.Automation, - DurationLabel = scene.DurationLabel + DurationLabel = scene.DurationLabel, + DynamicGallery = scene.DynamicGallery }; OnPropertyChanged(nameof(ProgramScene)); @@ -13553,7 +13777,12 @@ private void OnPreviewCanvasLayerChanged(SceneCanvasLayerViewModel layer) layer.ApplyRoute(); SceneRoutingService.ApplyNormalizeRouteUpdate(routes[layer.LayerIndex], RoomVideoParticipants); ApplyKnownColorGradeToRoute(routes[layer.LayerIndex]); - layer.SyncFromRoute(RoomVideoParticipants, CaptureDevices, ShowInputs, VisualMediaAssets); + layer.SyncFromRoute( + routes[layer.LayerIndex], + RoomVideoParticipants, + CaptureDevices, + ShowInputs, + VisualMediaAssets); layer.SetSurface(ResolveLayerSurface(routes[layer.LayerIndex], layer.LayerIndex)); CommandStatus = $"{PreviewScene.Name} source {layer.LayerIndex + 1} updated on canvas"; diff --git a/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml b/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml index d130eb35..b7b7d63b 100644 --- a/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml +++ b/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml @@ -41,7 +41,8 @@ - + @@ -49,6 +50,10 @@ + + + + - + + + + + + diff --git a/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml.cs b/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml.cs index 8639d9b8..bf2112ae 100644 --- a/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml.cs +++ b/native-shell/CoreVideoPro.WinUI/Views/SourcesInputsPage.xaml.cs @@ -146,25 +146,45 @@ private void OnProductionRoleComboLoaded(object sender, RoutedEventArgs e) return; } + SyncProductionRoleCombo(combo, row); + } + + private void OnFeedHealthElementPrepared(ItemsRepeater sender, ItemsRepeaterElementPreparedEventArgs args) + { + if (args.Element is not FrameworkElement root || + FindDescendant(root, "ProductionRoleCombo") is not { } combo || + combo.Tag is not FeedHealthRow row) + { + return; + } + + SyncProductionRoleCombo(combo, row); + } + + private void SyncProductionRoleCombo(ComboBox combo, FeedHealthRow row) + { try { - var options = combo.ItemsSource as IEnumerable; - if (options is null) - { - return; // ItemsSource binding has not resolved yet - } + // ElementName bindings inside a recycled ItemsRepeater template are + // not guaranteed to resolve before Loaded. Make the page view-model + // authoritative and suppress write-back while rebinding the row. + var options = ViewModel?.ProductionRoleAssignmentOptions; + if (options is null) return; + + combo.SelectionChanged -= OnProductionRoleChanged; + combo.ItemsSource = options; var roleId = row.ProductionRoleId ?? string.Empty; if (!options.Any(option => option.Value == roleId)) { - return; // stale/unknown role: leave the box alone rather than throw - } - if (!Equals(combo.SelectedValue, roleId)) - { - combo.SelectedValue = roleId; + roleId = string.Empty; } + combo.SelectedValue = roleId; + combo.SelectionChanged += OnProductionRoleChanged; } catch (Exception ex) { + combo.SelectionChanged -= OnProductionRoleChanged; + combo.SelectionChanged += OnProductionRoleChanged; LaunchLog.Write($"sources: production-role selection skipped ({ex.GetType().Name}: {ex.Message})"); } } diff --git a/native-shell/CoreVideoPro.WinUI/Views/SourcesPage.xaml b/native-shell/CoreVideoPro.WinUI/Views/SourcesPage.xaml index 2cb69f74..f3900a36 100644 --- a/native-shell/CoreVideoPro.WinUI/Views/SourcesPage.xaml +++ b/native-shell/CoreVideoPro.WinUI/Views/SourcesPage.xaml @@ -70,6 +70,30 @@ CommandParameter="studio" /> + + + + + + + + +