diff --git a/README.md b/README.md index 3dcc204..a5e3068 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,12 @@ cannot keep a camera session active during multitasking fall back to screen-only The Video settings offer H.264 (default) or HEVC. HEVC gives roughly a 40% quality-per-bit gain on text-heavy screen content in the 2–8 Mbps band, but needs -a compatible ingest: it rides SRT (MPEG-TS), WHIP, and *enhanced*-RTMP (the +a compatible ingest: it rides SRT (MPEG-TS) and *enhanced*-RTMP (the publisher advertises the `hvc1` FourCC in the E-RTMP connect command). Traditional -RTMP services such as Restream speak H.264 only, so leave the codec on H.264 for -them — HEVC over RTMP should be verified against the specific endpoint on a real -device. Low-latency VideoToolbox rate control is enabled for both codecs. +RTMP services such as Restream and the current WHIP transport speak H.264 only, so +leave the codec on H.264 for them — HEVC over RTMP should be verified against the +specific endpoint on a real device. Low-latency VideoToolbox rate control is enabled +for both codecs. ## Notes diff --git a/Stream/AudioInputProvider.swift b/Stream/AudioInputProvider.swift index 8aea91f..16b39b4 100644 --- a/Stream/AudioInputProvider.swift +++ b/Stream/AudioInputProvider.swift @@ -1,6 +1,9 @@ import Foundation import AVFAudio +import Observation import StreamCore +import SwiftUI +import os /// Observable helper that configures `AVAudioSession` so Bluetooth/DJI mic /// inputs surface, enumerates `availableInputs`, exposes name/uid pairs, and @@ -53,7 +56,7 @@ final class AudioInputProvider { /// Observer token for `AVAudioSession.routeChangeNotification`. Marked /// `nonisolated(unsafe)` so `deinit` (which is nonisolated on a `@MainActor` /// type) can remove it; an `NSObjectProtocol` token is safe to touch there. - private nonisolated(unsafe) var routeChangeObserver: NSObjectProtocol? + @ObservationIgnored private nonisolated(unsafe) var routeChangeObserver: NSObjectProtocol? deinit { if let routeChangeObserver { @@ -97,10 +100,7 @@ final class AudioInputProvider { func select(uid: String?, into settings: inout StreamSettings) { settings.preferredAudioInputUID = uid guard permission == .granted, let uid else { return } - let session = AVAudioSession.sharedInstance() - if let port = session.availableInputs?.first(where: { $0.uid == uid }) { - try? session.setPreferredInput(port) - } + AudioSessionCoordinator.shared.setPreferredInput(uid) } // MARK: - Live route changes @@ -129,8 +129,7 @@ final class AudioInputProvider { switch reason { case .newDeviceAvailable, .oldDeviceUnavailable: guard permission == .granted else { return } - configureAndEnumerate(announceNewBluetooth: true) - onInputsChanged?() + configureAndEnumerate(announceNewBluetooth: true, notifyInputsChanged: true) default: break } @@ -141,20 +140,17 @@ final class AudioInputProvider { /// - Parameter announceNewBluetooth: when true, fire `onBluetoothConnected` /// for each Bluetooth input not seen on the previous enumeration. False for /// the initial `refresh()`, which only seeds the baseline set. - private func configureAndEnumerate(announceNewBluetooth: Bool = false) { - let session = AVAudioSession.sharedInstance() - // Setting the category is sufficient to enumerate Bluetooth inputs. Do - // not activate merely for enumeration; activation is asynchronous on - // iOS 27 and can disturb ScreenCaptureKit's microphone session. - for options in Self.bluetoothOptionLadder() { - do { - try session.setCategory(.playAndRecord, mode: .default, options: options) - break - } catch { - continue - } + private func configureAndEnumerate(announceNewBluetooth: Bool = false, + notifyInputsChanged: Bool = false) { + Task { [weak self] in + // Category negotiation may consult media-server synchronously. Keep it + // off the UI actor and ordered with meter/broadcast session handoffs. + await AudioSessionCoordinator.shared.prepareForEnumeration() + guard let self else { return } + enumerate(from: AVAudioSession.sharedInstance(), + announceNewBluetooth: announceNewBluetooth) + if notifyInputsChanged { onInputsChanged?() } } - enumerate(from: session, announceNewBluetooth: announceNewBluetooth) } private func enumerate(from session: AVAudioSession, announceNewBluetooth: Bool) { @@ -184,12 +180,21 @@ final class AudioInputProvider { /// prefer the built-in mic and hides Bluetooth HFP inputs (the reason DJI/BT /// mics weren't selectable). Tries the richest Bluetooth option set first and /// degrades, so one unsupported option never blocks enumeration. + /// Performs the potentially blocking audio-server category/activation handoff. + /// This is deliberately nonisolated so the session coordinator can run it on + /// its dedicated serial queue; + /// `AVAudioSession.setActive` may take long enough to visibly stall a sheet push. @discardableResult - static func activateBluetoothRecording(_ session: AVAudioSession) async -> Bool { + nonisolated static func activateBluetoothRecording(preferredInputUID: String?) -> Bool { + let session = AVAudioSession.sharedInstance() for options in bluetoothOptionLadder() { do { try session.setCategory(.playAndRecord, mode: .default, options: options) try session.setActive(true, options: []) + if let preferredInputUID, + let port = session.availableInputs?.first(where: { $0.uid == preferredInputUID }) { + try? session.setPreferredInput(port) + } return true } catch { continue @@ -200,7 +205,7 @@ final class AudioInputProvider { /// Bluetooth-capable record options, richest first. `.allowBluetoothHFP` is the /// iOS 26 replacement for the deprecated `.allowBluetooth`. - static func bluetoothOptionLadder() -> [AVAudioSession.CategoryOptions] { + nonisolated static func bluetoothOptionLadder() -> [AVAudioSession.CategoryOptions] { #if compiler(>=6.2) if #available(iOS 26.0, *) { return [ @@ -216,3 +221,516 @@ final class AudioInputProvider { #endif } } + +/// Serializes every app-owned `AVAudioSession` activation/deactivation away from +/// the main actor. In particular, leaving the Audio settings pane enqueues its +/// deactivation before a subsequent broadcast activation, so a late meter teardown +/// can never turn off ScreenCaptureKit's microphone session. +final class AudioSessionCoordinator: @unchecked Sendable { + static let shared = AudioSessionCoordinator() + + private let queue = DispatchQueue( + label: "com.joeblau.Stream.audio-session", + qos: .userInitiated + ) + + private init() {} + + /// Configures Bluetooth-capable recording without activating the session. + /// Category setup is enough for `availableInputs` enumeration. + func prepareForEnumeration() async { + await withCheckedContinuation { continuation in + queue.async { + let session = AVAudioSession.sharedInstance() + for options in AudioInputProvider.bluetoothOptionLadder() { + do { + try session.setCategory(.playAndRecord, mode: .default, + options: options) + break + } catch { + continue + } + } + continuation.resume() + } + } + } + + func setPreferredInput(_ uid: String) { + queue.async { + let session = AVAudioSession.sharedInstance() + guard let port = session.availableInputs?.first(where: { $0.uid == uid }) else { + return + } + try? session.setPreferredInput(port) + } + } + + func activate(preferredInputUID: String?) async -> Bool { + await withCheckedContinuation { continuation in + queue.async { + continuation.resume(returning: AudioInputProvider.activateBluetoothRecording( + preferredInputUID: preferredInputUID + )) + } + } + } + + /// Enqueues teardown immediately and returns without blocking the caller. + /// A later `activate` is submitted behind it on the same serial queue. + func deactivate() { + queue.async { + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + } + } + + /// Used by capture teardown paths that must know the session is inactive before + /// completing. It shares ordering with the fire-and-forget meter teardown. + func deactivateAndWait() async { + await withCheckedContinuation { continuation in + queue.async { + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + continuation.resume() + } + } + } +} + +// MARK: - Microphone level metering + +/// The smoothed 0...1 microphone level, shared by the main toolbar's status meter +/// and the Audio settings pane. While a broadcast is live the level arrives free +/// over `MicrophoneLevelChannel` (published by the capture pipeline, already +/// post-gain); when idle the monitor opens its own `AVAudioEngine` tap on the +/// user's selected input. +/// +/// Exactly one instance exists — owned by `ContentView` — because a second one +/// would open a competing record session. Callers therefore start/stop it with a +/// `Client` token and the meter runs until the last of them lets go. +@MainActor +@Observable +final class MicrophoneLevelMonitor { + private(set) var level = 0.0 + private(set) var isReceiving = false + + private static let log = Logger(subsystem: "com.joeblau.Stream", category: "mic-meter") + @ObservationIgnored private let channel = MicrophoneLevelChannel() + @ObservationIgnored private var task: Task? + @ObservationIgnored private var engine: AVAudioEngine? + @ObservationIgnored private let meterLevel = OSAllocatedUnfairLock(initialState: 0) + @ObservationIgnored private var gain = 1.0 + @ObservationIgnored private var preferredInputUID: String? + @ObservationIgnored private var nextRecorderAttemptAt: UInt64 = 0 + @ObservationIgnored private var broadcastIsLive = false + @ObservationIgnored private var activationInFlight = false + /// Invalidates an activation that is awaiting the serialized audio-session + /// queue. Teardown enqueues deactivation immediately; a stale continuation must + /// then return without submitting another operation behind a new broadcast. + @ObservationIgnored private var localCaptureGeneration: UInt = 0 + + /// Observer for `AVAudioEngineConfigurationChange`. The engine posts it when its + /// I/O format changes underneath the running graph — most importantly when a + /// Bluetooth HFP link finishes its asynchronous handoff after `setPreferredInput`. + /// The tap was installed at the pre-switch format, so without rebuilding it on + /// this notification the meter goes silent once the route lands on the BT mic. + @ObservationIgnored private nonisolated(unsafe) var configChangeObserver: NSObjectProtocol? + + /// A surface currently displaying the meter. The toolbar meter and the Audio + /// settings pane can be on screen at the same time, so the sampling loop is + /// reference-counted across them — closing the settings sheet mid-broadcast + /// must not silence the toolbar meter behind it. + enum Client: Hashable { + case statusBar + case settings + } + + @ObservationIgnored private var clients: Set = [] + + func start(for client: Client) { + clients.insert(client) + startSampling() + } + + func stop(for client: Client) { + clients.remove(client) + guard clients.isEmpty else { return } + stopSampling() + } + + private func startSampling() { + guard task == nil else { return } + task = Task { [weak self] in + // Let the presenting transition (a settings push, or the go-live + // handoff) finish before touching AVAudioEngine. If the user backs out + // immediately, cancellation avoids starting audio at all. + do { + try await Task.sleep(for: .milliseconds(350)) + } catch { + return + } + while !Task.isCancelled { + do { + try await Task.sleep(nanoseconds: 100_000_000) + } catch { + return + } + guard let self else { return } + let target: Double + if let extensionLevel = channel.read() { + stopLocalCapture() + isReceiving = true + target = Double(extensionLevel) + } else if broadcastIsLive { + // Never open a second mic capture while ScreenCaptureKit's + // microphone is paused/off and not publishing meter samples. + stopLocalCapture() + isReceiving = false + target = 0 + } else if let localLevel = await readLocalLevel() { + isReceiving = true + target = localLevel + } else { + isReceiving = false + target = 0 + } + var next = target >= level + ? level * 0.3 + target * 0.7 + : max(target, level * 0.82) + if next < 0.005 { next = 0 } + // Only publish on change — @Observable fires on every set, so an + // idle meter otherwise re-renders the view 20x/s for nothing. + if next != level { level = next } + } + } + } + + private func stopSampling() { + task?.cancel() + task = nil + invalidateLocalCapture() + level = 0 + isReceiving = false + } + + func setGain(_ gain: Double) { + self.gain = max(0, min(gain, 2)) + } + + func setBroadcasting(_ isLive: Bool) { + guard broadcastIsLive != isLive else { return } + broadcastIsLive = isLive + if isLive { + invalidateLocalCapture() + } else { + nextRecorderAttemptAt = 0 + } + } + + /// Routes the local meter to the user's selected input (incl. Bluetooth/DJI). + /// Without this the recorder captures the built-in mic, so a selected DJI/BT + /// mic never registers on the meter. + func setPreferredInput(_ uid: String?) { + guard uid != preferredInputUID else { return } + preferredInputUID = uid + restartLocalCapture() + } + + func restartLocalCapture() { + invalidateLocalCapture() + nextRecorderAttemptAt = 0 + } + + private func readLocalLevel() async -> Double? { + if engine == nil { await startLocalCaptureIfAvailable() } + guard let engine, engine.isRunning else { return nil } + // RMS captured on the audio render thread by the input tap. + let rms = Double(meterLevel.withLock { $0 }) + let postGain = max(0.000_001, min(1, rms * gain)) + let decibels = 20 * log10(postGain) + return max(0, min(1, (decibels + 60) / 60)) + } + + private func startLocalCaptureIfAvailable() async { + guard AVAudioApplication.shared.recordPermission == .granted else { return } + let now = DispatchTime.now().uptimeNanoseconds + guard now >= nextRecorderAttemptAt else { return } + nextRecorderAttemptAt = now + 1_000_000_000 + let generation = localCaptureGeneration + + // Route the meter to the SAME input the user picked. Without activating a + // Bluetooth-capable record session and setting the preferred input, the + // recorder silently captures the built-in mic — so a selected DJI/BT mic + // never registers on the meter. (Only reached when NOT broadcasting, so it + // never competes with the extension's session.) + let uid = preferredInputUID + activationInFlight = true + let activated = await AudioSessionCoordinator.shared.activate( + preferredInputUID: uid + ) + activationInFlight = false + guard activated else { return } + // `stop()` can cancel this task while audio-server activation is in flight. + // Do not resurrect an engine after the pane disappeared or a broadcast took + // ownership; the queued deactivation remains ordered with future activation. + guard !Task.isCancelled, !broadcastIsLive, + generation == localCaptureGeneration else { return } + var keepSessionActive = false + defer { + if !keepSessionActive { AudioSessionCoordinator.shared.deactivate() } + } + let session = AVAudioSession.sharedInstance() + + // Meter from the engine's input node — it reflects the ACTIVE input route + // (the DJI once it's the preferred input) and gives real PCM buffers, unlike + // the /dev/null AVAudioRecorder metering trick which can read nothing. + let engine = AVAudioEngine() + let input = engine.inputNode + let inputFormat = input.inputFormat(forBus: 0) + guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { return } + // A tap alone does NOT reliably pull a Bluetooth HFP input — the engine only + // renders its input when the graph drives an output, so an unconnected + // input node hands the tap silent buffers (the "DJI selected, meter flat" + // bug). Route input → main mixer to force a full I/O cycle, and mute the + // mixer output so nothing is monitored back to the speaker/HFP earpiece. + do { + try engine.connectNode(input, to: engine.mainMixerNode, format: inputFormat) + } catch { + return + } + engine.mainMixerNode.outputVolume = 0 + guard installMeterTap(on: engine) else { return } + engine.prepare() + do { + try engine.start() + } catch { + engine.inputNode.removeTap(onBus: 0) + return + } + self.engine = engine + keepSessionActive = true + + // A Bluetooth HFP link settles asynchronously AFTER setPreferredInput, so the + // format above may still be the built-in mic's. Rebuild the tap on the new + // format when the engine reconfigures, else the BT meter reads flat forever. + observeConfigChanges(of: engine) + + let route = session.currentRoute.inputs + .map { "\($0.portName)/\($0.portType.rawValue)" } + .joined(separator: ",") + let format = engine.inputNode.inputFormat(forBus: 0) + Self.log.info("Mic meter started: route=[\(route, privacy: .public)] format=\(format.sampleRate, privacy: .public)Hz/\(format.channelCount, privacy: .public)ch pref=\(self.preferredInputUID ?? "nil", privacy: .public)") + } + + /// Reads the input node's CURRENT format and installs the metering tap. Split out + /// so it can be re-run on a configuration change (when the live route/format has + /// changed). Returns false when the route has no usable input yet. + private func installMeterTap(on engine: AVAudioEngine) -> Bool { + let input = engine.inputNode + let format = input.inputFormat(forBus: 0) + guard format.sampleRate > 0, format.channelCount > 0 else { return false } + let level = meterLevel + // @Sendable so the tap runs on the audio render thread — WITHOUT it the + // closure inherits this @MainActor class's isolation and iOS crashes with + // a libdispatch queue assertion when the render thread invokes it. + do { + try input.__installTap(onBus: 0, bufferSize: 1024, format: format, error: ()) { + @Sendable buffer, _ in + guard let channel = buffer.floatChannelData?[0] else { return } + let count = Int(buffer.frameLength) + guard count > 0 else { return } + var sumOfSquares: Float = 0 + for i in 0.. Bool { + guard !isMuted, monitor.isReceiving else { return false } + return monitor.level > Double(index) / Double(Self.segmentCount) + } + + /// Green through the healthy range, amber approaching the ceiling, red at it — + /// same green→yellow→red reading as the full meter in Audio settings. + private func segmentTint(_ index: Int) -> Color { + switch index { + case Self.segmentCount - 1: return .red + case Self.segmentCount - 2: return .yellow + default: return .green + } + } + + private var accessibilityValue: String { + if isMuted { return "Muted" } + guard monitor.isReceiving else { return "No signal" } + return "\(Int((monitor.level * 100).rounded())) percent" + } +} diff --git a/Stream/CameraSupport.swift b/Stream/CameraSupport.swift index 6fc1746..8b43b53 100644 --- a/Stream/CameraSupport.swift +++ b/Stream/CameraSupport.swift @@ -22,16 +22,23 @@ final class CameraSupport { private(set) var permission: Permission = .undetermined /// Whether the camera can run during a system broadcast on this device. - let multitaskingSupported: Bool + private(set) var multitaskingSupported = false init() { - // Reading the capability off a non-running session reflects the hardware. - multitaskingSupported = AVCaptureSession().isMultitaskingCameraAccessSupported syncPermission() } func refresh() { syncPermission() + // Constructing an AVCaptureSession can synchronously consult media-server + // state. Do it only when PiP is opened and never in the sheet's main-actor + // presentation turn. + Task { + let supported = await Task.detached(priority: .utility) { + AVCaptureSession().isMultitaskingCameraAccessSupported + }.value + multitaskingSupported = supported + } } /// Requests camera permission when the user enables the facecam. diff --git a/Stream/ContentView.swift b/Stream/ContentView.swift index eb62a3f..9e480f2 100644 --- a/Stream/ContentView.swift +++ b/Stream/ContentView.swift @@ -15,9 +15,11 @@ private struct BluetoothConnection: Equatable, Identifiable { /// sheet. Settings are owned here and threaded into `SettingsView`; every edit is /// persisted via `SettingsStore` before ScreenCaptureKit starts the stream. struct ContentView: View { + @Environment(\.scenePhase) private var scenePhase + /// The single source of truth for the editable settings, loaded from the /// shared App Group suite on launch. - @State private var settings: StreamSettings = SettingsStore().load() + @State private var settings: StreamSettings /// Owns ScreenCaptureKit selection/capture and the network publisher. @State private var capture = ScreenCaptureController() @@ -31,6 +33,11 @@ struct ContentView: View { /// and is threaded into the Settings sheet so both share one instance/observer. @State private var audio = AudioInputProvider() + /// The single mic level monitor, driving both the toolbar's status meter and the + /// Audio settings pane. Owned here so one instance serves both — a second would + /// open a competing record session whenever the meter falls back to local capture. + @State private var micLevel = MicrophoneLevelMonitor() + /// The Bluetooth device that most recently paired mid-session, shown as a /// transient banner. Set from `audio.onBluetoothConnected`; auto-cleared after /// a few seconds by a `.task` keyed on its `id` (a fresh `id` restarts the @@ -39,6 +46,16 @@ struct ContentView: View { @State private var showingSettings = false + /// Coalesces rapid UI edits and performs Keychain/file writes away from the + /// main actor. Text fields and sliders can update dozens of times per second; + /// synchronously persisting each intermediate value made sheet navigation and + /// control drags contend with Security.framework and atomic file I/O. + @State private var settingsPersistence: SettingsPersistenceCoordinator + + /// Prevents a second go-live tap while the final settings snapshot is being + /// flushed. Capture never starts before the latest scheduled save completes. + @State private var isPreparingBroadcast = false + /// Gates the "Stop Broadcast" confirmation. A live stream is easy to kill by a /// stray tap on the toolbar's stop button, so tapping it while live asks first /// instead of tearing the broadcast down immediately. @@ -48,17 +65,31 @@ struct ContentView: View { /// moment they mute so toggling back returns to their chosen level, not unity. @State private var preMuteVolume: Double = 1.0 - /// Persists `settings` into the shared App Group suite. Called on every edit. + init() { + let loaded = SettingsStore().load() + _settings = State(initialValue: loaded) + _settingsPersistence = State( + initialValue: SettingsPersistenceCoordinator(initialSettings: loaded) + ) + } + + /// Schedules a coalesced background save. `startBroadcast()` and sheet dismissal + /// explicitly flush the latest snapshot, so debounce never weakens durability. private func persist() { - SettingsStore().save(settings) + settingsPersistence.schedule(settings) + } + + private func flushSettings() { + let snapshot = settings + Task { await settingsPersistence.flush(snapshot) } } /// A gain at/below this reads as muted — matches the Settings "Muted" label. private var isMicMuted: Bool { settings.micVolume <= 0.0001 } /// Toggles the microphone between muted (gain 0) and the last chosen level. - /// Persists and posts the live-apply signal so a running broadcast responds - /// immediately, exactly like the Settings mic-volume slider does. + /// Persists and applies directly so a running broadcast responds immediately, + /// exactly like the Settings mic-volume slider does. private func toggleMicMute() { Haptics.tap() if isMicMuted { @@ -68,7 +99,30 @@ struct ContentView: View { settings.micVolume = 0 } persist() - BroadcastControl.post(BroadcastControl.micVolumeSignal) + capture.setMicVolume(settings.micVolume) + // The idle meter applies gain itself, so the toolbar bars have to be told + // about a mute that didn't come from the Settings slider. + micLevel.setGain(settings.micVolume) + } + + /// Keeps the toolbar meter live whenever the app is foregrounded, so the mic can + /// be verified before going live and not only during a broadcast. While live the + /// level arrives free over `MicrophoneLevelChannel`; when idle the monitor opens + /// its own input tap, which is why this releases it the moment the app leaves the + /// foreground rather than holding the mic (and the orange privacy indicator) open + /// behind other apps. + /// + /// - Parameter phase: the incoming phase when called from `onChange`, whose + /// closure still sees the pre-change `scenePhase`. + private func syncStatusMeter(phase: ScenePhase? = nil) { + micLevel.setBroadcasting(capture.isLive) + micLevel.setGain(settings.micVolume) + micLevel.setPreferredInput(settings.preferredAudioInputUID) + if (phase ?? scenePhase) == .active { + micLevel.start(for: .statusBar) + } else { + micLevel.stop(for: .statusBar) + } } var body: some View { @@ -89,6 +143,8 @@ struct ContentView: View { .animation(.spring(duration: 0.3, bounce: 0.2), value: capture.isLive) .animation(.spring(duration: 0.35, bounce: 0.25), value: bluetoothBanner) .navigationTitle("Stream") + // Kept for VoiceOver/system context only — the principal toolbar + // item above replaces the visible title with the mic meter. .navigationBarTitleDisplayMode(.inline) .toolbar { ToolbarItem(placement: .topBarLeading) { @@ -100,6 +156,12 @@ struct ContentView: View { } .accessibilityLabel("Settings") } + // Takes the title's slot: "Stream" is redundant on the app's own + // root screen, and the mic meter is the one thing worth a glance + // every time you look up here. + ToolbarItem(placement: .principal) { + MicrophoneStatusMeter(monitor: micLevel, isMuted: isMicMuted) + } ToolbarItem(placement: .topBarTrailing) { broadcastButton } } .safeAreaInset(edge: .bottom) { @@ -107,7 +169,9 @@ struct ContentView: View { setupBanner } } - .sheet(isPresented: $showingSettings) { settingsSheet } + .sheet(isPresented: $showingSettings, onDismiss: flushSettings) { + settingsSheet + } } .task { chat.autoConnect() @@ -116,12 +180,16 @@ struct ContentView: View { bluetoothBanner = BluetoothConnection(name: name) Haptics.tap() } - // Seed the baseline set + start the route-change observer without - // prompting for mic access here (Settings owns the permission ask). - // Already-connected devices only populate the baseline; they don't - // trigger a banner — only devices that arrive afterward do. - audio.refresh(requestPermission: false) + Haptics.warmUp() + // Seed the baseline set + start the route-change observer. The ask has + // to happen here now that the toolbar meter is always on screen — + // without permission it can only ever read "no signal", with nothing on + // the main screen to explain why. Already-connected devices only + // populate the baseline; they don't trigger a banner — only arrivals do. + audio.refresh(requestPermission: true) + syncStatusMeter() } + .onChange(of: capture.isLive) { _, _ in syncStatusMeter() } // Auto-dismiss the Bluetooth banner. Keyed on the connection id so a new // device restarts the timer; cancellation (id change) skips the stale clear. .task(id: bluetoothBanner?.id) { @@ -130,6 +198,10 @@ struct ContentView: View { guard !Task.isCancelled else { return } bluetoothBanner = nil } + .onChange(of: scenePhase) { _, phase in + if phase != .active { flushSettings() } + syncStatusMeter(phase: phase) + } .confirmationDialog( "Stop the broadcast?", isPresented: $showingStopConfirmation, @@ -162,16 +234,30 @@ struct ContentView: View { if capture.isLive { showingStopConfirmation = true } else { - persist() - capture.presentPicker(settings: settings) + startBroadcast() } } label: { Image(systemName: capture.isLive ? "stop.circle" : "record.circle") } .tint(capture.isLive ? .red : .primary) + .disabled(isPreparingBroadcast || capture.isStopping) .accessibilityLabel(capture.isLive ? "Stop broadcast" : "Start broadcast") } + /// Flushes through the serial writer before handing the immutable snapshot to + /// ScreenCaptureKit. This preserves the existing "saved before live" contract + /// without doing Keychain or disk work in the toolbar button's main-actor turn. + private func startBroadcast() { + guard !isPreparingBroadcast, !capture.isStopping else { return } + isPreparingBroadcast = true + let snapshot = settings + Task { + await settingsPersistence.flush(snapshot) + isPreparingBroadcast = false + capture.presentPicker(settings: snapshot) + } + } + // MARK: - Mute FAB /// Floating mic mute/unmute control, pinned to the bottom-trailing corner while @@ -288,11 +374,74 @@ struct ContentView: View { // MARK: - Settings sheet - /// One Vaul-style drawer. `SettingsView` owns the navigation stack and sizes - /// the sheet to the exact height of whatever content is on screen (measured - /// from the scroll view's real content size), resizing as sections are pushed. + /// One Vaul-style drawer. `SettingsView` owns its two-level navigation and + /// measures only the visible pane to drive its content-hugging detent. private var settingsSheet: some View { - SettingsView(settings: $settings, chat: chat, capture: capture, onChange: persist, audio: audio) + SettingsView(settings: $settings, chat: chat, capture: capture, + onChange: persist, audio: audio, micLevel: micLevel) + } +} + +/// Serializes settings writes so an older debounced snapshot can never finish +/// after a newer flush and overwrite it. Actor isolation also keeps the synchronous +/// Keychain + atomic-file implementation off the main actor. +private actor SettingsWriter { + private let store = SettingsStore() + private var lastSaved: StreamSettings + + init(lastSaved: StreamSettings) { + self.lastSaved = lastSaved + } + + func save(_ settings: StreamSettings) { + if settings.selectedProtocol != lastSaved.selectedProtocol + || settings.rtmpURL != lastSaved.rtmpURL + || settings.streamKey != lastSaved.streamKey { + store.saveConnection(settings) + } + + var redacted = settings + redacted.rtmpURL = "" + redacted.streamKey = "" + var previousRedacted = lastSaved + previousRedacted.rtmpURL = "" + previousRedacted.streamKey = "" + if redacted != previousRedacted { + store.saveNonSecret(settings) + } + lastSaved = settings + } +} + +/// Main-actor scheduler owned by `ContentView`. Cancellation is cheap while the +/// task is sleeping; once a write reaches `SettingsWriter`, subsequent writes queue +/// behind it and therefore retain strict snapshot order. +@MainActor +private final class SettingsPersistenceCoordinator { + private let writer: SettingsWriter + private var pending: Task? + + init(initialSettings: StreamSettings) { + writer = SettingsWriter(lastSaved: initialSettings) + } + + func schedule(_ settings: StreamSettings) { + pending?.cancel() + pending = Task { [writer] in + do { + try await Task.sleep(for: .milliseconds(250)) + } catch { + return + } + guard !Task.isCancelled else { return } + await writer.save(settings) + } + } + + func flush(_ settings: StreamSettings) async { + pending?.cancel() + pending = nil + await writer.save(settings) } } diff --git a/Stream/DynamicDetentSheet.swift b/Stream/DynamicDetentSheet.swift index c792d44..a13eea8 100644 --- a/Stream/DynamicDetentSheet.swift +++ b/Stream/DynamicDetentSheet.swift @@ -69,6 +69,13 @@ final class DynamicDetentSheetModel { max(minimumContentHeight, content) + chrome } + /// Whether a pane already has a cached intrinsic height. Callers can use this + /// to choose between a fully synchronized cached resize and a lazy first-open + /// transition whose visible measurement starts the resize a frame later. + func hasMeasurement(for key: Key) -> Bool { + measured[AnyHashable(key)] != nil + } + /// Raw measurement from a pane's scroll/intrinsic geometry. Cheap + idempotent: /// rounded, thresholded, cached under the pane's key, and it only commits an /// ANIMATED resize when the reporting pane is the ACTIVE one. A late measurement diff --git a/Stream/Haptics.swift b/Stream/Haptics.swift index d07b5e9..e10e98d 100644 --- a/Stream/Haptics.swift +++ b/Stream/Haptics.swift @@ -2,10 +2,26 @@ import UIKit /// Centralized haptic feedback so every control interaction feels consistent. /// Respects the system's haptic settings (no-ops when the user disables them). +@MainActor enum Haptics { + /// One long-lived generator, kept warm. Allocating a generator per tap makes + /// the Taptic Engine cold-start inside the button's action — a stall the user + /// feels as the whole control responding late, not just the haptic. Reusing a + /// prepared instance keeps the engine spun up so the tap fires immediately. + private static let impact = UIImpactFeedbackGenerator(style: .light) + /// A light tap for button presses. Call from SwiftUI action closures, which /// already run on the main actor. - @MainActor static func tap() { - UIImpactFeedbackGenerator(style: .light).impactOccurred() + static func tap() { + impact.impactOccurred() + // Re-arm for the next press. The engine idles down after a couple of + // seconds, so this is what keeps repeat taps snappy. + impact.prepare() + } + + /// Warms the Taptic Engine ahead of the first interaction (scene activation), + /// so even the very first button press doesn't pay the spin-up cost. + static func warmUp() { + impact.prepare() } } diff --git a/Stream/ScreenCaptureController.swift b/Stream/ScreenCaptureController.swift index bb94520..07eceed 100644 --- a/Stream/ScreenCaptureController.swift +++ b/Stream/ScreenCaptureController.swift @@ -17,6 +17,10 @@ private let captureLog = Logger(subsystem: "com.joeblau.Stream", category: "scre @Observable final class ScreenCaptureController: NSObject { private(set) var isLive = false + /// Remains true until capture, publisher, and audio-session teardown all finish. + /// The record button stays disabled so a new activation cannot race the old + /// broadcast's final deactivation. + private(set) var isStopping = false private(set) var errorMessage: String? /// A user-facing note when the device's thermal/power state is forcing a /// quality reduction, or nil when unrestricted. Shown in the live stats HUD. @@ -77,7 +81,10 @@ final class ScreenCaptureController: NSObject { } func presentPicker(settings: StreamSettings) { - guard stream == nil else { return } + // `pendingSettings` is set before the serialized audio activation awaits. + // Reject a second toolbar tap during that window so only one activation and + // one system-picker presentation can be queued. + guard stream == nil, pendingSettings == nil, !isStopping else { return } guard settings.isPublishable else { errorMessage = "Complete the connection settings before starting a stream." return @@ -95,7 +102,7 @@ final class ScreenCaptureController: NSObject { } private func presentSystemPicker(settings: StreamSettings) { - guard pendingSettings != nil, stream == nil else { return } + guard pendingSettings != nil, stream == nil, !isStopping else { return } var configuration = SCContentSharingPickerConfiguration() configuration.showsMicrophoneControl = true // ScreenCaptureKit's system camera effect only supports current-app @@ -208,8 +215,18 @@ final class ScreenCaptureController: NSObject { let configuration = SCStreamConfiguration() let nativeWidth = max(2, Int((filter.contentRect.width * CGFloat(filter.pointPixelScale)).rounded())) let nativeHeight = max(2, Int((filter.contentRect.height * CGFloat(filter.pointPixelScale)).rounded())) - configuration.width = nativeWidth - configuration.height = nativeHeight + // Ask ScreenCaptureKit for the actual encode dimensions up front. Capturing + // a native 3x display only to downscale it in VideoToolbox wastes memory + // bandwidth and GPU/encoder work (often 2–3x the pixels for a 720p stream). + // `encodeSize` preserves orientation/aspect, clamps to the device ceiling, + // and returns the even dimensions required by H.264/HEVC. + let captureSize = settings.encodeSize( + forOrientedWidth: nativeWidth, + height: nativeHeight, + maxShortEdge: StreamCapability.current.maxShortEdge + ) + configuration.width = Int(captureSize.width) + configuration.height = Int(captureSize.height) configuration.capturesAudio = settings.includeAppAudio configuration.sampleRate = 48_000 configuration.channelCount = 2 @@ -263,6 +280,10 @@ final class ScreenCaptureController: NSObject { } private func stopCapture() async { + guard !isStopping else { return } + isStopping = true + defer { isStopping = false } + let stream = self.stream let publisher = self.publisher let output = self.output @@ -319,9 +340,16 @@ final class ScreenCaptureController: NSObject { } private func applyMicVolume() { - guard let publisher else { return } let volume = SettingsStore().load().micVolume + setMicVolume(volume) + } + + /// Applies an in-memory UI edit directly to the live pipeline. This avoids a + /// synchronous settings reload (and a race with debounced persistence) for the + /// Settings slider and mute button; the Darwin observer remains as a fallback. + func setMicVolume(_ volume: Double) { output?.setMicVolume(volume) + guard let publisher else { return } Task { await publisher.setMicVolume(volume) } } @@ -354,40 +382,17 @@ final class ScreenCaptureController: NSObject { } private func configurePreferredMicrophone(_ uid: String?) async { - let session = AVAudioSession.sharedInstance() - var activated = false - for options in AudioInputProvider.bluetoothOptionLadder() { - do { - try session.setCategory(.playAndRecord, mode: .default, options: options) - try session.setActive(true, options: []) - activated = true - break - } catch { - continue - } - } + let activated = await AudioSessionCoordinator.shared.activate( + preferredInputUID: uid + ) guard activated else { captureLog.warning("Could not preconfigure the preferred microphone; ScreenCaptureKit will use the system default") return } - guard let uid, - let input = session.availableInputs?.first(where: { $0.uid == uid }) else { return } - do { - try session.setPreferredInput(input) - } catch { - captureLog.warning("Preferred microphone route failed: \(String(describing: error), privacy: .public)") - } } private func deactivateAudioSession() async { - do { - try AVAudioSession.sharedInstance().setActive( - false, - options: [.notifyOthersOnDeactivation] - ) - } catch { - captureLog.warning("Audio-session deactivation failed: \(String(describing: error), privacy: .public)") - } + await AudioSessionCoordinator.shared.deactivateAndWait() } private func publishState(_ live: Bool) { @@ -451,7 +456,7 @@ private final class UncheckedSendableBox: @unchecked Sendable { private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable { let sampleQueue = DispatchQueue(label: "com.joeblau.Stream.screen-capture.samples", - qos: .userInteractive) + qos: .userInitiated) private let publisher: any Publisher private let settings: StreamSettings @@ -468,6 +473,10 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg private let micLevelChannel = MicrophoneLevelChannel() private let facecam = FacecamCapture() private let compositor = FacecamCompositor() + /// Runtime thermal/Low-Power gate. The user setting is immutable for a live + /// session, but the governor can disable PiP without rebuilding the output. + /// The video consumer and governor run on different executors, so guard it. + private let pipAllowed: OSAllocatedUnfairLock private let videoSamples: AsyncStream private let videoContinuation: AsyncStream.Continuation private var videoConsumer: Task? @@ -489,6 +498,7 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg self.telemetry = telemetry self.onStopped = onStopped micLevelMeter = ScreenCaptureMicrophoneMeter(gain: settings.micVolume) + pipAllowed = OSAllocatedUnfairLock(initialState: settings.pipEnabled) targetFrameInterval = CMTime(value: 1, timescale: CMTimeScale(settings.encodeFrameRate(maxFrameRate: capability.maxFrameRate))) (videoSamples, videoContinuation) = AsyncStream.makeStream( @@ -511,10 +521,11 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg await publisher.setOutputSize(target, nativeShortEdge: min(width, height)) targetSize = target } - if settings.pipEnabled, let targetSize, let camera = self.facecam.latest.take() { + let shouldComposite = self.pipAllowed.withLock { $0 } + if shouldComposite, let targetSize { if let composited = self.compositor.composite( screen: image, - camera: camera, + camera: self.facecam.latest.freshest(), targetSize: targetSize, orientation: .up, corner: settings.pipCorner, @@ -527,11 +538,10 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg ) { await publisher.appendVideo(output) } else { - // Pool exhausted (a slow encoder holding surfaces) or the wrap - // failed: skip the overlay for this frame and send the raw - // screen. The frame itself is NOT lost — the overlay is. + // A raw native-size fallback would change the encoder's input + // format and force a VideoToolbox session rebuild. Drop this + // frame instead; the next pooled target-size frame stays stable. self.telemetry.recordDrop(.compositor) - await publisher.appendVideo(sampleBuffer) } } else { await publisher.appendVideo(sampleBuffer) @@ -541,6 +551,7 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg } func finish() { + pipAllowed.withLock { $0 = false } micLevelChannel.publish(0) facecam.stop() videoContinuation.finish() @@ -604,6 +615,8 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg /// power. Re-arms the facecam when the device recovers. PiP toggling only /// applies when the user has the facecam enabled. func applyThermalProfile(frameRateCap: Int, allowPiP: Bool) { + let shouldAllowPiP = settings.pipEnabled && allowPiP + pipAllowed.withLock { $0 = shouldAllowPiP } sampleQueue.async { [weak self] in guard let self else { return } // Cap to the device ceiling first, then the thermal frame-rate cap. @@ -612,7 +625,7 @@ private final class ScreenCaptureOutput: NSObject, SCStreamOutput, SCStreamDeleg self.targetFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps)) } guard settings.pipEnabled else { return } - if allowPiP { + if shouldAllowPiP { facecam.start(with: settings) } else { facecam.stop() @@ -728,6 +741,7 @@ private final class ScreenCaptureMicrophoneMeter: @unchecked Sendable { @Observable final class ScreenCaptureController { private(set) var isLive = false + private(set) var isStopping = false private(set) var errorMessage: String? private(set) var thermalNotice: String? private(set) var liveStats: LiveStats? @@ -739,6 +753,8 @@ final class ScreenCaptureController { func stop() {} + func setMicVolume(_ volume: Double) {} + func clearError() { errorMessage = nil } diff --git a/Stream/SettingsView.swift b/Stream/SettingsView.swift index b86d96a..d32c219 100644 --- a/Stream/SettingsView.swift +++ b/Stream/SettingsView.swift @@ -6,7 +6,7 @@ import UIKit import os /// The settings sections. Each is launched from the settings list into its own -/// Vaul-style, content-sized sheet holding exactly that section's controls. +/// Vaul-style sheet holding exactly that section's controls. private enum SettingsSection: String, Identifiable, CaseIterable { case connection, video, backup, audio, pip, chat @@ -46,8 +46,8 @@ private enum SettingsSection: String, Identifiable, CaseIterable { } } -/// The settings launcher: a list of sections, each of which opens a dynamic, -/// content-sized sheet holding exactly that section's controls. Every field edit +/// The settings launcher: a list of sections, each of which opens its controls in +/// the same content-hugging sheet. Every field edit /// mutates the bound `settings` and calls `onChange()` so the parent persists the /// snapshot via `SettingsStore` BEFORE the user can start a broadcast. struct SettingsView: View { @@ -76,8 +76,10 @@ struct SettingsView: View { /// Photos add-only permission helper for the Local Backup feature. @State private var photos = PhotosSupport() - /// Smoothed live level published by the ScreenCaptureKit sample router. - @State private var micLevel = MicrophoneLevelMonitor() + /// Smoothed live level published by the ScreenCaptureKit sample router. Owned by + /// the root view and shared with the toolbar's status meter — two instances would + /// open competing record sessions when the meter falls back to local capture. + var micLevel: MicrophoneLevelMonitor // MARK: - Chat credential fields @@ -130,18 +132,16 @@ struct SettingsView: View { /// The section currently shown, or `nil` for the launcher. Not a full stack — /// the drawer is exactly two levels deep (launcher → one section), so a single /// optional models it. Owning the level ourselves (instead of a NavigationStack - /// push) lets ONE `withAnimation` drive the content morph and the detent resize - /// together as a single motion. + /// push) lets one animation drive the content morph and sheet resize together. @State private var selected: SettingsSection? - /// Owns the content-sized detent and animates every resize with ITS curve, and — - /// crucially — guards a late re-measurement from restarting the spring mid-resize - /// (the real cause of the height feeling "desynced"). The resize speed lives in - /// this one initializer: 0.45s reads as settled where the old 0.3s outran the slide. + /// Vaul-style content-hugging detent. Only the pane currently on screen reports + /// a height; previously visited panes are cached by the model. This preserves + /// the dynamic drawer without rebuilding six hidden Forms behind every update. @State private var detent = DynamicDetentSheetModel( initialHeight: 420, - chrome: 100, // nav bar + grabber + home-indicator inset - minimumContentHeight: 160, // floor so a short section never collapses + chrome: 100, + minimumContentHeight: 160, animation: .spring(duration: 0.45, bounce: 0.1) ) @@ -150,8 +150,7 @@ struct SettingsView: View { // bar, glass toolbar buttons, and inline title. It does NOT push via the nav // stack: the launcher and section are swapped with a directional slide // (section in from trailing / launcher out to leading = a forward push; the - // reverse on back = a pop), driven by our own `withAnimation` so the slide - // and the detent resize ride the same transaction. + // reverse on back = a pop), driven by our own `withAnimation`. NavigationStack { ZStack(alignment: .top) { if let section = selected { @@ -163,9 +162,6 @@ struct SettingsView: View { .transition(.move(edge: .leading)) } } - // Measure every section's height off-screen so the FIRST open resizes - // in lockstep with the slide, not a beat after it. - .background(sectionHeightPrewarm) .navigationTitle(selected?.title ?? "Settings") .navigationBarTitleDisplayMode(.inline) .toolbar { @@ -180,53 +176,63 @@ struct SettingsView: View { } } } - .dynamicHeightDetents(detent) // curve-controlled height + drag indicator + .dynamicHeightDetents(detent) .presentationCornerRadius(55) // Haptic when a section is opened (push) or closed (pop). .sensoryFeedback(.impact(weight: .light), trigger: selected) - .onAppear { - // Launcher is the active pane on present; in-place field growth then - // animates automatically via each pane's measuredDetentHeight report. + .onAppear { [audio, micLevel, capture] in detent.activate(SettingsSection?.none) - // Enumerate inputs/capabilities so the launcher summaries are accurate; - // the live mic meter only runs while the Audio detail is open. - audio.refresh(requestPermission: false) // When a mic is (dis)connected mid-session, the provider re-enumerates // and calls this back: re-apply the persisted input to the session and // re-route the meter so a headset plugged in now is immediately live — // no app restart. Skipped while broadcasting (extension owns the route). - audio.onInputsChanged = { - guard !BroadcastStateStore.isLive() else { return } + audio.onInputsChanged = { [weak audio, weak micLevel, weak capture] in + guard let audio, let micLevel else { return } + guard capture?.isLive != true else { return } let uid = settings.preferredAudioInputUID audio.select(uid: uid, into: &settings) micLevel.setPreferredInput(uid) micLevel.restartLocalCapture() } - camera.refresh() - photos.refresh() + } + .onDisappear { + // `audio` outlives this sheet. Clear its callback so it cannot retain + // the sheet's Binding/state graph or restart a meter after dismissal. + audio.onInputsChanged = nil + micLevel.stop(for: .settings) } } - // MARK: - Drill-in navigation (custom, so the content morph + resize are one motion) + // MARK: - Drill-in navigation - /// Open a section: it slides in (push) and the drawer resizes to fit — both - /// inside ONE `withAnimation`, so they move as a single motion. + /// Open a section and resize the drawer under the same spring transaction. private func open(_ section: SettingsSection) { // No explicit tap here: `.sensoryFeedback(trigger: selected)` already fires - // one light impact whenever `selected` changes. The slide (.transition) and the - // detent resize ride ONE transaction (detent.animation), so they move together. - detent.animatingResize { - selected = section - detent.activate(Optional(section)) - } + // one light impact whenever `selected` changes. + navigate(to: section) } /// Return to the launcher (the nav-bar back button). private func back() { // Haptic comes from `.sensoryFeedback(trigger: selected)` (see `open`). - detent.animatingResize { - selected = nil - detent.activate(SettingsSection?.none) + navigate(to: nil) + } + + /// Cached panes resize in exact lockstep with the directional transition. On a + /// first visit there is intentionally no hidden prewarm tree; activate the key + /// first, then let its visible geometry report start the height spring while the + /// slide is already moving. + private func navigate(to destination: SettingsSection?) { + if detent.hasMeasurement(for: destination) { + detent.animatingResize { + selected = destination + detent.activate(destination) + } + } else { + detent.activate(destination) + withAnimation(detent.animation) { + selected = destination + } } } @@ -256,7 +262,7 @@ struct SettingsView: View { } } .scrollBounceBehavior(.basedOnSize) - .measuredDetentHeight(SettingsSection?.none, into: detent) // launcher's own identity + .measuredDetentHeight(SettingsSection?.none, into: detent) } // MARK: - Launcher rows @@ -290,12 +296,8 @@ struct SettingsView: View { // MARK: - Section detail body - /// One section's controls — the Form that fills the section pane below its - /// header. The enclosing drawer resizes to fit whichever section is on screen - /// (via `sectionHeights` → `currentTarget`), and re-sizes as fields appear/hide - /// within it. - /// A section's Form (controls only), reused by both the visible detail and the - /// hidden height pre-warm — so measuring it off-screen triggers no side effects. + /// One section's controls — the Form that fills the section pane and scrolls + /// when its content is taller than the current content-hugging detent. @ViewBuilder private func sectionForm(_ section: SettingsSection) -> some View { Form { @@ -315,33 +317,29 @@ struct SettingsView: View { /// own identity and runs the live mic meter only while the Audio detail is open. private func sectionDetail(_ section: SettingsSection) -> some View { sectionForm(section) - .measuredDetentHeight(Optional(section), into: detent) // this section's own identity + .measuredDetentHeight(Optional(section), into: detent) .onAppear { + switch section { + case .audio: + micLevel.setGain(settings.micVolume) + micLevel.setPreferredInput(settings.preferredAudioInputUID) + micLevel.setBroadcasting(capture.isLive) + micLevel.start(for: .settings) + case .pip: + camera.refresh() + case .backup: + photos.refresh() + default: + break + } + } + .onChange(of: capture.isLive) { _, isLive in guard section == .audio else { return } - micLevel.setGain(settings.micVolume) - micLevel.setPreferredInput(settings.preferredAudioInputUID) - micLevel.start() + micLevel.setBroadcasting(isLive) } .onDisappear { - if section == .audio { micLevel.stop() } - } - } - - /// Renders every section's Form once, hidden, so its content height is measured - /// and cached BEFORE its first open — then `open` resizes in lockstep with the - /// slide instead of a two-step (height jumping after the push). `report` only - /// CACHES for non-active panes (never resizes the sheet), and a `.background` - /// never affects the foreground's size. - private var sectionHeightPrewarm: some View { - ZStack { - ForEach(SettingsSection.allCases) { section in - sectionForm(section) - .measuredDetentHeight(Optional(section), into: detent) + if section == .audio { micLevel.stop(for: .settings) } } - } - .opacity(0) - .allowsHitTesting(false) - .accessibilityHidden(true) } // MARK: - Launcher summaries @@ -360,7 +358,7 @@ struct SettingsView: View { let quality = min(settings.videoQuality, capability.maxShortEdge) let fps = min(settings.frameRate, capability.maxFrameRate) // Surface HEVC (the opt-in) in the collapsed row; H.264 stays implicit. - let codec = settings.videoCodec == .hevc ? " · HEVC" : "" + let codec = settings.effectiveVideoCodec == .hevc ? " · HEVC" : "" return "\(quality)p · \(bitrateLabel(settings.videoBitrate)) · \(fps) fps\(codec)" case .backup: return "Off" @@ -409,6 +407,9 @@ struct SettingsView: View { var updated = settings SettingsStore().switchProtocol(to: newProtocol, in: &updated) settings = updated + // Cancels any pending snapshot for the previous protocol and + // queues this switched state behind the serial writer. + onChange() } )) { ForEach(StreamProtocol.allCases, id: \.self) { proto in @@ -522,10 +523,10 @@ struct SettingsView: View { } Picker("Codec", selection: Binding( - get: { settings.videoCodec }, + get: { settings.effectiveVideoCodec }, set: { settings.videoCodec = $0; onChange() } )) { - ForEach(VideoCodec.allCases, id: \.self) { codec in + ForEach(settings.selectedProtocol.supportedVideoCodecs, id: \.self) { codec in Text(codec.displayName).tag(codec) } } @@ -540,8 +541,8 @@ struct SettingsView: View { private var videoFooter: some View { VStack(alignment: .leading, spacing: 6) { Text("Bitrate is a maximum and drops automatically when the uplink is congested. Resolution and frame rate are limited to what this device can sustain, and are reduced automatically when it runs warm or low on power.") - if settings.videoCodec == .hevc { - Label("HEVC saves roughly 40% bitrate on screen content, but needs a compatible ingest — SRT, WHIP, or an enhanced-RTMP server. Traditional RTMP services (e.g. Restream) require H.264.", + if settings.effectiveVideoCodec == .hevc { + Label("HEVC saves roughly 40% bitrate on screen content, but needs a compatible ingest — SRT or an enhanced-RTMP server. WHIP and traditional RTMP services (e.g. Restream) require H.264.", systemImage: "info.circle") .foregroundStyle(.secondary) } @@ -611,7 +612,7 @@ struct SettingsView: View { } } - microphoneLevelMeter + MicrophoneLevelMeterView(monitor: micLevel) VStack(alignment: .leading) { HStack { @@ -637,7 +638,7 @@ struct SettingsView: View { onEditingChanged: { editing in // Apply live to a running broadcast once the drag settles. if !editing { - BroadcastControl.post(BroadcastControl.micVolumeSignal) + capture.setMicVolume(settings.micVolume) } } ) @@ -649,8 +650,10 @@ struct SettingsView: View { Spacer() Button { Haptics.tap() - micLevel.restartLocalCapture() audio.refresh(requestPermission: true) + // Session enumeration and the meter restart share one serial + // audio queue, so repeated refreshes cannot overlap handoffs. + micLevel.restartLocalCapture() } label: { Label("Refresh Inputs", systemImage: "arrow.clockwise") } @@ -663,43 +666,6 @@ struct SettingsView: View { } } - private var microphoneLevelMeter: some View { - VStack(alignment: .leading, spacing: 8) { - HStack { - Text("Mic Level") - Spacer() - Text(micLevel.isReceiving ? "Live" : "No signal") - .font(.caption) - .foregroundStyle(.secondary) - } - - GeometryReader { geometry in - ZStack(alignment: .leading) { - Capsule().fill(.quaternary) - // Gradient is FIXED across the full track (green at 0% → red at - // 100%); the fill just reveals it up to the current level, so a - // quiet signal shows only green, not a shrunk green→red bar. - LinearGradient( - colors: [.green, .yellow, .red], - startPoint: .leading, - endPoint: .trailing - ) - .frame(width: geometry.size.width) - .mask(alignment: .leading) { - Capsule() - .frame(width: geometry.size.width * micLevel.level) - } - } - } - .frame(height: 10) - .animation(.linear(duration: 0.05), value: micLevel.level) - .accessibilityLabel("Microphone level") - .accessibilityValue(micLevel.isReceiving - ? "\(Int((micLevel.level * 100).rounded())) percent" - : "No signal") - } - } - @ViewBuilder private var audioFooter: some View { switch audio.permission { @@ -874,260 +840,10 @@ struct SettingsView: View { } } -@MainActor -@Observable -private final class MicrophoneLevelMonitor { - private(set) var level = 0.0 - private(set) var isReceiving = false - - private static let log = Logger(subsystem: "com.joeblau.Stream", category: "mic-meter") - @ObservationIgnored private let channel = MicrophoneLevelChannel() - @ObservationIgnored private var task: Task? - @ObservationIgnored private var engine: AVAudioEngine? - @ObservationIgnored private let meterLevel = OSAllocatedUnfairLock(initialState: 0) - @ObservationIgnored private var gain = 1.0 - @ObservationIgnored private var preferredInputUID: String? - @ObservationIgnored private var nextRecorderAttemptAt: UInt64 = 0 - @ObservationIgnored private var lastBroadcastCheckAt: UInt64 = 0 - @ObservationIgnored private var broadcastIsLive = false - - /// Observer for `AVAudioEngineConfigurationChange`. The engine posts it when its - /// I/O format changes underneath the running graph — most importantly when a - /// Bluetooth HFP link finishes its asynchronous handoff after `setPreferredInput`. - /// The tap was installed at the pre-switch format, so without rebuilding it on - /// this notification the meter goes silent once the route lands on the BT mic. - @ObservationIgnored private nonisolated(unsafe) var configChangeObserver: NSObjectProtocol? - - func start() { - guard task == nil else { return } - task = Task { [weak self] in - while !Task.isCancelled { - do { - try await Task.sleep(nanoseconds: 50_000_000) - } catch { - return - } - guard let self else { return } - refreshBroadcastStateIfNeeded() - let target: Double - if let extensionLevel = channel.read() { - stopLocalCapture() - isReceiving = true - target = Double(extensionLevel) - } else if broadcastIsLive { - // Never open a second mic capture while ScreenCaptureKit's - // microphone is paused/off and not publishing meter samples. - stopLocalCapture() - isReceiving = false - target = 0 - } else if let localLevel = await readLocalLevel() { - isReceiving = true - target = localLevel - } else { - isReceiving = false - target = 0 - } - var next = target >= level - ? level * 0.3 + target * 0.7 - : max(target, level * 0.82) - if next < 0.005 { next = 0 } - // Only publish on change — @Observable fires on every set, so an - // idle meter otherwise re-renders the view 20x/s for nothing. - if next != level { level = next } - } - } - } - - func stop() { - task?.cancel() - task = nil - stopLocalCapture() - level = 0 - isReceiving = false - } - - func setGain(_ gain: Double) { - self.gain = max(0, min(gain, 2)) - } - - /// Routes the local meter to the user's selected input (incl. Bluetooth/DJI). - /// Without this the recorder captures the built-in mic, so a selected DJI/BT - /// mic never registers on the meter. - func setPreferredInput(_ uid: String?) { - guard uid != preferredInputUID else { return } - preferredInputUID = uid - restartLocalCapture() - } - - func restartLocalCapture() { - stopLocalCapture() - nextRecorderAttemptAt = 0 - } - - private func refreshBroadcastStateIfNeeded() { - let now = DispatchTime.now().uptimeNanoseconds - guard now &- lastBroadcastCheckAt >= 500_000_000 else { return } - lastBroadcastCheckAt = now - broadcastIsLive = BroadcastStateStore.isLive() - } - - private func readLocalLevel() async -> Double? { - if engine == nil { await startLocalCaptureIfAvailable() } - guard let engine, engine.isRunning else { return nil } - // RMS captured on the audio render thread by the input tap. - let rms = Double(meterLevel.withLock { $0 }) - let postGain = max(0.000_001, min(1, rms * gain)) - let decibels = 20 * log10(postGain) - return max(0, min(1, (decibels + 60) / 60)) - } - - private func startLocalCaptureIfAvailable() async { - guard AVAudioApplication.shared.recordPermission == .granted else { return } - let now = DispatchTime.now().uptimeNanoseconds - guard now >= nextRecorderAttemptAt else { return } - nextRecorderAttemptAt = now + 1_000_000_000 - - // Route the meter to the SAME input the user picked. Without activating a - // Bluetooth-capable record session and setting the preferred input, the - // recorder silently captures the built-in mic — so a selected DJI/BT mic - // never registers on the meter. (Only reached when NOT broadcasting, so it - // never competes with the extension's session.) - let session = AVAudioSession.sharedInstance() - guard await AudioInputProvider.activateBluetoothRecording(session) else { return } - if let uid = preferredInputUID, - let port = session.availableInputs?.first(where: { $0.uid == uid }) { - try? session.setPreferredInput(port) - } - - // Meter from the engine's input node — it reflects the ACTIVE input route - // (the DJI once it's the preferred input) and gives real PCM buffers, unlike - // the /dev/null AVAudioRecorder metering trick which can read nothing. - let engine = AVAudioEngine() - let input = engine.inputNode - let inputFormat = input.inputFormat(forBus: 0) - guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { return } - // A tap alone does NOT reliably pull a Bluetooth HFP input — the engine only - // renders its input when the graph drives an output, so an unconnected - // input node hands the tap silent buffers (the "DJI selected, meter flat" - // bug). Route input → main mixer to force a full I/O cycle, and mute the - // mixer output so nothing is monitored back to the speaker/HFP earpiece. - engine.connect(input, to: engine.mainMixerNode, format: inputFormat) - engine.mainMixerNode.outputVolume = 0 - guard installMeterTap(on: engine) else { return } - engine.prepare() - do { - try engine.start() - } catch { - engine.inputNode.removeTap(onBus: 0) - return - } - self.engine = engine - - // A Bluetooth HFP link settles asynchronously AFTER setPreferredInput, so the - // format above may still be the built-in mic's. Rebuild the tap on the new - // format when the engine reconfigures, else the BT meter reads flat forever. - observeConfigChanges(of: engine) - - let route = session.currentRoute.inputs - .map { "\($0.portName)/\($0.portType.rawValue)" } - .joined(separator: ",") - let format = engine.inputNode.inputFormat(forBus: 0) - Self.log.info("Mic meter started: route=[\(route, privacy: .public)] format=\(format.sampleRate, privacy: .public)Hz/\(format.channelCount, privacy: .public)ch pref=\(self.preferredInputUID ?? "nil", privacy: .public)") - } - - /// Reads the input node's CURRENT format and installs the metering tap. Split out - /// so it can be re-run on a configuration change (when the live route/format has - /// changed). Returns false when the route has no usable input yet. - private func installMeterTap(on engine: AVAudioEngine) -> Bool { - let input = engine.inputNode - let format = input.inputFormat(forBus: 0) - guard format.sampleRate > 0, format.channelCount > 0 else { return false } - let level = meterLevel - // @Sendable so the tap runs on the audio render thread — WITHOUT it the - // closure inherits this @MainActor class's isolation and iOS crashes with - // a libdispatch queue assertion when the render thread invokes it. - do { - try input.__installTap(onBus: 0, bufferSize: 1024, format: format, error: ()) { - @Sendable buffer, _ in - guard let channel = buffer.floatChannelData?[0] else { return } - let count = Int(buffer.frameLength) - guard count > 0 else { return } - var sumOfSquares: Float = 0 - for i in 0.. CVPixelBuffer? { + /// recent frame while it is fresh. Unlike a destructive take, this lets a + /// 30 fps camera feed a 60 fps screen stream without alternating PiP on/off. + func freshest() -> CVPixelBuffer? { + let now = DispatchTime.now().uptimeNanoseconds os_unfair_lock_lock(&lock) - let b = buffer - buffer = nil + let result: CVPixelBuffer? + if let buffer, now &- storedAt <= Self.maximumAgeNanoseconds { + result = buffer + } else { + buffer = nil + storedAt = 0 + result = nil + } os_unfair_lock_unlock(&lock) - return b + return result } /// Drops any stored frame (used during teardown). func clear() { os_unfair_lock_lock(&lock) buffer = nil + storedAt = 0 os_unfair_lock_unlock(&lock) } } diff --git a/StreamBroadcast/RTMPPublisher.swift b/StreamBroadcast/RTMPPublisher.swift index 44cf9f4..c885a9a 100644 --- a/StreamBroadcast/RTMPPublisher.swift +++ b/StreamBroadcast/RTMPPublisher.swift @@ -108,7 +108,8 @@ actor RTMPPublisher: Publisher { audioFourCcInfoMap: enhanced ? RTMPConnection.supportedAudioFourCcInfoMap : nil, capsEx: 0, requestTimeout: 5_000, - qualityOfService: .userInteractive) + // Continuous socket work must not outrank touch/animation delivery. + qualityOfService: .userInitiated) } private func startAudioConsumers() { @@ -935,7 +936,7 @@ extension VideoCodec { /// publishers. Assigning it to `VideoCodecSettings.profileLevel` also flips the /// encoder's internal `format` to HEVC (its `didSet` keys off the "HEVC" /// substring), so the encoded bitstream, the RTMP `hvc1` exHeader framing, and - /// the SRT/WHIP payload all follow from this one property. + /// the transport payload all follow from this one property. var videoToolboxProfileLevel: String { switch self { case .h264: return kVTProfileLevel_H264_Main_AutoLevel as String diff --git a/StreamBroadcast/SessionPublisher.swift b/StreamBroadcast/SessionPublisher.swift index 1c283c8..b97918d 100644 --- a/StreamBroadcast/SessionPublisher.swift +++ b/StreamBroadcast/SessionPublisher.swift @@ -261,7 +261,7 @@ actor SessionPublisher: Publisher { // Re-apply the locked encoder size to the fresh stream (across reconnects). if let outputSize { - try? await stream.setVideoSettings( + try await stream.setVideoSettings( await makeVideoSettings(await stream.videoSettings, size: outputSize) ) } @@ -641,10 +641,10 @@ actor SessionPublisher: Publisher { v.frameInterval = await networkController.currentFrameInterval() v.maxKeyFrameIntervalDuration = 2 v.bitRateMode = .average - // HEVC when the user opted in — SRT (MPEG-TS) and WHIP both carry it — else - // H.264 Main. The profileLevel string also switches the encoder's codec + // HEVC when the user opted in and the transport can packetize it (SRT); + // WHIP is currently H.264-only. The profileLevel string also switches the encoder's codec // (VideoCodecSettings flips to HEVC on the "HEVC" substring). - v.profileLevel = settings.videoCodec.videoToolboxProfileLevel + v.profileLevel = settings.effectiveVideoCodec.videoToolboxProfileLevel // Low-latency VideoToolbox rate control: tightens encoder queuing latency // (makeEncoderSpecification enables EnableLowLatencyRateControl). v.isLowLatencyRateControlEnabled = true diff --git a/StreamCore/SettingsStore.swift b/StreamCore/SettingsStore.swift index 15434e1..94fec75 100644 --- a/StreamCore/SettingsStore.swift +++ b/StreamCore/SettingsStore.swift @@ -60,8 +60,19 @@ public struct SettingsStore: Sendable { /// settings to the container file with the secrets blanked, so they are never /// persisted in plaintext. public func save(_ settings: StreamSettings) { + saveConnection(settings) + saveNonSecret(settings) + } + + /// Writes only the selected protocol's sensitive fields. UI persistence can + /// skip this Security.framework round-trip for video/audio/PiP-only edits. + public func saveConnection(_ settings: StreamSettings) { keychain.set(settings.rtmpURL, for: .url(settings.selectedProtocol)) keychain.set(settings.streamKey, for: .key(settings.selectedProtocol)) + } + + /// Writes only the redacted JSON settings snapshot. + public func saveNonSecret(_ settings: StreamSettings) { persistNonSecret(settings) } @@ -72,6 +83,9 @@ public struct SettingsStore: Sendable { keychain.set(settings.rtmpURL, for: .url(settings.selectedProtocol)) keychain.set(settings.streamKey, for: .key(settings.selectedProtocol)) settings.selectedProtocol = newProtocol + if !newProtocol.supports(settings.videoCodec) { + settings.videoCodec = .h264 + } settings.rtmpURL = keychain.string(for: .url(newProtocol)) ?? "" settings.streamKey = keychain.string(for: .key(newProtocol)) ?? "" persistNonSecret(settings) diff --git a/StreamCore/StreamSettings.swift b/StreamCore/StreamSettings.swift index 10804fb..e7bba14 100644 --- a/StreamCore/StreamSettings.swift +++ b/StreamCore/StreamSettings.swift @@ -38,8 +38,9 @@ public enum BackupQuality: String, Codable, CaseIterable, Sendable { /// The video codec the encoder targets. HEVC (H.265) yields roughly a 40% /// quality-per-bit gain over H.264 on text-heavy screen content in the 2–8 Mbps -/// band, at the cost of ingest compatibility: it rides SRT (MPEG-TS), WHIP, and -/// *enhanced*-RTMP (E-RTMP `hvc1` negotiation) but NOT traditional RTMP ingests +/// band, at the cost of ingest compatibility: it rides SRT (MPEG-TS) and +/// *enhanced*-RTMP (E-RTMP `hvc1` negotiation) but NOT WHIP in the current RTC +/// transport or traditional RTMP ingests /// such as Restream, which speak H.264 only. H.264 Main is therefore the safe, /// universally-decodable default; HEVC is an opt-in the user validates against /// their real endpoint (see the encoder wiring in RTMPPublisher/SessionPublisher). @@ -111,6 +112,20 @@ public enum StreamProtocol: String, Codable, CaseIterable, Sendable { /// StreamSession publisher. WHIP (WebRTC) is experimental — validate memory on /// device, as libdatachannel adds meaningful memory and CPU overhead. public var isPublishingSupported: Bool { true } + + /// Codecs the bundled transport can actually packetize. RTCHaishinKit's WHIP + /// stream currently has an H.264 RTP packetizer only; advertising HEVC there + /// silently left the encoder at a default/fallback configuration. + public var supportedVideoCodecs: [VideoCodec] { + switch self { + case .whip: return [.h264] + case .rtmp, .rtmps, .srt: return VideoCodec.allCases + } + } + + public func supports(_ codec: VideoCodec) -> Bool { + supportedVideoCodecs.contains(codec) + } } // MARK: - StreamSettings (the ONLY shared persisted model) @@ -126,7 +141,7 @@ public struct StreamSettings: Codable, Equatable, Sendable { public var videoBitrate: Int // bits per second public var audioBitrate: Int // bits per second public var frameRate: Int // fps hint - /// The encoder's target video codec. HEVC is honored on SRT/WHIP and + /// The encoder's target video codec. HEVC is honored on SRT and /// enhanced-RTMP; traditional RTMP ingests fall back to H.264 (see `VideoCodec`). public var videoCodec: VideoCodec public var pipEnabled: Bool @@ -217,6 +232,12 @@ public struct StreamSettings: Codable, Equatable, Sendable { public static let `default` = StreamSettings() + /// Defense-in-depth for restored/legacy settings: never hand a codec to a + /// transport that cannot packetize it, even if the UI has not normalized yet. + public var effectiveVideoCodec: VideoCodec { + selectedProtocol.supports(videoCodec) ? videoCodec : .h264 + } + /// True only when the selected protocol can currently publish AND its URL /// (plus its key, where the protocol requires one) are present and valid. public var isPublishable: Bool { diff --git a/StreamCoreTests/StreamProtocolTests.swift b/StreamCoreTests/StreamProtocolTests.swift index b49da71..18ce96c 100644 --- a/StreamCoreTests/StreamProtocolTests.swift +++ b/StreamCoreTests/StreamProtocolTests.swift @@ -44,6 +44,16 @@ import StreamCore } } + @Test("WHIP is H.264-only while the other transports expose both codecs") + func codecCompatibility() { + #expect(StreamProtocol.whip.supportedVideoCodecs == [.h264]) + #expect(!StreamProtocol.whip.supports(.hevc)) + for proto in [StreamProtocol.rtmp, .rtmps, .srt] { + #expect(proto.supportedVideoCodecs == [.h264, .hevc]) + #expect(proto.supports(.hevc)) + } + } + @Test("Placeholders and key-field labels are always populated for the UI") func uiStringsNonEmpty() { for proto in StreamProtocol.allCases { diff --git a/StreamCoreTests/VideoCodecTests.swift b/StreamCoreTests/VideoCodecTests.swift index 9112e8d..474791f 100644 --- a/StreamCoreTests/VideoCodecTests.swift +++ b/StreamCoreTests/VideoCodecTests.swift @@ -34,6 +34,16 @@ import StreamCore #expect(StreamSettings.default.videoCodec == .h264) } + @Test("Effective codec safely falls back for an incompatible transport") + func effectiveCodecHonorsTransport() { + var settings = StreamSettings.default + settings.videoCodec = .hevc + settings.selectedProtocol = .srt + #expect(settings.effectiveVideoCodec == .hevc) + settings.selectedProtocol = .whip + #expect(settings.effectiveVideoCodec == .h264) + } + @Test("Each codec round-trips through Codable") func codableRoundTrip() throws { for codec in VideoCodec.allCases {