diff --git a/.github/screenshots/idle-model-unload-setting.png b/.github/screenshots/idle-model-unload-setting.png new file mode 100644 index 000000000..042ba3334 Binary files /dev/null and b/.github/screenshots/idle-model-unload-setting.png differ diff --git a/.github/screenshots/notch-model-loading-bar.png b/.github/screenshots/notch-model-loading-bar.png new file mode 100644 index 000000000..5613bd008 Binary files /dev/null and b/.github/screenshots/notch-model-loading-bar.png differ diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 2845e9201..85e163934 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -3558,7 +3558,6 @@ struct ContentView: View { TranscriptionSoundPlayer.shared.playStartSound() } self.captureRecordingContext() - self.prewarmPrivateAIDictationIfNeeded(for: .primary) DebugLogger.shared.benchmark( "APP_BENCH", message: "overlay_phase phase=recording trigger=first_pcm", @@ -3570,12 +3569,15 @@ struct ContentView: View { } } - // Pre-load model in background while recording (avoids 10s freeze on stop) + // Pre-load model in background while recording (avoids 10s freeze on stop). + // The AI runtime is warmed only after the speech model is ready so the two + // loads don't compete for disk and GPU; both finish long before stop. Task { do { DebugLogger.shared.debug("ContentView: pre-load model task started", source: "ContentView") try await self.asr.ensureAsrReady() DebugLogger.shared.debug("Model pre-loaded during recording", source: "ContentView") + self.prewarmPrivateAIDictationIfNeeded(for: .primary) } catch { DebugLogger.shared.error("Failed to pre-load model: \(error)", source: "ContentView") } @@ -4228,7 +4230,12 @@ extension ContentView { TranscriptionSoundPlayer.shared.playStartSound() } self.captureRecordingContext() - self.prewarmPrivateAIDictationIfNeeded(for: slot) + // Speech model first, AI runtime second, so the two loads don't + // compete. ensureAsrReady() joins the load start() already kicked off. + Task { + _ = try? await self.asr.ensureAsrReady() + self.prewarmPrivateAIDictationIfNeeded(for: slot) + } self.appBench("overlay_phase phase=recording trigger=first_pcm") }) if startOutcome == .failed { diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 5c8b4fedb..8dc031081 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -20,6 +20,9 @@ final class SettingsStore: ObservableObject { static let defaultTranscriptionPreviewCharLimit = 150 static let privateAIContextTokenLimitRange: ClosedRange = 2048...8192 static let privateAIContextTokenLimitStep = 512 + /// Minutes of inactivity before on-device models are released. 0 = never. + static let modelIdleUnloadMinuteOptions = [0, 2, 5, 10, 30] + static let defaultModelIdleUnloadMinutes = 10 static let defaultPrivateAIContextTokenLimit = 4096 static let privateAIDictationSystemOverheadTokens = 1280 static let privateAIDictationMinimumOutputTokens = 256 @@ -1663,6 +1666,19 @@ final class SettingsStore: ObservableObject { } } + var modelIdleUnloadMinutes: Int { + get { + guard self.defaults.object(forKey: Keys.modelIdleUnloadMinutes) != nil else { + return Self.defaultModelIdleUnloadMinutes + } + return max(0, self.defaults.integer(forKey: Keys.modelIdleUnloadMinutes)) + } + set { + objectWillChange.send() + self.defaults.set(max(0, newValue), forKey: Keys.modelIdleUnloadMinutes) + } + } + private func migratePrivateAIContextDefaultTo4KIfNeeded() { guard self.defaults.bool(forKey: Keys.privateAIContextDefaultMigratedTo4K) == false else { return } let storedValue = self.defaults.object(forKey: Keys.privateAIContextTokenLimit) as? Int @@ -5324,6 +5340,7 @@ private extension SettingsStore { static let privateAIBoostEnabled = "PrivateAIProviderBoostEnabled" static let privateAIBackendPreference = SettingsStore.privateAIBackendPreferenceDefaultsKey static let privateAIContextTokenLimit = "PrivateAIProviderContextTokenLimit" + static let modelIdleUnloadMinutes = "ModelIdleUnloadMinutes" static let privateAIContextDefaultMigratedTo4K = "PrivateAIProviderContextDefaultMigratedTo4K" static let providerAPIKeys = "ProviderAPIKeys" static let providerAPIKeyIdentifiers = "ProviderAPIKeyIdentifiers" diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 7dc78252e..939b3d785 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -188,6 +188,8 @@ final class ASRService: ObservableObject { @Published private(set) var isCancellingModelPreparation: Bool = false @Published var modelsExistOnDisk: Bool = false @Published var downloadProgress: Double? = nil + /// Seconds the last cached-model load took. The notch loading bar paces itself on this. + static let lastModelLoadSecondsKey = "LastSpeechModelLoadSeconds" @Published var modelPreparationPhase: ModelPreparationPhase? = nil @Published var downloadingModelId: String? = nil // Tracks which model is currently being downloaded @Published private(set) var isCancellingModelDownload: Bool = false @@ -603,6 +605,14 @@ final class ASRService: ObservableObject { } } + /// Release the loaded speech model to reclaim memory while idle. The next + /// `ensureAsrReady` reloads it; recording itself never waits on the model. + func unloadForIdle() { + guard self.isAsrReady, !self.isRunningOrStarting else { return } + DebugLogger.shared.info("ASRService: unloading speech model after idle", source: "ASRService") + self.resetTranscriptionProvider() + } + /// Call this when the transcription provider setting changes to reset state func resetTranscriptionProvider() { let newModel = SettingsStore.shared.selectedSpeechModel @@ -1727,6 +1737,7 @@ final class ASRService: ObservableObject { DebugLogger.shared.warning("⚠️ START() blocked - already running (started: \(self.isRunning), starting: \(self.isStarting))", source: "ASRService") return .alreadyActive } + IdleModelUnloader.shared.cancel() guard self.isTerminating == false else { DebugLogger.shared.warning("START() blocked - app is terminating", source: "ASRService") return .failed @@ -2019,6 +2030,20 @@ final class ASRService: ObservableObject { DebugLogger.shared.debug("ℹ️ No device to monitor", source: "ASRService") } + // Load the model in the background while audio buffers. Without this the + // hotkey path only loads at stop(), so the user waits after releasing. + // ensureAsrReady() de-duplicates against any load already in flight. + if !self.isAsrReady, !forDictionaryTraining { + DebugLogger.shared.info("Model not ready at start; loading in background while recording", source: "ASRService") + Task { [weak self] in + do { + try await self?.ensureAsrReady() + } catch { + DebugLogger.shared.error("Background model load failed: \(error)", source: "ASRService") + } + } + } + // Only start streaming for models that support it (large Whisper models are too slow) let model = SettingsStore.shared.selectedSpeechModel if model.supportsStreaming, !forDictionaryTraining { @@ -2346,6 +2371,7 @@ final class ASRService: ObservableObject { defer { self.applyPendingParakeetVocabularyReloadIfNeeded() self.isDictionaryTrainingCaptureActive = false + IdleModelUnloader.shared.recordActivity() } await self.cancelAudioRouteRecoveryAndWait() @@ -2773,6 +2799,7 @@ final class ASRService: ObservableObject { defer { self.applyPendingParakeetVocabularyReloadIfNeeded() self.isDictionaryTrainingCaptureActive = false + IdleModelUnloader.shared.recordActivity() } await self.cancelAudioRouteRecoveryAndWait() @@ -4339,6 +4366,9 @@ final class ASRService: ObservableObject { guard self.ensureReadyOperationID == operationID else { throw CancellationError() } let downloadDuration = Date().timeIntervalSince(downloadStartTime) DebugLogger.shared.info("✓ Provider preparation completed in \(String(format: "%.1f", downloadDuration)) seconds", source: "ASRService") + if modelsAlreadyCached { + UserDefaults.standard.set(downloadDuration, forKey: Self.lastModelLoadSecondsKey) + } self.isDownloadingModel = false // Keep isLoadingModel true until first transcription completes (for large models that need warm-up) @@ -4360,6 +4390,16 @@ final class ASRService: ObservableObject { self.isAsrReady = true self.isCancellingModelPreparation = false self.refreshWordBoostStatus() + // start() skips the preview loop when the model isn't ready yet. If the + // load finished while audio was buffering, start it now; the first chunk + // transcribes the whole buffered prefix, then continues incrementally. + if self.isRunning, + !self.isDictionaryTrainingCaptureActive, + SettingsStore.shared.selectedSpeechModel.supportsStreaming + { + DebugLogger.shared.info("Model became ready mid-recording; starting streaming preview", source: "ASRService") + self.startStreamingTranscription() + } self.finishModelDownloadAnalytics(operationID: operationID, outcome: .succeeded) } catch is CancellationError { self.finishModelDownloadAnalytics(operationID: operationID, outcome: .cancelled) diff --git a/Sources/Fluid/Services/IdleModelUnloader.swift b/Sources/Fluid/Services/IdleModelUnloader.swift new file mode 100644 index 000000000..be17af095 --- /dev/null +++ b/Sources/Fluid/Services/IdleModelUnloader.swift @@ -0,0 +1,61 @@ +import Foundation + +/// Frees on-device models after a period without dictation so the app stops +/// pinning gigabytes while idle. Recording never waits on this: the models +/// reload in the background on the next hotkey press. +@MainActor +final class IdleModelUnloader { + static let shared = IdleModelUnloader() + + private var countdown: Task? + + private init() {} + + // MARK: - Public + + /// Restart the idle countdown. Call whenever a model was just used. + func recordActivity() { + let minutes = SettingsStore.shared.modelIdleUnloadMinutes + guard minutes > 0 else { + self.cancel() + return + } + self.schedule(after: .seconds(minutes * 60)) + } + + /// Release both models immediately (settings button). Same busy guard as the timer. + func unloadNow() async { + await self.fire() + } + + /// Stop the countdown while a session is active. + func cancel() { + self.countdown?.cancel() + self.countdown = nil + } + + // MARK: - Private + + private func schedule(after delay: Duration) { + self.countdown?.cancel() + self.countdown = Task { [weak self] in + guard (try? await Task.sleep(for: delay)) != nil else { return } + await self?.fire() + } + } + + private func fire() async { + let asr = AppServices.shared.asr + let overlay = NotchContentState.shared + if asr.isRunningOrStarting || overlay.isProcessing || overlay.isCommandProcessing { + // Busy windows last seconds, not minutes, so a fixed short retry is enough. + DebugLogger.shared.debug("Idle unload deferred: session active", source: "IdleModelUnloader") + self.schedule(after: .seconds(60)) + return + } + DebugLogger.shared.info("Idle unload: releasing models", source: "IdleModelUnloader") + self.countdown = nil + await PrivateAIIntegrationService.shared.unloadCachedRuntime(reason: "idle") + asr.unloadForIdle() + } +} diff --git a/Sources/Fluid/Services/PrivateAIIntegrationService.swift b/Sources/Fluid/Services/PrivateAIIntegrationService.swift index f1797bbab..7617c1ab6 100644 --- a/Sources/Fluid/Services/PrivateAIIntegrationService.swift +++ b/Sources/Fluid/Services/PrivateAIIntegrationService.swift @@ -208,6 +208,7 @@ actor PrivateAIIntegrationService { context: AppContext ) async throws -> EnhancementResult { try Self.validateDictationHeadroom(inputText, contextTokenLimit: runtime.contextTokenLimit) + defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } } return try await Self.provider.enhanceDictation(inputText, runtime: runtime, context: context) } @@ -218,6 +219,7 @@ actor PrivateAIIntegrationService { streamHandler: PrivateAIStreamHandler? ) async throws -> EnhancementResult { try Self.validateDictationHeadroom(inputText, contextTokenLimit: runtime.contextTokenLimit) + defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } } return try await Self.provider.enhanceDictation( inputText, runtime: runtime, @@ -242,7 +244,8 @@ actor PrivateAIIntegrationService { runtime: RuntimeConfiguration, context: AppContext ) async throws -> EnhancementResult { - try await Self.provider.rewrite( + defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } } + return try await Self.provider.rewrite( inputText, systemPrompt: systemPrompt, runtime: runtime, diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift index c55b5d2cf..4a7618479 100644 --- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift +++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift @@ -153,6 +153,10 @@ extension VoiceEngineSettingsView { // Filler Words Section self.fillerWordsSection + + Divider().padding(.vertical, 4) + + self.modelIdleUnloadSection } } } @@ -856,6 +860,35 @@ extension VoiceEngineSettingsView { } } + var modelIdleUnloadSection: some View { + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 2) { + Text("Unload Models When Idle") + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.voiceEngineTitleText) + Text("Frees memory after a break. No need to wait on your next dictation: just start talking and the models reload while you speak.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.voiceEngineSecondaryText) + } + Spacer() + Picker("", selection: self.$settings.modelIdleUnloadMinutes) { + ForEach(SettingsStore.modelIdleUnloadMinuteOptions, id: \.self) { minutes in + Text(minutes == 0 ? "Never" : "\(minutes) min").tag(minutes) + } + } + .labelsHidden() + .pickerStyle(.menu) + .frame(width: 96) + .onChange(of: self.settings.modelIdleUnloadMinutes) { _, _ in + IdleModelUnloader.shared.recordActivity() + } + Button("Unload Now") { + Task { await IdleModelUnloader.shared.unloadNow() } + } + .help("Release the models right away. The next dictation reloads them while you speak.") + } + } + // MARK: - Speech Model Logo View private func speechModelLogoView(for model: SettingsStore.SpeechModel) -> some View { diff --git a/Sources/Fluid/Views/NotchContentViews.swift b/Sources/Fluid/Views/NotchContentViews.swift index 101846872..46a736110 100644 --- a/Sources/Fluid/Views/NotchContentViews.swift +++ b/Sources/Fluid/Views/NotchContentViews.swift @@ -492,6 +492,114 @@ final class CompositorShimmerSweepView: NSView { } } +// MARK: - Model Loading Indicator + +/// Shown under the waveform while the speech model loads or downloads mid-recording. +/// Audio keeps buffering underneath; this only explains why the preview is late. +/// The sweep runs on the compositor (same as ShimmerText), so it costs no timers. +struct NotchModelLoadingView: View { + private enum Phase: Equatable { + case downloading(Double?) + case loading + case warmingUp + } + + private let phase: Phase + let color: Color + /// Tighter metrics for the one-line strip under the standard notch. + let compact: Bool + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @State private var fill: CGFloat = 0 + + init(asr: ASRService, color: Color, compact: Bool = false) { + if asr.isDownloadingModel { + self.phase = .downloading(asr.downloadProgress) + } else if asr.isAsrReady { + self.phase = .warmingUp + } else { + self.phase = .loading + } + self.color = color + self.compact = compact + } + + private var label: String { + switch self.phase { + case let .downloading(progress?): + return "Downloading model · \(Int((progress * 100).rounded()))%" + case .downloading(nil): + return "Downloading model…" + case .loading: + return "Keep talking · loading model" + case .warmingUp: + return "Keep talking · transcribing" + } + } + + private var trackHeight: CGFloat { self.compact ? 2 : 3 } + private var font: Font { .system(size: self.compact ? 9 : 10, weight: .medium) } + + /// Loads report no progress, so the bar eases toward 85% over the last measured + /// load time and completes when the model reports ready. Same trick as Safari's + /// page-load bar: it feels determinate without pretending to know the future. + private var expectedLoadSeconds: Double { + let stored = UserDefaults.standard.double(forKey: ASRService.lastModelLoadSecondsKey) + return min(max(stored == 0 ? 1.5 : stored, 0.6), 15) + } + + private func motion(_ base: Animation) -> Animation? { + self.reduceMotion ? nil : base + } + + private func advance(to phase: Phase) { + switch phase { + case let .downloading(progress): + withAnimation(self.motion(.spring(response: 0.4, dampingFraction: 0.85))) { + self.fill = CGFloat(min(max(progress ?? 0, 0), 1)) + } + case .loading: + withAnimation(self.motion(.easeOut(duration: self.expectedLoadSeconds))) { + self.fill = 0.85 + } + case .warmingUp: + withAnimation(self.motion(.easeOut(duration: 0.35))) { + self.fill = 1 + } + } + } + + var body: some View { + VStack(alignment: .leading, spacing: self.compact ? 3 : 5) { + Group { + if self.reduceMotion { + Text(self.label) + .font(self.font) + .foregroundStyle(.white.opacity(0.75)) + } else { + ShimmerText(text: self.label, color: .white, font: self.font) + } + } + .lineLimit(1) + .id(self.label) + .transition(.opacity) + .animation(self.motion(.easeInOut(duration: 0.2)), value: self.label) + + GeometryReader { geometry in + ZStack(alignment: .leading) { + Capsule() + .fill(Color.white.opacity(0.12)) + Capsule() + .fill(self.color) + .frame(width: max(self.trackHeight, geometry.size.width * self.fill)) + } + } + .frame(height: self.trackHeight) + } + .onAppear { self.advance(to: self.phase) } + .onChange(of: self.phase) { _, newPhase in self.advance(to: newPhase) } + } +} + // MARK: - Expanded View (Main Content) - Minimal Design struct NotchExpandedView: View { @@ -499,6 +607,7 @@ struct NotchExpandedView: View { @ObservedObject private var contentState = NotchContentState.shared @ObservedObject private var settings = SettingsStore.shared @ObservedObject private var activeAppMonitor = ActiveAppMonitor.shared + @ObservedObject private var asr = AppServices.shared.asr @Environment(\.theme) private var theme @State private var isHoveringPromptChip = false @State private var isHoveringPromptMenu = false @@ -552,6 +661,16 @@ struct NotchExpandedView: View { !self.visiblePreviewText.isEmpty } + /// Speech model is loading or downloading while audio is already buffering. + /// `isLoadingModel` stays true until the first chunk lands, so this hands off + /// straight to the preview text. + private var isModelLoading: Bool { + self.contentState.mode == .dictation && + !self.contentState.isProcessing && + !self.hasTranscription && + (self.asr.isLoadingModel || self.asr.isDownloadingModel) + } + private var visiblePreviewText: String { let previewText = self.contentState.cachedPreviewText.trimmingCharacters(in: .whitespacesAndNewlines) guard !Self.transientOverlayStatusTexts.contains(previewText) else { return "" } @@ -987,6 +1106,7 @@ struct NotchExpandedView: View { } } .animation(.spring(response: 0.25, dampingFraction: 0.8), value: self.hasTranscription) + .animation(.spring(response: 0.25, dampingFraction: 0.8), value: self.isModelLoading) .animation(.easeInOut(duration: 0.2), value: self.contentState.mode) .animation(.easeInOut(duration: 0.25), value: self.contentState.isProcessing) } @@ -1061,6 +1181,11 @@ struct NotchExpandedView: View { .foregroundStyle(.white.opacity(0.9)) .frame(width: self.previewMaxWidth, alignment: .leading) .transition(.opacity.combined(with: .scale(scale: 0.95))) + } else if self.isModelLoading { + NotchModelLoadingView(asr: self.asr, color: self.modeColor) + .frame(width: self.previewMaxWidth, alignment: .leading) + .frame(maxWidth: .infinity, alignment: .center) + .transition(.opacity.combined(with: .scale(scale: 0.95))) } else if self.presentationPolicy.showsStreamingPreview && self.hasTranscription && !self.contentState.isProcessing { let previewText = self.visiblePreviewText if !previewText.isEmpty { @@ -1303,6 +1428,7 @@ struct NotchCompactTrailingView: View { struct NotchCompactBottomView: View { @ObservedObject private var contentState = NotchContentState.shared + @ObservedObject private var asr = AppServices.shared.asr private let previewWidth: CGFloat = 250 private let previewHeight: CGFloat = 20 @@ -1326,8 +1452,16 @@ struct NotchCompactBottomView: View { return trimmed } + private var isModelLoading: Bool { + SettingsStore.shared.enableStreamingPreview && + self.contentState.mode == .dictation && + !self.contentState.isProcessing && + self.compactPreviewText.isEmpty && + (self.asr.isLoadingModel || self.asr.isDownloadingModel) + } + private var shouldShowPreview: Bool { - SettingsStore.shared.enableStreamingPreview && !self.compactPreviewText.isEmpty + SettingsStore.shared.enableStreamingPreview && !self.compactPreviewText.isEmpty && !self.isModelLoading } var body: some View { @@ -1339,12 +1473,19 @@ struct NotchCompactBottomView: View { .truncationMode(.head) .offset(y: self.shouldShowPreview ? 0 : -4) .opacity(self.shouldShowPreview ? 1 : 0) + + if self.isModelLoading { + NotchModelLoadingView(asr: self.asr, color: self.contentState.mode.notchColor, compact: true) + .frame(width: self.previewWidth, alignment: .leading) + .transition(.opacity.combined(with: .offset(y: -4))) + } } .frame(width: self.previewWidth, height: SettingsStore.shared.enableStreamingPreview ? self.previewHeight : 0, alignment: .leading) .padding(.horizontal, SettingsStore.shared.enableStreamingPreview ? 10 : 0) .padding(.bottom, SettingsStore.shared.enableStreamingPreview ? 8 : 0) .clipped() .animation(.easeOut(duration: 0.2), value: self.shouldShowPreview) + .animation(.easeOut(duration: 0.2), value: self.isModelLoading) } }