feat: unload idle models and show load progress in the notch overlay

Sources/Fluid/ContentView.swift

@@ -3558,7 +3558,6 @@ struct ContentView: View {
                     TranscriptionSoundPlayer.shared.playStartSound()
                 }
                 self.captureRecordingContext()
-                self.prewarmPrivateAIDictationIfNeeded(for: .primary)
                 DebugLogger.shared.benchmark(
                     "APP_BENCH",
                     message: "overlay_phase phase=recording trigger=first_pcm",
@@ -3570,12 +3569,15 @@ struct ContentView: View {
             }
         }
 
-        // Pre-load model in background while recording (avoids 10s freeze on stop)
+        // Pre-load model in background while recording (avoids 10s freeze on stop).
+        // The AI runtime is warmed only after the speech model is ready so the two
+        // loads don't compete for disk and GPU; both finish long before stop.
         Task {
             do {
                 DebugLogger.shared.debug("ContentView: pre-load model task started", source: "ContentView")
                 try await self.asr.ensureAsrReady()
                 DebugLogger.shared.debug("Model pre-loaded during recording", source: "ContentView")
+                self.prewarmPrivateAIDictationIfNeeded(for: .primary)
             } catch {
                 DebugLogger.shared.error("Failed to pre-load model: \(error)", source: "ContentView")
             }
@@ -4228,7 +4230,12 @@ extension ContentView {
                     TranscriptionSoundPlayer.shared.playStartSound()
                 }
                 self.captureRecordingContext()
-                self.prewarmPrivateAIDictationIfNeeded(for: slot)
+                // Speech model first, AI runtime second, so the two loads don't
+                // compete. ensureAsrReady() joins the load start() already kicked off.
+                Task {
+                    _ = try? await self.asr.ensureAsrReady()
+                    self.prewarmPrivateAIDictationIfNeeded(for: slot)
+                }
                 self.appBench("overlay_phase phase=recording trigger=first_pcm")
             })
             if startOutcome == .failed {

Sources/Fluid/Persistence/SettingsStore.swift

@@ -20,6 +20,9 @@ final class SettingsStore: ObservableObject {
     static let defaultTranscriptionPreviewCharLimit = 150
     static let privateAIContextTokenLimitRange: ClosedRange<Int> = 2048...8192
     static let privateAIContextTokenLimitStep = 512
+    /// Minutes of inactivity before on-device models are released. 0 = never.
+    static let modelIdleUnloadMinuteOptions = [0, 2, 5, 10, 30]
+    static let defaultModelIdleUnloadMinutes = 10
     static let defaultPrivateAIContextTokenLimit = 4096
     static let privateAIDictationSystemOverheadTokens = 1280
     static let privateAIDictationMinimumOutputTokens = 256
@@ -1663,6 +1666,19 @@ final class SettingsStore: ObservableObject {
         }
     }
 
+    var modelIdleUnloadMinutes: Int {
+        get {
+            guard self.defaults.object(forKey: Keys.modelIdleUnloadMinutes) != nil else {
+                return Self.defaultModelIdleUnloadMinutes
+            }
+            return max(0, self.defaults.integer(forKey: Keys.modelIdleUnloadMinutes))
+        }
+        set {
+            objectWillChange.send()
+            self.defaults.set(max(0, newValue), forKey: Keys.modelIdleUnloadMinutes)
+        }
+    }
+
     private func migratePrivateAIContextDefaultTo4KIfNeeded() {
         guard self.defaults.bool(forKey: Keys.privateAIContextDefaultMigratedTo4K) == false else { return }
         let storedValue = self.defaults.object(forKey: Keys.privateAIContextTokenLimit) as? Int
@@ -5324,6 +5340,7 @@ private extension SettingsStore {
         static let privateAIBoostEnabled = "PrivateAIProviderBoostEnabled"
         static let privateAIBackendPreference = SettingsStore.privateAIBackendPreferenceDefaultsKey
         static let privateAIContextTokenLimit = "PrivateAIProviderContextTokenLimit"
+        static let modelIdleUnloadMinutes = "ModelIdleUnloadMinutes"
         static let privateAIContextDefaultMigratedTo4K = "PrivateAIProviderContextDefaultMigratedTo4K"
         static let providerAPIKeys = "ProviderAPIKeys"
         static let providerAPIKeyIdentifiers = "ProviderAPIKeyIdentifiers"

Sources/Fluid/Services/ASRService.swift

@@ -188,6 +188,8 @@ final class ASRService: ObservableObject {
     @Published private(set) var isCancellingModelPreparation: Bool = false
     @Published var modelsExistOnDisk: Bool = false
     @Published var downloadProgress: Double? = nil
+    /// Seconds the last cached-model load took. The notch loading bar paces itself on this.
+    static let lastModelLoadSecondsKey = "LastSpeechModelLoadSeconds"
     @Published var modelPreparationPhase: ModelPreparationPhase? = nil
     @Published var downloadingModelId: String? = nil // Tracks which model is currently being downloaded
     @Published private(set) var isCancellingModelDownload: Bool = false
@@ -603,6 +605,14 @@ final class ASRService: ObservableObject {
         }
     }
 
+    /// Release the loaded speech model to reclaim memory while idle. The next
+    /// `ensureAsrReady` reloads it; recording itself never waits on the model.
+    func unloadForIdle() {
+        guard self.isAsrReady, !self.isRunningOrStarting else { return }
+        DebugLogger.shared.info("ASRService: unloading speech model after idle", source: "ASRService")
+        self.resetTranscriptionProvider()
+    }
+
     /// Call this when the transcription provider setting changes to reset state
     func resetTranscriptionProvider() {
         let newModel = SettingsStore.shared.selectedSpeechModel
@@ -1727,6 +1737,7 @@ final class ASRService: ObservableObject {
             DebugLogger.shared.warning("⚠️ START() blocked - already running (started: \(self.isRunning), starting: \(self.isStarting))", source: "ASRService")
             return .alreadyActive
         }
+        IdleModelUnloader.shared.cancel()
         guard self.isTerminating == false else {
             DebugLogger.shared.warning("START() blocked - app is terminating", source: "ASRService")
             return .failed
@@ -2019,6 +2030,20 @@ final class ASRService: ObservableObject {
                 DebugLogger.shared.debug("ℹ️ No device to monitor", source: "ASRService")
             }
 
+            // Load the model in the background while audio buffers. Without this the
+            // hotkey path only loads at stop(), so the user waits after releasing.
+            // ensureAsrReady() de-duplicates against any load already in flight.
+            if !self.isAsrReady, !forDictionaryTraining {
+                DebugLogger.shared.info("Model not ready at start; loading in background while recording", source: "ASRService")
+                Task { [weak self] in
+                    do {
+                        try await self?.ensureAsrReady()
+                    } catch {
+                        DebugLogger.shared.error("Background model load failed: \(error)", source: "ASRService")
+                    }
+                }
+            }
+
             // Only start streaming for models that support it (large Whisper models are too slow)
             let model = SettingsStore.shared.selectedSpeechModel
             if model.supportsStreaming, !forDictionaryTraining {
@@ -2346,6 +2371,7 @@ final class ASRService: ObservableObject {
         defer {
             self.applyPendingParakeetVocabularyReloadIfNeeded()
             self.isDictionaryTrainingCaptureActive = false
+            IdleModelUnloader.shared.recordActivity()
         }
 
         await self.cancelAudioRouteRecoveryAndWait()
@@ -2773,6 +2799,7 @@ final class ASRService: ObservableObject {
         defer {
             self.applyPendingParakeetVocabularyReloadIfNeeded()
             self.isDictionaryTrainingCaptureActive = false
+            IdleModelUnloader.shared.recordActivity()
         }
 
         await self.cancelAudioRouteRecoveryAndWait()
@@ -4339,6 +4366,9 @@ final class ASRService: ObservableObject {
             guard self.ensureReadyOperationID == operationID else { throw CancellationError() }
             let downloadDuration = Date().timeIntervalSince(downloadStartTime)
             DebugLogger.shared.info("✓ Provider preparation completed in \(String(format: "%.1f", downloadDuration)) seconds", source: "ASRService")
+            if modelsAlreadyCached {
+                UserDefaults.standard.set(downloadDuration, forKey: Self.lastModelLoadSecondsKey)
+            }
 
             self.isDownloadingModel = false
             // Keep isLoadingModel true until first transcription completes (for large models that need warm-up)
@@ -4360,6 +4390,16 @@ final class ASRService: ObservableObject {
             self.isAsrReady = true
             self.isCancellingModelPreparation = false
             self.refreshWordBoostStatus()
+            // start() skips the preview loop when the model isn't ready yet. If the
+            // load finished while audio was buffering, start it now; the first chunk
+            // transcribes the whole buffered prefix, then continues incrementally.
+            if self.isRunning,
+               !self.isDictionaryTrainingCaptureActive,
+               SettingsStore.shared.selectedSpeechModel.supportsStreaming
+            {
+                DebugLogger.shared.info("Model became ready mid-recording; starting streaming preview", source: "ASRService")
+                self.startStreamingTranscription()
+            }
             self.finishModelDownloadAnalytics(operationID: operationID, outcome: .succeeded)
         } catch is CancellationError {
             self.finishModelDownloadAnalytics(operationID: operationID, outcome: .cancelled)

Sources/Fluid/Services/IdleModelUnloader.swift

@@ -0,0 +1,61 @@
+import Foundation
+
+/// Frees on-device models after a period without dictation so the app stops
+/// pinning gigabytes while idle. Recording never waits on this: the models
+/// reload in the background on the next hotkey press.
+@MainActor
+final class IdleModelUnloader {
+    static let shared = IdleModelUnloader()
+
+    private var countdown: Task<Void, Never>?
+
+    private init() {}
+
+    // MARK: - Public
+
+    /// Restart the idle countdown. Call whenever a model was just used.
+    func recordActivity() {
+        let minutes = SettingsStore.shared.modelIdleUnloadMinutes
+        guard minutes > 0 else {
+            self.cancel()
+            return
+        }
+        self.schedule(after: .seconds(minutes * 60))
+    }
+
+    /// Release both models immediately (settings button). Same busy guard as the timer.
+    func unloadNow() async {
+        await self.fire()
+    }
+
+    /// Stop the countdown while a session is active.
+    func cancel() {
+        self.countdown?.cancel()
+        self.countdown = nil
+    }
+
+    // MARK: - Private
+
+    private func schedule(after delay: Duration) {
+        self.countdown?.cancel()
+        self.countdown = Task { [weak self] in
+            guard (try? await Task.sleep(for: delay)) != nil else { return }
+            await self?.fire()
+        }
+    }
+
+    private func fire() async {
+        let asr = AppServices.shared.asr
+        let overlay = NotchContentState.shared
+        if asr.isRunningOrStarting || overlay.isProcessing || overlay.isCommandProcessing {
+            // Busy windows last seconds, not minutes, so a fixed short retry is enough.
+            DebugLogger.shared.debug("Idle unload deferred: session active", source: "IdleModelUnloader")
+            self.schedule(after: .seconds(60))
+            return
+        }
+        DebugLogger.shared.info("Idle unload: releasing models", source: "IdleModelUnloader")
+        self.countdown = nil
+        await PrivateAIIntegrationService.shared.unloadCachedRuntime(reason: "idle")
+        asr.unloadForIdle()
+    }
+}

Sources/Fluid/Services/PrivateAIIntegrationService.swift

@@ -208,6 +208,7 @@ actor PrivateAIIntegrationService {
         context: AppContext
     ) async throws -> EnhancementResult {
         try Self.validateDictationHeadroom(inputText, contextTokenLimit: runtime.contextTokenLimit)
+        defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } }
         return try await Self.provider.enhanceDictation(inputText, runtime: runtime, context: context)
     }
 
@@ -218,6 +219,7 @@ actor PrivateAIIntegrationService {
         streamHandler: PrivateAIStreamHandler?
     ) async throws -> EnhancementResult {
         try Self.validateDictationHeadroom(inputText, contextTokenLimit: runtime.contextTokenLimit)
+        defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } }
         return try await Self.provider.enhanceDictation(
             inputText,
             runtime: runtime,
@@ -242,7 +244,8 @@ actor PrivateAIIntegrationService {
         runtime: RuntimeConfiguration,
         context: AppContext
     ) async throws -> EnhancementResult {
-        try await Self.provider.rewrite(
+        defer { Task { @MainActor in IdleModelUnloader.shared.recordActivity() } }
+        return try await Self.provider.rewrite(
             inputText,
             systemPrompt: systemPrompt,
             runtime: runtime,

Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift

@@ -153,6 +153,10 @@ extension VoiceEngineSettingsView {
 
                         // Filler Words Section
                         self.fillerWordsSection
+
+                        Divider().padding(.vertical, 4)
+
+                        self.modelIdleUnloadSection
                     }
                 }
             }
@@ -856,6 +860,35 @@ extension VoiceEngineSettingsView {
         }
     }
 
+    var modelIdleUnloadSection: some View {
+        HStack(alignment: .center) {
+            VStack(alignment: .leading, spacing: 2) {
+                Text("Unload Models When Idle")
+                    .font(self.theme.typography.bodyStrong)
+                    .foregroundStyle(self.voiceEngineTitleText)
+                Text("Frees memory after a break. No need to wait on your next dictation: just start talking and the models reload while you speak.")
+                    .font(self.theme.typography.bodySmall)
+                    .foregroundStyle(self.voiceEngineSecondaryText)
+            }
+            Spacer()
+            Picker("", selection: self.$settings.modelIdleUnloadMinutes) {
+                ForEach(SettingsStore.modelIdleUnloadMinuteOptions, id: \.self) { minutes in
+                    Text(minutes == 0 ? "Never" : "\(minutes) min").tag(minutes)
+                }
+            }
+            .labelsHidden()
+            .pickerStyle(.menu)
+            .frame(width: 96)
+            .onChange(of: self.settings.modelIdleUnloadMinutes) { _, _ in
+                IdleModelUnloader.shared.recordActivity()
+            }
+            Button("Unload Now") {
+                Task { await IdleModelUnloader.shared.unloadNow() }
+            }
+            .help("Release the models right away. The next dictation reloads them while you speak.")
+        }
+    }
+
     // MARK: - Speech Model Logo View
 
     private func speechModelLogoView(for model: SettingsStore.SpeechModel) -> some View {